Add in FastAPI as a benchmark

* Introduce FastAPI as a new benchmark
* Add a new gunicorn benchmark for ninja
* Remove graph as it becomes stale too easily
* Sort imports
* Remove unused code from run_test.py
* Intentionally unfreeze the requirements.txt - this is to allow for the benchmark to run with the latest versions.

Author: jensenbox

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.2
+FROM python:3
 
 ENV PYTHONUNBUFFERED=1 PIP_DISABLE_PIP_VERSION_CHECK=on
 
@@ -9,4 +9,5 @@ COPY common_django_settings.py /common_django_settings.py
 COPY app_drf /app_drf
 COPY app_flask_marshmallow /app_flask_marshmallow
 COPY app_ninja /app_ninja
+COPY app_fastapi /app_fastapi
 COPY network_service.py /network_service.py

README.md

@@ -16,4 +16,15 @@ python run_test.py
 
 ### Results
 
-![Django Ninja REST Framework](img/results.png)
+```text
+Framework               :        1        2        4        8       16       32       64
+drf_uwsgi               :     9.37    18.95    37.15    70.35   123.77   209.73   234.63
+fastapi_gunicorn        :     9.57    17.52    36.13    52.74   113.62   208.13   262.85
+fastapi_uvicorn         :     9.50    17.94    31.52    55.63   118.94   167.54   194.12
+flask_marshmallow_uwsgi :     9.53    18.75    37.04    72.74   131.16   229.92   326.55
+ninja_gunicorn          :   240.49   278.57   317.09   309.32   285.71   246.44   256.98
+ninja_uvicorn           :   265.31   289.09   280.00   305.05   247.62   237.71   212.58
+ninja_uwsgi             :     9.29    18.56    36.04    68.98   128.32   180.89   182.31
+Columns indicate the number of workers in each run
+Values are in requests per second - higher is better.
+```

app_fastapi/main.py

@@ -0,0 +1,48 @@
+import datetime
+from typing import List
+
+import requests
+from fastapi import FastAPI
+from pydantic import BaseModel, Field
+
+
+class Location(BaseModel):
+    latitude: float | None = None
+    longitude: float | None = None
+
+
+class Skill(BaseModel):
+    subject: str
+    subject_id: int
+    category: str
+    qual_level: str
+    qual_level_id: int
+    qual_level_ranking: float = 0
+
+
+class Model(BaseModel):
+    id: int
+    client_name: str = Field(max_length=255)
+    sort_index: float
+    client_phone: str | None = Field(None, max_length=255)
+    location: Location
+    contractor: int | None = Field(None, gt=0)
+    upstream_http_referrer: str | None = Field(None, max_length=1023)
+    grecaptcha_response: str = Field(min_length=20, max_length=1000)
+    last_updated: datetime.datetime | None
+    skills: List[Skill]
+
+
+app = FastAPI()
+
+
+@app.post("/api/create")
+async def create(data: Model):
+    return {"success": True}, 201
+
+
+@app.get("/api/iojob")
+async def iojob():
+    response = requests.get('http://network_service:8000/job')
+    assert response.status_code == 200
+    return {"success": True}, 200

docker-compose.yml

@@ -30,6 +30,21 @@ services:
     working_dir: /app_ninja
     command: uvicorn djninja.asgi:application --host 0.0.0.0 --workers ${WORKERS}
 
+  ninja_gunicorn:
+    <<: *common
+    working_dir: /app_ninja
+    command: gunicorn djninja.asgi:application --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 --workers ${WORKERS}
+
+  fastapi_gunicorn:
+    <<: *common
+    working_dir: /app_fastapi
+    command: gunicorn main:app --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 --workers ${WORKERS}
+
+  fastapi_uvicorn:
+    <<: *common
+    working_dir: /app_fastapi
+    command: uvicorn main:app --host 0.0.0.0 --workers ${WORKERS}
+
   network_service:
     build: .
     command: python network_service.py > /dev/null

img/results.png

@@ -1,18 +1,17 @@
-Django==3.1b1
+Django
 
-django-ninja==0.2.0
-pydantic==1.5.1
+django-ninja
+fastapi[standard]
+djangorestframework
+Flask
 
-djangorestframework==3.11.0
+pydantic
+marshmallow
+sanic
 
-Flask==1.1.2
-marshmallow==3.6.1
-
-sanic==20.6.3
-
-uvicorn==0.11.5
-uWSGI==2.0.19.1
-
-aiohttp==3.6.2
+uvicorn[standard]
+gunicorn
+uWSGI
 
+aiohttp
 requests

requirements.txt

@@ -1,20 +1,16 @@
-import time
-import re
 import os
-import sys
+import re
 import subprocess
-
-
-C1_FRAMEWORKS = [
-    'flask_marshmallow_uwsgi',
-    'drf_uwsgi',
-    'ninja_uwsgi',
-]
+import time
 
 CONCURRENT_FRAMEWORKS = [
     'flask_marshmallow_uwsgi',
     'drf_uwsgi',
+    'ninja_uwsgi',
+    'ninja_gunicorn',
     'ninja_uvicorn',
+    'fastapi_gunicorn',
+    'fastapi_uvicorn'
 ]
 
 
@@ -45,29 +41,21 @@ def benchmark(url, concurency, count, payload=None):
 
 
 def parse_benchmark(output: str):
-    # print(output)
     rps = re.findall(r'Requests per second: \s+(.*?)\s', output)[0]
     p50 = re.findall(r'\s+50%\s+(\d+)', output)[0]
     p99 = re.findall(r'\s+99%\s+(\d+)', output)[0]
-    return (rps, p50, p99)
-    # print()
-    # re.findall(r'')
+    return rps, p50, p99
 
 
 def preheat():
     benchmark('http://127.0.0.1:8000/api/create', 1, 5, 'payload.json')
     benchmark('http://127.0.0.1:8000/api/iojob', 1, 5)
 
 
-def run_c1_test():
-    return benchmark('http://127.0.0.1:8000/api/create', 1, 1000, 'payload.json')
-
-
-WORKERS_CASES = list(range(1, 25))  # [14, 15, 16, 17, 18, 19, 20]
+WORKERS_CASES = [1, 2, 4, 8, 16, 32, 64]
 
 
 def test_concurrent(name):
-
     results = {}
     for workers in WORKERS_CASES:
         with FrameworkService(name, workers):
@@ -88,13 +76,17 @@ def main():
     print('Framework               :', end='')
     for w in WORKERS_CASES:
         print(f'{w:>9}', end='')
-    print('')
+
+    print()
+
     for framework, results in sorted(results.items()):
         print(f'{framework:<23} :', end='')
         for w in WORKERS_CASES:
             print(f'{results[w][0]:>9}', end='')
-        print('')
+        print()
 
+    print('Columns indicate the number of workers in each run')
+    print('Values are in requests per second - higher is better.')
 
 if __name__ == '__main__':
     main()