Merge pull request #6 from Never-Over/celery-django-settings

Refactor Django Celery configuration

Author: emdoyle

bridge/cli/init/render.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from typing import Optional
 
 from pydantic import BaseModel
 
@@ -10,13 +11,31 @@
     start_worker_sh_template,
 )
 from bridge.console import console
+from bridge.framework.base import Framework
 from bridge.utils.filesystem import (
     resolve_dot_bridge,
     resolve_project_dir,
     set_executable,
 )
 
 
+def detect_framework() -> Framework:
+    # TODO: auto-detect framework (assuming Django)
+    return Framework.DJANGO
+
+
+def detect_django_settings_module(project_name: str = "") -> str:
+    settings_path = f"{project_name}/settings.py"
+    if os.path.exists(settings_path):
+        return f"{project_name}.settings"
+    else:
+        # TODO: validate input
+        return console.input(
+            "Please provide the path to your"
+            " Django settings module (ex: myapp.settings):\n> "
+        )
+
+
 def detect_application_callable(project_name: str = "") -> str:
     wsgi_path = f"{project_name}/wsgi.py"
     asgi_path = f"{project_name}/asgi.py"
@@ -25,34 +44,48 @@ def detect_application_callable(project_name: str = "") -> str:
     elif os.path.exists(asgi_path):
         return f"{project_name}.asgi:application"
     else:
+        # TODO: validate input
         return console.input(
             "Please provide the path to your WSGI or ASGI application callable "
             "(ex: myapp.wsgi:application):\n> "
         )
 
 
+class DjangoConfig(BaseModel):
+    settings_module: str
+
+
 class RenderPlatformInitConfig(BaseModel):
+    framework: Framework = Framework.DJANGO
     project_name: str
     app_path: str
     bridge_path: str
+    django_config: Optional[DjangoConfig] = None
 
 
 def build_render_init_config() -> RenderPlatformInitConfig:
     # NOTE: this method may request user input directly on the CLI
     #   to determine configuration when it cannot be auto-detected
+    framework = detect_framework()
     project_name = resolve_project_dir().name
     app_path = detect_application_callable(project_name=project_name)
-    # May be able to remove this call since all scripts, YAML have moved to root
     bridge_path = resolve_dot_bridge()
-    return RenderPlatformInitConfig(
+    init_config = RenderPlatformInitConfig(
         project_name=project_name, app_path=app_path, bridge_path=str(bridge_path)
     )
 
+    # Provide framework-specific configuration
+    if framework == Framework.DJANGO:
+        settings_module = detect_django_settings_module(project_name=project_name)
+        init_config.django_config = DjangoConfig(settings_module=settings_module)
+
+    return init_config
+
 
 def initialize_render_platform(config: RenderPlatformInitConfig):
     build_sh_path = Path("./render-build.sh")
     with build_sh_path.open(mode="w") as f:
-        f.write(build_sh_template())
+        f.write(build_sh_template(framework=config.framework))
     set_executable(build_sh_path)
 
     start_sh_path = Path("./render-start.sh")
@@ -62,13 +95,17 @@ def initialize_render_platform(config: RenderPlatformInitConfig):
 
     start_worker_sh_path = Path("./render-start-worker.sh")
     with start_worker_sh_path.open(mode="w") as f:
-        f.write(start_worker_sh_template())
+        f.write(start_worker_sh_template(framework=config.framework))
     set_executable(start_worker_sh_path)
 
     with open("render.yaml", "w") as f:
         f.write(
             render_yaml_template(
+                framework=config.framework,
                 service_name=config.project_name,
                 database_name=f"{config.project_name}_db",
+                django_settings_module=config.django_config.settings_module
+                if config.django_config
+                else "",
             )
         )

bridge/cli/init/templates/render__yaml.py

@@ -1,3 +1,4 @@
+from bridge.framework.base import Framework
 from bridge.utils.sanitize import sanitize_postgresql_identifier
 
 template = """services:
@@ -10,8 +11,6 @@
     envVars:
       - key: BRIDGE_PLATFORM
         value: render
-      - key: BRIDGE_PROJECT_NAME
-        value: {service_name}
       - key: SECRET_KEY
         generateValue: true
       - key: WEB_CONCURRENCY
@@ -39,8 +38,8 @@
     envVars:
       - key: BRIDGE_PLATFORM
         value: render
-      - key: BRIDGE_PROJECT_NAME
-        value: {service_name}
+      - key: DJANGO_SETTINGS_MODULE
+        value: {django_settings_module}
       - key: SECRET_KEY
         generateValue: true
       - key: TASK_CONCURRENCY
@@ -66,12 +65,29 @@
 
 
 def render_yaml_template(
-    service_name: str, database_name: str = "", database_user: str = ""
+    framework: Framework,
+    service_name: str,
+    database_name: str = "",
+    database_user: str = "",
+    django_settings_module: str = "",
 ) -> str:
+    if framework != Framework.DJANGO:
+        # TODO: use a real templating engine to allow flexibility across frameworks
+        raise NotImplementedError(
+            f"Unsupported framework for Render platform: {framework}"
+        )
+
+    if not django_settings_module:
+        raise ValueError(
+            "Failed to template render.yaml:"
+            " DJANGO_SETTINGS_MODULE is required for Django projects"
+        )
+
     database_name = database_name or service_name
     database_user = database_user or service_name
     return template.format(
         service_name=service_name,
         database_name=sanitize_postgresql_identifier(database_name),
         database_user=sanitize_postgresql_identifier(database_user),
+        django_settings_module=django_settings_module,
     )

bridge/cli/init/templates/render_build__sh.py

@@ -1,3 +1,5 @@
+from bridge.framework.base import Framework
+
 template = """#!/usr/bin/env bash
 # Exit on error
 set -o errexit
@@ -16,6 +18,9 @@
 """
 
 
-def build_sh_template() -> str:
-    # TODO: support other package managers, frameworks, etc.
+def build_sh_template(framework: Framework) -> str:
+    if framework != Framework.DJANGO:
+        raise NotImplementedError(
+            f"Unsupported framework for Render platform: {framework}"
+        )
     return template.format()

bridge/cli/init/templates/render_start_worker__sh.py

@@ -1,7 +1,13 @@
+from bridge.framework.base import Framework
+
 template = """#!/usr/bin/env bash
-celery -A bridge.service.celery worker -l INFO --concurrency="${{TASK_CONCURRENCY:-4}}"
+celery -A bridge.service.django_celery worker -l INFO --concurrency="${{TASK_CONCURRENCY:-4}}"
 """
 
 
-def start_worker_sh_template() -> str:
+def start_worker_sh_template(framework: Framework) -> str:
+    if framework != Framework.DJANGO:
+        raise NotImplementedError(
+            f"Unsupported framework for Render platform: {framework}"
+        )
     return template.format()

bridge/framework/base.py

@@ -1,5 +1,6 @@
 import os
 from abc import ABC, abstractmethod
+from enum import Enum
 from typing import Any
 
 import docker
@@ -9,7 +10,15 @@
 from bridge.service.redis import RedisConfig, RedisService
 
 
+class Framework(Enum):
+    DJANGO = "django"
+    FLASK = "flask"
+    FASTAPI = "fastapi"
+
+
 class FrameWorkHandler(ABC):
+    FRAMEWORK: Framework = NotImplemented
+
     def __init__(
         self,
         project_name: str,

bridge/framework/django.py

@@ -4,13 +4,15 @@
 from typing import Any
 
 from bridge.console import log_task, log_warning
-from bridge.framework.base import FrameWorkHandler
+from bridge.framework.base import Framework, FrameWorkHandler
 from bridge.platform import Platform
 from bridge.platform.postgres import build_postgres_environment
 from bridge.platform.redis import build_redis_environment
 
 
 class DjangoHandler(FrameWorkHandler):
+    FRAMEWORK = Framework.DJANGO
+
     def is_remote(self) -> bool:
         # Django's DEBUG mode should be disabled in production,
         # so we use it to differentiate between running locally
@@ -112,21 +114,22 @@ def configure_staticfiles(self, platform: Platform):
                     middleware.insert(0, "whitenoise.middleware.WhiteNoiseMiddleware")
 
     def configure_worker(self, platform: Platform) -> None:
+        # This will make sure the app is always imported when
+        # Django starts so that shared_task will use this app.
+        from bridge.service.django_celery import app  # noqa: F401 type: ignore
+
         environment = build_redis_environment(platform)
         self.framework_locals["CELERY_BROKER_URL"] = environment.url
         self.framework_locals["CELERY_RESULT_BACKEND"] = environment.url
 
     def start_local_worker(self) -> None:
-        # Confirm we are in a `runserver` command
-        if "runserver" in sys.argv or "runserver_plus" in sys.argv:
-            # This will make sure the app is always imported when
-            # Django starts so that shared_task will use this app.
-            from bridge.service.celery import app  # noqa: F401 type: ignore
-
+        # Confirm we are in a command which expects Celery to be available
+        expected_command_args = {"runserver", "runserver_plus", "shell", "shell_plus"}
+        if set(sys.argv) & expected_command_args:
             with log_task("Starting local worker", "Local worker started"):
                 try:
                     subprocess.run(
-                        ["celery", "-A", "bridge.service.celery", "status"],
+                        ["celery", "-A", "bridge.service.django_celery", "status"],
                         check=True,
                         stdout=subprocess.DEVNULL,
                         stderr=subprocess.STDOUT,
@@ -137,7 +140,7 @@ def start_local_worker(self) -> None:
                         [
                             "celery",
                             "-A",
-                            "bridge.service.celery",
+                            "bridge.service.django_celery",
                             "worker",
                             "-l",
                             "INFO",

bridge/service/celery.py

@@ -0,0 +1,25 @@
+import os
+
+from celery import Celery
+
+if "DJANGO_SETTINGS_MODULE" not in os.environ:
+    raise ValueError(
+        "DJANGO_SETTINGS_MODULE must be set in the environment to run Celery"
+    )
+
+project_name = os.environ["DJANGO_SETTINGS_MODULE"].split(".")[0]
+app = Celery(project_name)
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True, ignore_result=True)
+def debug_task(self):
+    print(f"Request: {self.request!r}")

bridge/service/django_celery.py

@@ -1,6 +1,6 @@
 [project]
 name = "python-bridge"
-version = "0.0.20"
+version = "0.0.21"
 authors = [
   { name="Caelean Barnes", email="[email protected]" },
   { name="Evan Doyle", email="[email protected]" },

pyproject.toml

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-celery -A bridge.service.celery worker -l INFO --concurrency="${TASK_CONCURRENCY:-4}"
+celery -A bridge.service.django_celery worker -l INFO --concurrency="${TASK_CONCURRENCY:-4}"