From dfe423869e4dbca83b7da0090d03d9b11cb0c391 Mon Sep 17 00:00:00 2001 From: Eddy van den Aker Date: Mon, 10 Mar 2025 16:10:58 +0100 Subject: [PATCH 1/2] Replaced RabbitMQ with Redis --- docker-compose.yml | 23 ++++++++--------------- training/app/config.py | 4 ++-- training/requirements.txt | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3162533..b2ebef2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,8 +52,8 @@ services: environment: - LEGACY_PLUGIN_NAME=Emotion Recognition - TRACKING_URI=http://monitoring-server:8999 - - CELERY_BROKER_URL=amqp://message-queue - - CELERY_RESULTS_BACKEND=rpc://message-queue + - CELERY_BROKER_URL=redis://message-queue:6379 + - CELERY_RESULT_BACKEND=redis://message-queue:6379 - DATASTORE_ACCESS_KEY=yZmhrURuUhaeVSUagMRa - DATASTORE_SECRET_KEY=cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba - DATASTORE_URI=datastore:9000 @@ -80,8 +80,8 @@ services: environment: - LEGACY_PLUGIN_NAME=Emotion Recognition - TRACKING_URI=http://monitoring-server:8999 - - CELERY_BROKER_URL=amqp://message-queue - - CELERY_RESULTS_BACKEND=rpc://message-queue + - CELERY_BROKER_URL=redis://message-queue:6379 + - CELERY_RESULT_BACKEND=redis://message-queue:6379 - DATASTORE_ACCESS_KEY=yZmhrURuUhaeVSUagMRa - DATASTORE_SECRET_KEY=cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba - DATASTORE_URI=datastore:9000 @@ -114,8 +114,8 @@ services: environment: - LEGACY_PLUGIN_NAME=Emotion Recognition - TRACKING_URI=http://monitoring-server:8999 - - CELERY_BROKER_URL=amqp://message-queue - - CELERY_RESULTS_BACKEND=rpc://message-queue + - CELERY_BROKER_URL=redis://message-queue:6379 + - CELERY_RESULT_BACKEND=redis://message-queue:6379 - DATASTORE_ACCESS_KEY=yZmhrURuUhaeVSUagMRa - DATASTORE_SECRET_KEY=cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba - DATASTORE_URI=datastore:9000 @@ -193,20 +193,13 @@ services: # : serves as a message broker for the CHIMP system. message-queue: restart: unless-stopped - image: rabbitmq:3.13-management + image: redis:7.4.2 container_name: message-queue - healthcheck: - test: rabbitmq-diagnosis check_port_connectivity - start_period: 30s - interval: 1m - timeout: 30s - retries: 5 networks: - public-network - backend-network ports: - - "5672:5672" - - "15672:15672" + - "6379:6379" profiles: - '' - 'gpu' diff --git a/training/app/config.py b/training/app/config.py index 3940ebf..9496e2f 100644 --- a/training/app/config.py +++ b/training/app/config.py @@ -24,8 +24,8 @@ basedir, "../datasets" ) -CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") or "amqp://localhost" -CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND") or "rpc://localhost" +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") or "redis://localhost" +CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND") or "redis://localhost" DATASTORE_URI = os.environ.get("DATASTORE_URI") or "localhost:9000" DATASTORE_ACCESS_KEY = os.environ.get("DATASTORE_ACCESS_KEY") or "" diff --git a/training/requirements.txt b/training/requirements.txt index a5d74eb..3e9c524 100644 --- a/training/requirements.txt +++ b/training/requirements.txt @@ -3,6 +3,6 @@ flask-cors==4.0.0 gevent==24.2.1 gunicorn==21.2.0 python-dotenv==1.0.1 -celery==5.4.0 +celery[redis]==5.4.0 mlflow==2.11.3 minio==7.2.7 \ No newline at end of file From 5dc3ea6f06f668ff7c66268354a710811bc8e409 Mon Sep 17 00:00:00 2001 From: Eddy van den Aker Date: Mon, 10 Mar 2025 16:19:43 +0100 Subject: [PATCH 2/2] Updated readme --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6c2a45e..c08355a 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ a continual AI pipeline. It was originally part of a bachelor thesis project by called "front-end", it also contains a back-end component in the form of a Flask API that communicates with the rest of the CHIMP system). - **mlflow-tracking:** MLFlow is used to track the different models and log metrics for these models. +- **minio-datastore:** Minio is used as a centralized datastore for storing and managing datasets. +- **message-queue:** Redis is used as a message queue to coordinate jobs between the APIs and the workers. The table below shows which ports are used by the different components. The "Local Dev Port" is the port that is used when a component is run on the host instead of in Docker. @@ -22,6 +24,7 @@ The table below shows which ports are used by the different components. The "Loc | ml-frontend | 5252 | 5252 | 8000 | | mlflow-tracking | n.a. | 8999 | 8999 | | minio-datastore | n.a. | 9000, 9001 (web) | 9000, 9001 | +| message-queue | n.a. | 6379 | 6379 | ```mermaid graph RL @@ -29,10 +32,11 @@ graph RL subgraph Application afe[Emotion recognition front-end
- HTML/CSS/JS] --> abe[Emotion recognition back-end
- Python/Flask]; end - abe --> tapi[Training service API
- Python/Flask
- TalosML/Tensorflow]; + abe --> tapi[Training service API
- Python/Flask]; abe --> srv[Serving_api service
- Python/Flask]; subgraph Services - tapi --> twork[Training service worker
- Python]; + tapi --> mq[Redis message queue]; + mq --> twork[Training service worker
- Python]; twork --> mlf[Tracking
- MLFlow]; srv --> mlf; mlf --> db[Database
- SQLite]; @@ -69,6 +73,8 @@ run the external services, such as MLFlow and RabbitMQ (e.g. when running the Py follows: `docker-compose --profile services up -d`. Please note that the `--profile ` should come before any further commands, such as `up -d`. On some setups, to use the default profile (denoted by an empty string, or ''), it should be explicitly included in the call, for example: `docker-compose --profile '' up -d`. +To monitor and work with the Redis message queue during development, a tool like "Another Redis Desktop Manager" can be used. + ### Local development setup (on host outside of Docker) To run the Python/Flask based CHIMP components outside of Docker (for example, when you want to run a component with a debugger attached), you can use the following steps: - Run the MLFlow service in Docker using `docker-compose --profile services up -d`