Data versioning tests

.env

@@ -9,3 +9,9 @@ MESSAGE_BROKER="localhost"
 DATASTORE_ACCESS_KEY="yZmhrURuUhaeVSUagMRa"
 DATASTORE_SECRET_KEY="cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba"
 
+DATABASE_URI="localhost"
+DATABASE_NAME="chimp_database"
+DATABASE_USER="chimp_user"
+DATABASE_PASSWORD="chimp_password"
+
+REDIS_HOST="localhost"

.gitignore

@@ -1,7 +1,11 @@
 ### Custom ignores
+logs/*
+!logs/.gitkeep
 docker-data/monitoring/mlruns/
 docker-data/monitoring/mlflow.db
 docker-data/datastore/*
+docker-data/postgres/*
+docker-data/files/*
 experimentation/
 ml-frontend/logs
 serving/mlruns
@@ -25,6 +29,7 @@ training/app/plugins/*
 !training/app/plugins/example2.py
 !training/app/plugins/game_art_style_detector
 !training/app/plugins/emotion_recognition
+!training/app/plugins/active_learning
 training/datasets/*
 
 ### Generic ignores

.vscode/launch.json

@@ -1,41 +1,19 @@
+// TODO: Update launch profiles to allow local debugging when necessary. The to-do can be part a larger refactoring effort.
 {
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Experimentation",
+            "name": "Debug Celery Worker",
             "type": "debugpy",
             "request": "launch",
-            "program": "${workspaceFolder}/experimentation/main.py",
+            "program": "celery", // Adjust this path to your virtual environment
+            "args": [
+                "-A", "manage:celery_app", 
+                "worker", 
+                "--pool=solo"
+            ],
             "console": "integratedTerminal",
-            "python": "${workspaceFolder}/experimentation/env/Scripts/python",
-            "cwd": "${workspaceFolder}/experimentation",
-            "gevent": true,
-        },
-        {
-            "name": "Serving",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/serving/main.py",
-            "console": "integratedTerminal",
-            "python": "${workspaceFolder}/serving/env/Scripts/python",
-            "cwd": "${workspaceFolder}/serving",
-            "gevent": true,
-        },
-        {
-            "name": "ML-Frontend",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/ml-frontend/main.py",
-            "console": "integratedTerminal",
-            "python": "${workspaceFolder}/ml-frontend/env/Scripts/python",
-            "cwd": "${workspaceFolder}/ml-frontend",
-            "gevent": true,
-        },
-    ],
-    "compounds": [
-        {
-            "name": "Run all",
-            "configurations": ["Experimentation", "Serving", "ML-Frontend"]
+            "justMyCode": false,
         }
     ]
-} 
+}

README.md

@@ -50,23 +50,11 @@ graph RL
 ## Development setup
 
 1. Fork this repository and clone the fork to your local machine.
-2. Install Docker and Docker Compose.
-3. [OPTIONAL] Download the docker-data.zip [here](https://drive.google.com/file/d/1ILWH1V4SOSCMbYuRBi4xylDAPRm3WG0B/view?usp=sharing) and extract it into the docker-data folder
-4. Run `docker-compose up` in the root of the repository
-   - On some systems you need to run `docker-compose --profile '' up` instead to run the default profile
-5. Open your browser and navigate to the datastore dashboard on `http://localhost:9001`
-6. On the datastore dashboard, go to "Access Keys" and click on "Create access key"
-7. Add the access key and secret key created here to the `.env` file in the root of the CHIMP project
-8. Open your browser and navigate to `http://localhost:5252`
-9. Allow the CHIMP front-end to use your webcam.
-10. Click on browse and select the calibration-data.zip (which can be downloaded on Teams).
-11. Click on "Train New Model" and wait for the training to finish.
-    - You can monitor the training progress in the terminal by running `docker logs -f experimentation-server`.
-12. Once the training is finished, navigate to `http://localhost:8999` (MLFlow).
-13. Click on "Models" and select "onnx emotion model".
-14. Click on the last model and set the model to "Production" by changing the "stage" setting at the top of the screen.  
-    - [OPTIONAL] Click on the second to last model and sit the model to "Staging" by changing the "stage" setting at the top of the screen.
-15. Refresh the CHIMP front-end. You should now see percentages of the detected emotions.
+2. Install Docker and Docker Compose. Use this [manual](https://docs.docker.com/desktop/features/wsl/) for Windows install.
+3. Run `docker-compose build` in the root of the repository and on success `docker-compose up`
+4. [OPTIONAL] Initialize the database. Create a Python venv. Navigate to folder `initialize_empty_CHIMP` and execute `pip3 install --no-cache-dir -r initialize_requirements.txt -c constraints.txt` and on success execute  `initialize_populate_empty_CHIMP.ipynb`
+5. Open your browser and navigate to `http://localhost:5252`
+6. Allow the CHIMP front-end to use your webcam.
 
 To run the GPU enabled version of CHIMP use the "gpu" profile as follows: `docker-compose --profile gpu up -d`. To only
 run the external services, such as MLFlow and RabbitMQ (e.g. when running the Python code directly on the host), use the "services" profile as

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
     # application
     # : sandbox application that enables emotion recognition from facial features
@@ -17,6 +15,7 @@ services:
             - UID=1000
             - GID=1000
             - EXPERIMENT_NAME=onnx_emo_datastore
+            - MODEL_NAME=onnx_emo_datastore
             - MODEL_INFERENCE_URL=http://serving-api:8000
             - TRAINING_SERVER_URL=http://training-api:8000
         #volumes:
@@ -54,14 +53,22 @@ services:
             - TRACKING_URI=http://monitoring-server:8999
             - CELERY_BROKER_URL=redis://message-queue:6379
             - CELERY_RESULT_BACKEND=redis://message-queue:6379
+            - REDIS_HOST=message-queue
+            - REDIS_PORT=6379
             - DATASTORE_ACCESS_KEY=yZmhrURuUhaeVSUagMRa
             - DATASTORE_SECRET_KEY=cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba
             - DATASTORE_URI=datastore:9000
+           # - DATABASE_URI=postgres-db:5432
+            - DATABASE_URI=postgres-db
+            - DATABASE_USER=chimp_user
+            - DATABASE_PASSWORD=chimp_password
+            - DATABASE_NAME=chimp_database
         volumes:
             - ${DATASETS_FOLDER:-./training/datasets}:/usr/src/app/datasets
-            - ./training/app/plugins:/usr/src/app/app/plugins
+            - ./training:/usr/src/app
         depends_on:
             - monitoring-server
+            - postgres-db
         profiles:
             - ''
             - 'gpu'
@@ -82,14 +89,23 @@ services:
             - TRACKING_URI=http://monitoring-server:8999
             - CELERY_BROKER_URL=redis://message-queue:6379
             - CELERY_RESULT_BACKEND=redis://message-queue:6379
+            - REDIS_HOST=message-queue
+            - REDIS_PORT=6379
             - DATASTORE_ACCESS_KEY=yZmhrURuUhaeVSUagMRa
             - DATASTORE_SECRET_KEY=cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba
             - DATASTORE_URI=datastore:9000
+           # - DATABASE_URI=postgres-db:5432
+            - DATABASE_URI=postgres-db
+            - DATABASE_USER=chimp_user
+            - DATABASE_PASSWORD=chimp_password
+            - DATABASE_NAME=chimp_database
+            - TRAINING_SERVER_URL=http://training-api:8000
         volumes:
             - ${DATASETS_FOLDER:-./training/datasets}:/usr/src/app/datasets:ro
-            - ./training/app/plugins:/usr/src/app/app/plugins
+            - ./training:/usr/src/app
         depends_on:
             - monitoring-server
+            - postgres-db
         profiles:
             - ''
             - 'worker-only'
@@ -116,14 +132,23 @@ services:
             - TRACKING_URI=http://monitoring-server:8999
             - CELERY_BROKER_URL=redis://message-queue:6379
             - CELERY_RESULT_BACKEND=redis://message-queue:6379
+            - REDIS_HOST=message-queue
+            - REDIS_PORT=6379
             - DATASTORE_ACCESS_KEY=yZmhrURuUhaeVSUagMRa
             - DATASTORE_SECRET_KEY=cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba
             - DATASTORE_URI=datastore:9000
+          #  - DATABASE_URI=postgres-db:5432
+            - DATABASE_URI=postgres-db
+            - DATABASE_NAME=chimp_database
+            - DATABASE_USER=chimp_user
+            - DATABASE_PASSWORD=chimp_password
+            - TRAINING_SERVER_URL=http://training-api:8000
         volumes:
             - ${DATASETS_FOLDER:-./training/datasets}:/usr/src/app/datasets
-            - ./training/app/plugins:/usr/src/app/app/plugins
+            - ./training:/usr/src/app
         depends_on:
             - monitoring-server
+            - postgres-db
         profiles:
             - 'gpu'
             - 'worker-only-gpu'
@@ -231,6 +256,68 @@ services:
             - 'gpu'
             - 'services'
 
+    # postgresql
+    # : database for managed datasets        
+    postgres-db:
+        image: postgres:17
+        container_name: chimp-postgres
+        restart: unless-stopped
+        networks:
+            - backend-network
+        ports:
+            - "5432:5432"
+        environment:
+            POSTGRES_USER: chimp_user
+            POSTGRES_PASSWORD: chimp_password
+            POSTGRES_DB: chimp_database
+        volumes:
+            - ./docker-data/postgres:/var/lib/postgresql/data
+        profiles:
+            - ''
+            - 'gpu'
+            - 'services'
+
+
+    training-beat:
+        container_name: training-beat
+        build:
+            context: ./training
+            dockerfile: Dockerfile.worker
+        entrypoint: [""]
+        command: >
+          sh -c "celery -A manage:celery_app beat --loglevel=info"
+        volumes:
+            - ./training:/usr/src/app
+        environment:
+            - CELERY_BROKER_URL=redis://message-queue:6379
+            - CELERY_RESULT_BACKEND=redis://message-queue:6379
+            - REDIS_HOST=message-queue
+            - REDIS_PORT=6379
+            - DATASTORE_URI=datastore:9000
+            - DATASTORE_ACCESS_KEY=yZmhrURuUhaeVSUagMRa
+            - DATASTORE_SECRET_KEY=cnk0OxGuIgVx4La0prNaWUv7JpriCnxZfq2417ba
+            - TRACKING_URI=http://monitoring-server:8999
+            - DATABASE_URI=postgres-db
+            - DATABASE_USER=chimp_user
+            - DATABASE_PASSWORD=chimp_password
+            - DATABASE_NAME=chimp_database
+        depends_on:
+            message-queue:
+                condition: service_started
+            training-api:
+                condition: service_started
+            datastore:
+                condition: service_started
+        networks:
+            - backend-network
+        profiles:
+            - ''
+            - 'gpu'
+            - 'worker-only'
+
+
+
+
 networks:
     public-network:
         name: 'chimp-public-network'