Merge pull request #12 from micado-scale/develop

Better app clean-up on boot

Author: jaydesl

Dockerfile

@@ -21,4 +21,4 @@ ENV MICADO_DIR=/etc/micado/
 ENV MICADO_VERS=0.12.1
 ENV MICADO_SPEC=/etc/eec/micado_spec.yaml
 
-ENTRYPOINT ["gunicorn", "micado_eec.micado:app", "--bind", "0.0.0.0:5000", "--timeout", "1000", "--workers", "5"]
+ENTRYPOINT ["gunicorn", "micado_eec.micado:app", "-c", "micado_eec/gunicorn_conf.py"]

micado_eec/gunicorn_conf.py

@@ -0,0 +1,32 @@
+import redis
+from micado_eec.handle_micado import HandleMicado, r
+
+bind = '0.0.0.0:5000'
+
+workers = 5
+timeout = 30
+
+def on_starting(server):
+    server.log.info("Cleaning-up uninitialised apps...")
+
+    for thread_id in r.keys():
+        try:
+            micado = r.hget(thread_id, "micado_id")
+            updated = r.hget(thread_id, "last_app_refresh")
+        except redis.exceptions.ResponseError:
+            raise TypeError("Database corrupt - contains wrong data types.")
+        
+        if not micado:
+            server.log.info(f"App {thread_id} has no MiCADO, removing from DB.")
+            r.delete(thread_id)
+            continue
+
+        if not updated:
+            server.log.info(f"App {thread_id} has MiCADO, attempting abort.")
+            r.expire(thread_id, 60)
+            thread = HandleMicado(thread_id, f"process_{thread_id}")
+            thread.start()
+            thread.abort()
+
+    server.log.info("App clean-up done.")
+

micado_eec/handle_micado.py

@@ -3,11 +3,10 @@
 import base64
 import threading
 import time
-from datetime import datetime
 from typing import Optional
 
+import redis
 import ruamel.yaml as yaml
-from redis import StrictRedis
 from micado import MicadoClient
 
 from .utils import base64_to_yaml, load_yaml_file
@@ -43,7 +42,10 @@
 APP_ID_PARAM = "app_id"
 APP_PARAMS = "params"
 
-r = StrictRedis("redis", decode_responses=True)
+try:
+    r = redis.StrictRedis("redis", decode_responses=True)
+except redis.exceptions.ConnectionError as e:
+    raise ConnectionError(f"Cannot connect to Redis: {e}")
 if not r.ping():
     raise ConnectionError("Cannot connect to Redis")
 
@@ -86,7 +88,7 @@ def __init__(
         self.name = name
 
         if not r.hgetall(threadID):
-            r.hset(threadID, "submit_time", datetime.now().timestamp())
+            r.hset(threadID, "submit_time", time.time())
             self.set_status()
 
         self.artefact_data = artefact_data or {}
@@ -161,6 +163,7 @@ def run(self):
 
         # Wait for abort
         while True:
+            r.hset(self.threadID, "last_app_refresh", time.time())
             if self._is_aborted():
                 self.abort()
                 break

micado_eec/micado.py

@@ -1,33 +1,34 @@
 import json
 import uuid
 import tempfile
+import time
 from datetime import datetime
 
 import redis
 from flask import jsonify, Flask, request
 from werkzeug.exceptions import BadRequest, NotFound
 
-from .handle_micado import HandleMicado
+from .handle_micado import HandleMicado, r
 from .utils import base64_to_yaml, is_valid_adt, get_adt_inputs, get_csar_inputs, file_to_json
 
-r = redis.StrictRedis("redis", decode_responses=True)
-if not r.ping():
-    raise ConnectionError("Cannot connect to Redis")
+app = Flask(__name__)
+app.debug = True
 
 for thread_id in r.keys():
     try:
-        micado_id = r.hget(thread_id, "micado_id")
+        updated = r.hget(thread_id, "last_app_refresh")
     except redis.exceptions.ResponseError:
         raise TypeError("Database corrupt - contains wrong data types.")
-    
-    if not micado_id:
-        r.delete(thread_id)
+
+    if not updated:
         continue
-    thread = HandleMicado(thread_id, f"process_{thread_id}")
-    thread.start()
+    
+    age_seconds = time.time() - float(updated)
+    if age_seconds > 30:
+        r.hset(thread_id, "last_app_refresh", time.time())
+        thread = HandleMicado(thread_id, f"process_{thread_id}")
+        thread.start()
 
-app = Flask(__name__)
-app.debug = True
 
 @app.errorhandler(BadRequest)
 def handle_generic_bad_request(error):