feat: add jobs status to metrics

Author: gbonnefille

pesto-cli/integrationtests/tests-all.py

@@ -111,7 +111,7 @@ def check_build(use_ssl=False):
         service.run()
         time.sleep(5)
         endpoint = EndpointManager(server_url=service.server_url)
-        metrics = endpoint.metrics
+        metrics = endpoint.metrics()
         print("Metrics:")
         print(metrics)
         describe = endpoint.describe

pesto-cli/pesto/common/testing/endpoint_manager.py

@@ -12,7 +12,6 @@ class EndpointManager:
     def __init__(self, server_url):
         self.server_url = server_url
         self._describe = None
-        self._metrics = None
 
     @property
     def is_alive(self) -> bool:
@@ -23,12 +22,9 @@ def is_alive(self) -> bool:
         except:
             return False
 
-    @property
     def metrics(self) -> str:
-        if not self._metrics:
-            response = requests.get(url="{}/api/v1/metrics".format(self.server_url), verify=False)
-            self._metrics = response.text
-        return self._metrics
+        response = requests.get(url="{}/api/v1/metrics".format(self.server_url), verify=False)
+        return response.text
 
     @property
     def metrics_available(self) -> bool:

pesto-cli/pesto/common/testing/test_runner.py

@@ -73,7 +73,7 @@ def run_all(self, test_resources_path: Path):
 
             all_results = dict()
 
-            metrics = service_tester.endpoint_manager.metrics
+            metrics = service_tester.endpoint_manager.metrics()
             logger.info("Metrics:")
             logger.info(metrics)
 
@@ -113,6 +113,16 @@ def run_all(self, test_resources_path: Path):
 
                 self._serialize_response(response_path, response)
 
+                # Metrics after one more test
+                metrics = service_tester.endpoint_manager.metrics()
+                logger.info("Metrics after one more test:")
+                logger.info(metrics)
+
+            # Metrics after all tests
+            metrics = service_tester.endpoint_manager.metrics()
+            logger.info("Metrics after all tests:")
+            logger.info(metrics)
+
         with open(self._tmp_test_resources / "results.json", "w") as f:
             json.dump(all_results, f, indent=2)
 

pesto-cli/pesto/ws/v1.py

@@ -62,13 +62,22 @@ def health() -> Response:
     return Response(content='OK',status_code=200,media_type='text/plain')
 
 @v1.get('/metrics')
-def metrics() -> Response:
+def metrics(request: Request) -> Response:
     processing_version = DescribeService.compute_version().get("version", "unknown")
     metrics = (
         '# HELP pesto_instance_info Information about the Pesto instance\n'
         '# TYPE pesto_instance_info gauge\n'
         f'pesto_instance_info{{pesto_version="{PESTO_VERSION}",processing_version="{processing_version}"}} 1\n'
+        '# HELP pesto_jobs_total Information about the jobs\n'
+        '# TYPE pesto_jobs_total gauge\n'
     )
+    url_root = _get_url_root(request)
+    result = JobListService().job_list(url_root)
+    for job_id in result.keys():
+        job_status = JobStatusService(url_root, job_id).get_status()
+        status = job_status.get('status')
+        progress = job_status.get('progress')
+        metrics += f'pesto_job_total{{id="{job_id}",status="{status}",progress="{progress}"}} 1\n'
     return Response(content=metrics, status_code=200, media_type='text/plain')
 
 @v1.post('/jobs')