Merge pull request #164 from naved001/adjustable-metric-frequency

Author: naved001

openshift_metrics/config.py

@@ -14,3 +14,5 @@
 S3_SECRET_ACCESS_KEY = os.getenv("S3_OUTPUT_SECRET_ACCESS_KEY")
 S3_INVOICE_BUCKET = os.getenv("S3_INVOICE_BUCKET", "nerc-invoicing")
 S3_METRICS_BUCKET = os.getenv("S3_METRICS_BUCKET", "openshift_metrics")
+PROM_QUERY_INTERVAL_MINUTES = int(os.getenv("PROM_QUERY_INTERVAL_MINUTES", 15))
+assert PROM_QUERY_INTERVAL_MINUTES >= 1, "Query interval must be at least 1 minute"

openshift_metrics/merge.py

@@ -2,6 +2,7 @@
 Merges metrics from files and produces reports by pod and by namespace
 """
 
+import sys
 import logging
 import argparse
 from datetime import datetime, UTC
@@ -12,7 +13,7 @@
 
 from openshift_metrics import utils, invoice
 from openshift_metrics.metrics_processor import MetricsProcessor
-from openshift_metrics.config import S3_INVOICE_BUCKET
+from openshift_metrics.config import S3_INVOICE_BUCKET, PROM_QUERY_INTERVAL_MINUTES
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -106,7 +107,31 @@ def main():
     report_start_date = None
     report_end_date = None
     cluster_name = None
-    processor = MetricsProcessor()
+    interval_minutes = None
+
+    for file in files:
+        with open(file, "r") as jsonfile:
+            metrics_from_file = json.load(jsonfile)
+            if interval_minutes is None:
+                interval_minutes = metrics_from_file.get("interval_minutes")
+            else:
+                interval_minutes_from_file = metrics_from_file["interval_minutes"]
+                if interval_minutes != interval_minutes_from_file:
+                    sys.exit(
+                        f"Cannot process files with different intervals {interval_minutes} != {interval_minutes_from_file}"
+                    )
+
+    if interval_minutes is None:
+        logger.info(
+            f"No prometheus query interval minutes found in the given set of files. Using the provided interval: {PROM_QUERY_INTERVAL_MINUTES} minute(s)"
+        )
+        interval_minutes = PROM_QUERY_INTERVAL_MINUTES
+    else:
+        logger.info(
+            f"Prometheus Query interval set to {interval_minutes} minute(s) from file"
+        )
+
+    processor = MetricsProcessor(interval_minutes)
 
     for file in files:
         with open(file, "r") as jsonfile:

openshift_metrics/openshift_prometheus_metrics.py

@@ -26,6 +26,7 @@
     OPENSHIFT_PROMETHEUS_URL,
     OPENSHIFT_TOKEN,
     S3_METRICS_BUCKET,
+    PROM_QUERY_INTERVAL_MINUTES,
 )
 
 logging.basicConfig(level=logging.INFO)
@@ -89,14 +90,17 @@ def main():
         output_file = f"metrics-{report_start_date}-to-{report_end_date}.json"
 
     logger.info(
-        f"Generating report starting {report_start_date} and ending {report_end_date} in {output_file}"
+        f"Generating report starting {report_start_date} and ending {report_end_date} in {output_file} with interval {PROM_QUERY_INTERVAL_MINUTES} minute"
     )
 
-    prom_client = PrometheusClient(openshift_url, OPENSHIFT_TOKEN)
+    prom_client = PrometheusClient(
+        openshift_url, OPENSHIFT_TOKEN, PROM_QUERY_INTERVAL_MINUTES
+    )
 
     metrics_dict = {}
     metrics_dict["start_date"] = report_start_date
     metrics_dict["end_date"] = report_end_date
+    metrics_dict["interval_minutes"] = PROM_QUERY_INTERVAL_MINUTES
     metrics_dict["cluster_name"] = URL_CLUSTER_NAME_MAPPING.get(
         args.openshift_url, args.openshift_url
     )