WIP: Add support for project config

pyproject.toml

@@ -12,6 +12,7 @@ classifiers = [
 ]
 dependencies = [
     "click>=8.1.8",
+    "dynaconf>=3.2.11",
     "google-cloud-storage>=3.1.0",
     "hatchling>=1.27.0",
     "jinja2>=3.1.6",

src/commands/report.py

@@ -25,6 +25,7 @@
 from src.objects.jira_base import Jira
 from src.objects.job import Job
 from src.report.report import Report
+from src.project.project import Project
 
 
 def validate_verbose_test_failure_reporting_ticket_limit(
@@ -83,7 +84,7 @@ def validate_verbose_test_failure_reporting_ticket_limit(
     type=click.Path(exists=True),
 )
 @click.option(
-    "--firewatch-config-path",
+    "--rules-config-path",
     help="The path to the firewatch configuration file",
     required=False,
     type=click.Path(),
@@ -135,6 +136,11 @@ def validate_verbose_test_failure_reporting_ticket_limit(
     help="Drop to `ipdb` shell on exception",
     is_flag=True,
 )
+@click.option(
+    "--project-config-path",
+    help="The path to the project configuration file",
+    required=False,
+)
 @click.command("report")
 @click.pass_context
 def report(
@@ -145,7 +151,7 @@ def report(
     pr_id: str,
     gcs_bucket: str,
     gcs_creds_file: Optional[str],
-    firewatch_config_path: Optional[str],
+    rules_config_path: Optional[str],
     jira_config_path: str,
     fail_with_test_failures: bool,
     fail_with_pod_failures: bool,
@@ -154,21 +160,15 @@ def report(
     verbose_test_failure_reporting_ticket_limit: Optional[int],
     additional_labels_file: Optional[str],
     pdb: bool,
+    project_config_path: Optional[str],
 ) -> None:
     ctx.obj["PDB"] = pdb
 
+    project_data = Project(ctx.params)
+
     # Build Objects
-    jira_connection = Jira(jira_config_path=jira_config_path)
-    config = Configuration(
-        jira=jira_connection,
-        fail_with_test_failures=fail_with_test_failures,
-        fail_with_pod_failures=fail_with_pod_failures,
-        keep_job_dir=keep_job_dir,
-        verbose_test_failure_reporting=verbose_test_failure_reporting,
-        verbose_test_failure_reporting_ticket_limit=verbose_test_failure_reporting_ticket_limit,
-        config_file_path=firewatch_config_path,
-        additional_lables_file=additional_labels_file,
-    )
+    jira_connection = Jira(jira_config_path=project_data.jira_config_path)
+    config = Configuration(jira=jira_connection, project_data=project_data)
     job = Job(
         name=job_name,
         name_safe=job_name_safe,

src/objects/configuration.py

@@ -3,73 +3,45 @@
 import fnmatch
 from typing import Any
 from typing import Optional
-from typing import Union
 
 from simple_logger.logger import get_logger
 
 from src.objects.failure_rule import FailureRule
 from src.objects.jira_base import Jira
 from src.objects.rule import Rule
-
-
-def read_base_config_file(path: str) -> str:
-    from urllib.request import urlopen
-
-    try:
-        response = urlopen(path)
-        response_data = response.read()
-        return response_data
-    # Path is not a URL type
-    except ValueError:
-        # Read the contents of the config file
-        try:
-            with open(path) as file:
-                base_config_str = file.read()
-                return base_config_str
-        except Exception:
-            pass
-    # Path is an invalid or unreadable URL
-    except Exception:
-        pass
-
-    return None  # type: ignore
+from src.project.utils import read_url_config
+from src.project.project import Project
 
 
 class Configuration:
     def __init__(
         self,
         jira: Jira,
-        fail_with_test_failures: bool,
-        fail_with_pod_failures: bool,
-        keep_job_dir: bool,
-        verbose_test_failure_reporting: bool,
-        verbose_test_failure_reporting_ticket_limit: Optional[int] = 10,
-        config_file_path: Union[str, None] = None,
-        additional_lables_file: Optional[str] = None,
+        project_data: Project,
     ):
         """
         Constructs the Configuration object. This class is mainly used to validate the firewatch configuration given.
-
-        Args:
-            jira (Jira): A Jira object used to log in and interact with Jira
-            fail_with_test_failures (bool): If a test failure is found, after bugs are filed, firewatch will exit with a non-zero exit code
-            keep_job_dir (bool): If true, firewatch will not delete the job directory (/tmp/12345) that is created to hold logs and results for a job following execution.
-            verbose_test_failure_reporting (bool): If true, firewatch will report all test failures found in the job.
-            verbose_test_failure_reporting_ticket_limit (Optional[int]): Used as a safeguard to prevent firewatch from filing too many bugs. If verbose_test_reporting is set to true, this value will be used to limit the number of bugs filed. Defaults to 10.
-            config_file_path (Union[str, None], optional): The firewatch config can be stored in a file or an environment var. Defaults to None.
-            additional_lables_file (Optional[str]): If set, the filepath provided will be parsed for additional labels. Each label should be separated by a new line.
+        # Args:
+        #     jira (Jira): A Jira object used to log in and interact with Jira
+        #     fail_with_test_failures (bool): If a test failure is found, after bugs are filed, firewatch will exit with a non-zero exit code
+        #     keep_job_dir (bool): If true, firewatch will not delete the job directory (/tmp/12345) that is created to hold logs and results for a job following execution.
+        #     verbose_test_failure_reporting (bool): If true, firewatch will report all test failures found in the job.
+        #     verbose_test_failure_reporting_ticket_limit (Optional[int]): Used as a safeguard to prevent firewatch from filing too many bugs. If verbose_test_reporting is set to true, this value will be used to limit the number of bugs filed. Defaults to 10.
+        #     rules_file_path (Union[str, None], optional): The firewatch config can be stored in a file or an environment var. Defaults to None.
+        #     additional_labels_file (Optional[str]): If set, the filepath provided will be parsed for additional labels. Each label should be separated by a new line.
         """
         self.logger = get_logger(__name__)
 
+        self.project_data = project_data
         self.jira = jira
         self.default_jira_project = self._get_default_jira_project()
-        self.fail_with_test_failures = fail_with_test_failures
-        self.fail_with_pod_failures = fail_with_pod_failures
-        self.keep_job_dir = keep_job_dir
-        self.additional_labels_file = additional_lables_file
-        self.verbose_test_failure_reporting = verbose_test_failure_reporting
-        self.verbose_test_failure_reporting_ticket_limit = verbose_test_failure_reporting_ticket_limit
-        self.config_data = self._get_config_data(base_config_file_path=config_file_path)
+        self.fail_with_test_failures = project_data.fail_with_test_failures
+        self.fail_with_pod_failures = project_data.fail_with_pod_failures
+        self.keep_job_dir = project_data.keep_job_dir
+        self.additional_labels_file = project_data.additional_labels_file
+        self.verbose_test_failure_reporting = project_data.verbose_test_failure_reporting
+        self.verbose_test_failure_reporting_ticket_limit = project_data.verbose_test_failure_reporting_ticket_limit
+        self.config_data = self._get_rules_config_data(base_config_file_path=project_data.rules_config_file_path)
         self.success_rules = self._get_success_rules(
             rules_list=self.config_data.get("success_rules"),
         )
@@ -90,7 +62,7 @@ def _get_failure_rules(
         if rules_list is not None:
             rules = []
             for line in rules_list:
-                rules.append(FailureRule(rule_dict=line))
+                rules.append(FailureRule(rule_dict=line, project_data=self.project_data))
 
             if len(rules) > 0:
                 return rules
@@ -113,7 +85,7 @@ def _get_success_rules(
         if rules_list is not None:
             rules = []
             for line in rules_list:
-                rules.append(Rule(rule_dict=line))
+                rules.append(Rule(rule_dict=line, project_data=self.project_data))
 
             if len(rules) > 0:
                 return rules
@@ -128,7 +100,7 @@ def _get_default_jira_project(self) -> str:
             str: The default Jira project name defined in environment variable.
         """
 
-        default_project = os.getenv("FIREWATCH_DEFAULT_JIRA_PROJECT")
+        default_project = self.project_data.default_jira_project
 
         # Verify that value is a string if it exists, return
         if isinstance(default_project, str):
@@ -145,7 +117,7 @@ def _get_default_jira_project(self) -> str:
             )
             exit(1)
 
-    def _get_config_data(self, base_config_file_path: Optional[str]) -> dict[Any, Any]:
+    def _get_rules_config_data(self, base_config_file_path: Optional[str]) -> dict[Any, Any]:
         """
         Gets the config data from either a configuration file or from the FIREWATCH_CONFIG environment variable or
         both.
@@ -164,7 +136,7 @@ def _get_config_data(self, base_config_file_path: Optional[str]) -> dict[Any, An
         steps_map = {}
 
         if base_config_file_path is not None:
-            base_config_str = read_base_config_file(path=base_config_file_path)
+            base_config_str = read_url_config(path=base_config_file_path)
             if not base_config_str:
                 self.logger.error(
                     f"Unable to read configuration file at {base_config_file_path}."

src/objects/rule.py

@@ -5,10 +5,11 @@
 from typing import Optional
 
 from simple_logger.logger import get_logger
+from src.project.project import Project
 
 
 class Rule:
-    def __init__(self, rule_dict: dict[Any, Any]) -> None:
+    def __init__(self, rule_dict: dict[Any, Any], project_data: Project) -> None:
         """
         Initializes the Rule object.
 
@@ -17,6 +18,7 @@ def __init__(self, rule_dict: dict[Any, Any]) -> None:
         """
 
         self.logger = get_logger(__name__)
+        self.project_data = project_data
         self.jira_project = self._get_jira_project(rule_dict)
         self.jira_epic = self._get_jira_epic(rule_dict)
         self.jira_component = self._get_jira_component(rule_dict)
@@ -40,7 +42,7 @@ def _get_jira_project(self, rule_dict: dict[Any, Any]) -> str:
         jira_project = rule_dict.get("jira_project")
 
         if jira_project == "!default" or not jira_project:
-            jira_project = os.getenv("FIREWATCH_DEFAULT_JIRA_PROJECT")
+            jira_project = self.project_data.default_jira_project
 
         if not jira_project:
             self.logger.error(
@@ -173,7 +175,7 @@ def _get_jira_additional_labels(
         if isinstance(jira_additional_labels, list):
             # If the list contains "!default", include the list in the environment variable
             if "!default" in jira_additional_labels:
-                default_labels = os.getenv("FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS")
+                default_labels = self.project_data.default_jira_additional_labels
                 if default_labels:
                     try:
                         default_labels = json.loads(default_labels)

src/project/config.py

@@ -0,0 +1,49 @@
+import tempfile
+import yaml
+from dynaconf import Dynaconf
+from pathlib import Path
+from pathlib import PurePath
+from src.project.utils import get_project_config_data
+from src.project.constants import PROJECT_DATA_SCHEMA, PROJECT_TEMPLATE_PATH
+from typing import Optional
+
+
+def validate_settings(settings: dict, schema=PROJECT_DATA_SCHEMA):
+    errors = []
+    for key, value in settings.items():
+        if not value:
+            # Missing value
+            continue
+        schema_key = schema.get(key.lower())
+        if not schema_key:
+            # Redundant/extra var. Not applied on Firewatch. will not check
+            continue
+        expected_type = schema_key.get("type")
+        if expected_type:
+            if not isinstance(value, expected_type):
+                errors.append(
+                    f"Key '{key}' has invalid type: expected {expected_type.__name__}, "
+                    f"got {type(value).__name__} (value={value})"
+                )
+    if errors:
+        raise ValueError("Config validation failed:\n" + "\n".join(errors))
+
+
+def load_settings_from_dict(
+    config_path: Optional[str] = None, template_file=PROJECT_TEMPLATE_PATH, env_prefix="FIREWATCH"
+):
+    if config_path:
+        config_dict = get_project_config_data(project_file_path=config_path)
+        current_directory = Path().resolve()
+        settings_file = PurePath(current_directory, template_file)
+
+        with tempfile.NamedTemporaryFile(mode="w+", suffix=".yaml", delete=False) as tmpfile:
+            yaml.dump(config_dict, tmpfile)
+            tmpfile_path = tmpfile.name
+
+        return Dynaconf(
+            settings_files=[settings_file, tmpfile_path],
+            lowercase_read=True,
+            environments=False,
+            envvar_prefix=env_prefix,
+        )

src/project/constants.py

@@ -0,0 +1,11 @@
+PROJECT_TEMPLATE_PATH = "src/templates/settings.yaml"
+PROJECT_DATA_SCHEMA = {
+    "rules_config_file_path": {"type": str, "required": False},
+    "jira_config_path": {"type": str, "required": False},
+    "default_jira_project": {"type": str, "required": False},
+    "default_jira_additional_labels": {"type": str, "required": False},
+    "fail_with_test_failures": {"type": bool, "required": False},
+    "fail_with_pod_failures": {"type": bool, "required": False},
+    "keep_job_dir": {"type": bool, "required": False},
+    "additional_labels_file": {"type": str, "required": False},
+}

src/project/project.py

@@ -0,0 +1,33 @@
+from simple_logger.logger import get_logger
+from src.project.config import load_settings_from_dict, validate_settings
+
+
+class Project:
+    def __init__(
+        self,
+        project_params,
+    ):
+        self.logger = get_logger(__name__)
+        project_config_file_path = project_params.get("project_config_path")
+        settings = load_settings_from_dict(project_config_file_path)
+        self.project_data = settings.as_dict()
+        validate_settings(self.project_data)
+
+        self.jira_config_path = project_params.get("jira_config_path", settings.jira_config_path)
+        self.rules_config_file_path = project_params.get("rules_config_file_path", settings.rules_config_file_path)
+        self.default_jira_additional_labels = project_params.get(
+            "default_jira_additional_labels", settings.default_jira_additional_labels
+        )
+        self.default_jira_project = project_params.get("default_jira_project", settings.default_jira_project)
+        self.fail_with_test_failures = project_params.get("fail_with_test_failures", settings.fail_with_test_failures)
+        self.fail_with_pod_failures = project_params.get("fail_with_pod_failures", settings.fail_with_pod_failures)
+        self.keep_job_dir = project_params.get("keep_job_dir", settings.keep_job_dir)
+        self.additional_labels_file = project_params.get("additional_labels_file", settings.additional_labels_file)
+        self.verbose_test_failure_reporting = project_params.get(
+            "verbose_test_failure_reporting", settings.verbose_test_failure_reporting
+        )
+        # Click cli params
+        self.verbose_test_failure_reporting_ticket_limit = project_params.get(
+            "verbose_test_failure_reporting_ticket_limit"
+        )
+        self.rules_config_file_path = project_params.get("rules_config_path")