feat: resolve manual incidents automatically (keephq#3927)

Author: talboren

keep/api/bl/incidents_bl.py

@@ -19,13 +19,22 @@
     get_all_alerts_by_fingerprints,
     get_incident_by_id,
     get_incident_unique_fingerprint_count,
+    is_all_alerts_resolved,
+    is_first_incident_alert_resolved,
+    is_last_incident_alert_resolved,
     remove_alerts_to_incident_by_incident_id,
     update_incident_from_dto_by_id,
     update_incident_severity,
 )
 from keep.api.core.elastic import ElasticClient
-from keep.api.models.alert import IncidentDto, IncidentDtoIn, IncidentSeverity
+from keep.api.models.alert import (
+    IncidentDto,
+    IncidentDtoIn,
+    IncidentSeverity,
+    IncidentStatus,
+)
 from keep.api.models.db.alert import ActionType, Incident
+from keep.api.models.db.rule import ResolveOn
 from keep.api.utils.enrichment_helpers import convert_db_alerts_to_dto_alerts
 from keep.workflowmanager.workflowmanager import WorkflowManager
 
@@ -359,3 +368,30 @@ def __postprocess_incident_change(self, incident):
             extra={"incident_id": incident.id},
         )
         return new_incident_dto
+
+    @staticmethod
+    def resolve_incident_if_require(incident: Incident, session: Session) -> Incident:
+
+        should_resolve = False
+
+        if incident.resolve_on == ResolveOn.ALL.value and is_all_alerts_resolved(
+            incident=incident, session=session
+        ):
+            should_resolve = True
+
+        elif (
+            incident.resolve_on == ResolveOn.FIRST.value
+            and is_first_incident_alert_resolved(incident, session=session)
+        ):
+            should_resolve = True
+
+        elif (
+            incident.resolve_on == ResolveOn.LAST.value
+            and is_last_incident_alert_resolved(incident, session=session)
+        ):
+            should_resolve = True
+
+        if should_resolve:
+            incident.status = IncidentStatus.RESOLVED.value
+
+        return incident

keep/api/tasks/process_event_task.py

@@ -17,9 +17,11 @@
 # internals
 from keep.api.alert_deduplicator.alert_deduplicator import AlertDeduplicator
 from keep.api.bl.enrichments_bl import EnrichmentsBl
+from keep.api.bl.incidents_bl import IncidentBl
 from keep.api.bl.maintenance_windows_bl import MaintenanceWindowsBl
 from keep.api.core.db import (
     bulk_upsert_alert_fields,
+    enrich_alerts_with_incidents,
     get_alerts_by_fingerprint,
     get_all_presets_dtos,
     get_enrichment_with_session,
@@ -138,6 +140,7 @@ def __save_to_db(
                 session.add(audit)
 
         enriched_formatted_events = []
+        saved_alerts = []
 
         for formatted_event in formatted_events:
             formatted_event.pushed = True
@@ -211,6 +214,7 @@ def __save_to_db(
             alert = Alert(**alert_args)
             session.add(alert)
             session.flush()
+            saved_alerts.append(alert)
             alert_id = alert.id
             formatted_event.event_id = str(alert_id)
 
@@ -251,6 +255,29 @@ def __save_to_db(
                         value = value.strip()
                     setattr(formatted_event, enrichment, value)
             enriched_formatted_events.append(formatted_event)
+
+        logger.info("Checking for incidents to resolve", extra={"tenant_id": tenant_id})
+        try:
+            saved_alerts = enrich_alerts_with_incidents(
+                tenant_id, saved_alerts, session
+            )  # note: this only enriches incidents that were not yet ended
+            for alert in saved_alerts:
+                if alert.event.get("status") == AlertStatus.RESOLVED.value:
+                    logger.debug(
+                        "Checking for alert with status resolved",
+                        extra={"alert_id": alert.id, "tenant_id": tenant_id},
+                    )
+                    for incident in alert._incidents:
+                        IncidentBl.resolve_incident_if_require(incident, session)
+            logger.info(
+                "Completed checking for incidents to resolve",
+                extra={"tenant_id": tenant_id},
+            )
+        except Exception:
+            logger.exception(
+                "Failed to check for incidents to resolve",
+                extra={"tenant_id": tenant_id},
+            )
         session.commit()
 
         logger.info(

keep/rulesengine/rulesengine.py

@@ -19,22 +19,11 @@
     get_incident_for_grouping_rule,
 )
 from keep.api.core.db import get_rules as get_rules_db
-from keep.api.core.db import (
-    is_all_alerts_in_status,
-    is_all_alerts_resolved,
-    is_first_incident_alert_resolved,
-    is_last_incident_alert_resolved,
-)
+from keep.api.core.db import is_all_alerts_in_status
 from keep.api.core.dependencies import get_pusher_client
-from keep.api.models.alert import (
-    AlertDto,
-    AlertSeverity,
-    AlertStatus,
-    IncidentDto,
-    IncidentStatus,
-)
+from keep.api.models.alert import AlertDto, AlertSeverity, AlertStatus, IncidentDto
 from keep.api.models.db.alert import Incident
-from keep.api.models.db.rule import ResolveOn, Rule
+from keep.api.models.db.rule import Rule
 from keep.api.utils.cel_utils import preprocess_cel_expression
 from keep.api.utils.enrichment_helpers import convert_db_alerts_to_dto_alerts
 
@@ -155,13 +144,17 @@ def _run_cel_rules(
                                     )
                                     incident.is_confirmed = True
                                 elif rule.create_on == "all":
-                                    incident = self._process_event_for_history_based_rule(
-                                        incident, rule, session
+                                    incident = (
+                                        self._process_event_for_history_based_rule(
+                                            incident, rule, session
+                                        )
                                     )
 
                             send_created_event = incident.is_confirmed
 
-                        incident = self._resolve_incident_if_require(incident, session)
+                        incident = IncidentBl.resolve_incident_if_require(
+                            incident, session
+                        )
                         session.add(incident)
                         session.commit()
 
@@ -336,33 +329,6 @@ def _process_event_for_history_based_rule(
 
         return incident
 
-    @staticmethod
-    def _resolve_incident_if_require(incident: Incident, session: Session) -> Incident:
-
-        should_resolve = False
-
-        if incident.resolve_on == ResolveOn.ALL.value and is_all_alerts_resolved(
-            incident=incident, session=session
-        ):
-            should_resolve = True
-
-        elif (
-            incident.resolve_on == ResolveOn.FIRST.value
-            and is_first_incident_alert_resolved(incident, session=session)
-        ):
-            should_resolve = True
-
-        elif (
-            incident.resolve_on == ResolveOn.LAST.value
-            and is_last_incident_alert_resolved(incident, session=session)
-        ):
-            should_resolve = True
-
-        if should_resolve:
-            incident.status = IncidentStatus.RESOLVED.value
-
-        return incident
-
     @staticmethod
     def _extract_subrules(expression):
         # CEL rules looks like '(source == "sentry") || (source == "grafana" && severity == "critical")'