Hardware alerts integration

Link alertmanager with resource server to handle hardware monitoring alerts.

- Add resource server client for alarms service
- Distinguish between CaaS and hardware alert sources
- Add internal resource endpoint to resource server API
- Update RBAC for resource types and PrometheusRules access
- Support alarm source constants (AlarmSourceCaaS, AlarmSourceHardware)

Co-Authored-By: Claude <[email protected]>

Author: pixelsoccupied

config/rbac/role.yaml

@@ -9,6 +9,7 @@ rules:
   - /o2ims-infrastructureCluster/v1/alarmDictionaries
   - /o2ims-infrastructureCluster/v1/nodeClusterTypes
   - /o2ims-infrastructureCluster/v1/nodeClusters
+  - /o2ims-infrastructureInventory/v1/resourceTypes
   verbs:
   - get
   - list
@@ -30,6 +31,8 @@ rules:
   - /o2ims-infrastructureCluster/v1/alarmDictionaries/*
   - /o2ims-infrastructureCluster/v1/nodeClusterTypes/*
   - /o2ims-infrastructureCluster/v1/nodeClusters/*
+  - /o2ims-infrastructureInventory/v1/internal/resources/*
+  - /o2ims-infrastructureInventory/v1/resourceTypes/*
   verbs:
   - get
 - apiGroups:

internal/controllers/inventory_controller.go

@@ -71,6 +71,9 @@ import (
 //+kubebuilder:rbac:urls="/o2ims-infrastructureCluster/v1/nodeClusterTypes/*",verbs=get
 //+kubebuilder:rbac:urls="/o2ims-infrastructureCluster/v1/nodeClusters/*",verbs=get
 //+kubebuilder:rbac:urls="/o2ims-infrastructureCluster/v1/alarmDictionaries/*",verbs=get
+//+kubebuilder:rbac:urls="/o2ims-infrastructureInventory/v1/internal/resources/*",verbs=get
+//+kubebuilder:rbac:urls="/o2ims-infrastructureInventory/v1/resourceTypes",verbs=get;list
+//+kubebuilder:rbac:urls="/o2ims-infrastructureInventory/v1/resourceTypes/*",verbs=get
 //+kubebuilder:rbac:urls="/hardware-manager/inventory/*",verbs=get;list
 //+kubebuilder:rbac:groups="batch",resources=cronjobs,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch
@@ -1380,6 +1383,31 @@ func (t *reconcilerTask) createAlarmServerClusterRole(ctx context.Context) error
 					"get",
 				},
 			},
+			{
+				NonResourceURLs: []string{
+					"/o2ims-infrastructureInventory/v1/resourceTypes",
+				},
+				Verbs: []string{
+					"get",
+					"list",
+				},
+			},
+			{
+				NonResourceURLs: []string{
+					"/o2ims-infrastructureInventory/v1/resourceTypes/*",
+				},
+				Verbs: []string{
+					"get",
+				},
+			},
+			{
+				NonResourceURLs: []string{
+					"/o2ims-infrastructureInventory/v1/internal/resources/*",
+				},
+				Verbs: []string{
+					"get",
+				},
+			},
 		},
 	}
 

internal/service/alarms/api/server.go

@@ -493,7 +493,7 @@ func (a *AlarmsServer) UpdateAlarmServiceConfiguration(ctx context.Context, requ
 func (a *AlarmsServer) AmNotification(ctx context.Context, request api.AmNotificationRequestObject) (api.AmNotificationResponseObject, error) {
 	// TODO: AM auto retries if it receives 5xx error code. That means any error, even if permanent (e.g postgres syntax), will be processed the same way. Once we have a better retry mechanism for pg, update all 5xx to 4xx as needed.
 
-	if err := alertmanager.HandleAlerts(ctx, a.Infrastructure.Clients, a.AlarmsRepository, &request.Body.Alerts, alertmanager.Webhook); err != nil {
+	if err := alertmanager.HandleAlerts(ctx, a.Infrastructure.ClusterServer, a.Infrastructure.ResourceServer, a.AlarmsRepository, &request.Body.Alerts, alertmanager.Webhook); err != nil {
 		msg := "failed to handle alerts"
 		slog.Error(msg, "error", err)
 		return nil, fmt.Errorf("%s: %w", msg, err)

internal/service/alarms/internal/alertmanager/api.go

@@ -99,7 +99,7 @@ func (c *AMClient) SyncAlerts(ctx context.Context) error {
 	// Covert to Webhook payload to allow us to maintain a single point of entry in the DB
 	webhookPayload := ConvertAPIAlertsToWebhook(&apiPayload)
 	if len(webhookPayload) != 0 {
-		if err := HandleAlerts(ctx, c.infrastructure.Clients, c.alarmsRepository, &webhookPayload, API); err != nil {
+		if err := HandleAlerts(ctx, c.infrastructure.ClusterServer, c.infrastructure.ResourceServer, c.alarmsRepository, &webhookPayload, API); err != nil {
 			return fmt.Errorf("failed to handle alerts during full sync: %w", err)
 		}
 	}

internal/service/alarms/internal/alertmanager/api_test.go

@@ -113,9 +113,8 @@ var _ = Describe("Alertmanager API Client", func() {
 			Build()
 
 		infra = &infrastructure.Infrastructure{
-			Clients: []infrastructure.Client{
-				&infrastructure.ClusterServer{},
-			},
+			ClusterServer:  &infrastructure.ClusterServer{},
+			ResourceServer: &infrastructure.ResourceServer{},
 		}
 
 		amClient = alertmanager.NewAlertmanagerClient(fakeClient, mockRepo, infra)

internal/service/alarms/internal/alertmanager/converter.go

@@ -19,7 +19,7 @@ import (
 )
 
 // ConvertAmToAlarmEventRecordModels get alarmEventRecords based on the alertmanager notification and AlarmDefinition
-func ConvertAmToAlarmEventRecordModels(ctx context.Context, alerts *[]api.Alert, infrastructureClient infrastructure.Client) []models.AlarmEventRecord {
+func ConvertAmToAlarmEventRecordModels(ctx context.Context, alerts *[]api.Alert, clusterServer, resourceServer infrastructure.Client) []models.AlarmEventRecord {
 	records := make([]models.AlarmEventRecord, 0, len(*alerts))
 	for _, alert := range *alerts {
 		record := models.AlarmEventRecord{}
@@ -69,8 +69,21 @@ func ConvertAmToAlarmEventRecordModels(ctx context.Context, alerts *[]api.Alert,
 		// Update Extensions with things we didn't really process
 		record.Extensions = getExtensions(alert)
 
-		// for caas alerts object is the cluster ID
-		record.ObjectID = getClusterID(labels)
+		// Determine alert type (CaaS vs hardware) and select appropriate infrastructure client
+		// Hardware alerts have both type=hardware AND component=ironic labels
+		isHardware := isHardwareAlert(labels)
+		var infrastructureClient infrastructure.Client
+		if isHardware {
+			// For hardware alerts, object is the resource ID from instance_uuid
+			record.ObjectID = getResourceID(labels)
+			record.AlarmSource = models.AlarmSourceHardware
+			infrastructureClient = resourceServer
+		} else {
+			// For CaaS alerts, object is the cluster ID from managed_cluster
+			record.ObjectID = getClusterID(labels)
+			record.AlarmSource = models.AlarmSourceCaaS
+			infrastructureClient = clusterServer
+		}
 
 		// derive ObjectTypeID from ObjectID
 		if record.ObjectID != nil {
@@ -118,6 +131,30 @@ func getClusterID(labels map[string]string) *uuid.UUID {
 	return &id
 }
 
+// isHardwareAlert checks if an alert is a hardware alert by checking for type=hardware AND component=ironic labels
+func isHardwareAlert(labels map[string]string) bool {
+	alertType, hasType := labels["type"]
+	component, hasComponent := labels["component"]
+	return hasType && alertType == "hardware" && hasComponent && component == "ironic"
+}
+
+// getResourceID extracts resource ID from instance_uuid label for hardware alerts
+func getResourceID(labels map[string]string) *uuid.UUID {
+	val, ok := labels["instance_uuid"]
+	if !ok {
+		slog.Warn("Could not find instance_uuid for hardware alert", "labels", labels)
+		return nil
+	}
+
+	id, err := uuid.Parse(val)
+	if err != nil {
+		slog.Warn("Could not convert instance_uuid string to uuid", "labels", labels, "err", err.Error())
+		return nil
+	}
+
+	return &id
+}
+
 // getAlertName extract name from alert label
 func getAlertName(labels map[string]string) string {
 	val, ok := labels["alertname"]

internal/service/alarms/internal/alertmanager/converter_test.go

@@ -75,7 +75,7 @@ var _ = Describe("Alertmanager Functions", func() {
 				},
 			}
 
-			records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient)
+			records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient, mockInfraClient)
 
 			// Assert
 			Expect(records).To(HaveLen(1))
@@ -131,7 +131,7 @@ var _ = Describe("Alertmanager Functions", func() {
 				GetAlarmDefinitionID(gomock.Any(), objectTypeIDUUID, "TestAlert", "critical").
 				Return(alarmDefUUID, nil)
 
-			records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient)
+			records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient, mockInfraClient)
 
 			// Assert
 			Expect(records).To(HaveLen(1))
@@ -188,9 +188,9 @@ var _ = Describe("Alertmanager Functions", func() {
 				},
 			}
 
-			Expect(alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts1, mockInfraClient)).To(BeEmpty())
-			Expect(alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts2, mockInfraClient)).To(BeEmpty())
-			Expect(alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts3, mockInfraClient)).To(BeEmpty())
+			Expect(alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts1, mockInfraClient, mockInfraClient)).To(BeEmpty())
+			Expect(alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts2, mockInfraClient, mockInfraClient)).To(BeEmpty())
+			Expect(alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts3, mockInfraClient, mockInfraClient)).To(BeEmpty())
 		})
 
 		It("should handle infrastructure client errors gracefully", func() {
@@ -220,7 +220,7 @@ var _ = Describe("Alertmanager Functions", func() {
 				},
 			}
 
-			records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient)
+			records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient, mockInfraClient)
 
 			// Assert - should still create record but without ObjectTypeID
 			Expect(records).To(HaveLen(1))
@@ -265,7 +265,7 @@ var _ = Describe("Alertmanager Functions", func() {
 					},
 				}
 
-				records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient)
+				records := alertmanager.ConvertAmToAlarmEventRecordModels(ctx, &alerts, mockInfraClient, mockInfraClient)
 
 				// Assert
 				Expect(records).To(HaveLen(1))

internal/service/alarms/internal/alertmanager/handler.go

@@ -26,7 +26,7 @@ const (
 
 // HandleAlerts can be called when a payload from Webhook or API `/alerts` is received
 // Webhook is our primary and API as our backup and sync mechanism
-func HandleAlerts(ctx context.Context, clients []infrastructure.Client, repository repo.AlarmRepositoryInterface, alerts *[]api.Alert, source SourceType) error {
+func HandleAlerts(ctx context.Context, clusterServer, resourceServer infrastructure.Client, repository repo.AlarmRepositoryInterface, alerts *[]api.Alert, source SourceType) error {
 	// Handle nil alerts
 	if alerts == nil {
 		return nil
@@ -37,23 +37,8 @@ func HandleAlerts(ctx context.Context, clients []infrastructure.Client, reposito
 		return nil
 	}
 
-	// Get cached cluster server data
-	var (
-		clusterServer infrastructure.Client
-		found         bool
-	)
-	for i := range clients {
-		if clients[i].Name() == infrastructure.Name {
-			clusterServer = clients[i]
-			found = true
-		}
-	}
-	if !found {
-		return fmt.Errorf("no cluster server found with name %q", infrastructure.Name)
-	}
-
 	// Combine possible definitions with events
-	aerModels := ConvertAmToAlarmEventRecordModels(ctx, alerts, clusterServer)
+	aerModels := ConvertAmToAlarmEventRecordModels(ctx, alerts, clusterServer, resourceServer)
 
 	// Insert and update AlarmEventRecord and optionally resolve stale
 	if err := repository.WithTransaction(ctx, func(tx pgx.Tx) error {

internal/service/alarms/internal/db/models/alarm_event_record.go

@@ -13,6 +13,13 @@ import (
 	"github.com/openshift-kni/oran-o2ims/internal/service/alarms/api/generated"
 )
 
+const (
+	// AlarmSourceCaaS represents alarms from Container-as-a-Service (cluster alerts)
+	AlarmSourceCaaS = "caas"
+	// AlarmSourceHardware represents alarms from hardware resources
+	AlarmSourceHardware = "hardware"
+)
+
 // AlarmEventRecord represents a record in the alarm_event_record table.
 type AlarmEventRecord struct {
 	AlarmEventRecordID    uuid.UUID                             `db:"alarm_event_record_id" json:"alarm_event_record_id"`

internal/service/alarms/internal/db/repo/alarms_repository.go

@@ -141,7 +141,7 @@ func (ar *AlarmsRepository) UpsertAlarmEventCaaSRecord(ctx context.Context, tx p
 		"AlarmAcknowledged", "PerceivedSeverity", "Extensions",
 		"ObjectID", "ObjectTypeID", "AlarmStatus",
 		"Fingerprint", "AlarmDefinitionID", "ProbableCauseID",
-		"GenerationID",
+		"GenerationID", "AlarmSource",
 	})
 
 	// Set values
@@ -152,7 +152,7 @@ func (ar *AlarmsRepository) UpsertAlarmEventCaaSRecord(ctx context.Context, tx p
 			record.AlarmAcknowledged, record.PerceivedSeverity, record.Extensions,
 			record.ObjectID, record.ObjectTypeID, record.AlarmStatus,
 			record.Fingerprint, record.AlarmDefinitionID, record.ProbableCauseID,
-			generationID,
+			generationID, record.AlarmSource,
 		)))
 	}
 	query.Apply(values...)
@@ -169,7 +169,7 @@ func (ar *AlarmsRepository) UpsertAlarmEventCaaSRecord(ctx context.Context, tx p
 		im.SetExcluded(dbTags["AlarmDefinitionID"]),
 		im.SetExcluded(dbTags["ProbableCauseID"]),
 		im.SetExcluded(dbTags["GenerationID"]),
-		im.Where(psql.Quote(m.TableName(), dbTags["AlarmSource"]).EQ(psql.Arg("alertmanager"))),
+		im.SetExcluded(dbTags["AlarmSource"]),
 	))
 
 	sql, params, err := query.Build(ctx)
@@ -209,12 +209,12 @@ func (ar *AlarmsRepository) ResolveStaleAlarmEventCaaSRecord(ctx context.Context
 
 	query := psql.Update(
 		um.Table(tableName),
-		um.SetCol(alarmStatus).ToArg(api.Resolved),                       // Set to resolved
-		um.SetCol(perceivedSeverity).ToArg(api.CLEARED),                  // Set corresponding perceivedSeverity
-		um.Set(psql.Raw(updateClearedTimeCase, TimeNow())),               // Set a resolved time if not there already
-		um.Where(psql.Quote(generationIDCol).LT(psql.Arg(generationID))), // An alert is stale if its GenID is less than current
-		um.Where(psql.Quote(alarmSource).EQ(psql.Arg("alertmanager"))),   // This is only applicable for alertmanager rows
-		um.Where(psql.Quote(alarmStatus).NE(psql.Arg(api.Resolved))),     // If already resolved no need to process that row
+		um.SetCol(alarmStatus).ToArg(api.Resolved),                                                         // Set to resolved
+		um.SetCol(perceivedSeverity).ToArg(api.CLEARED),                                                    // Set corresponding perceivedSeverity
+		um.Set(psql.Raw(updateClearedTimeCase, TimeNow())),                                                 // Set a resolved time if not there already
+		um.Where(psql.Quote(generationIDCol).LT(psql.Arg(generationID))),                                   // An alert is stale if its GenID is less than current
+		um.Where(psql.Quote(alarmSource).In(psql.Arg(models.AlarmSourceCaaS, models.AlarmSourceHardware))), // Support both CaaS and hardware alerts
+		um.Where(psql.Quote(alarmStatus).NE(psql.Arg(api.Resolved))),                                       // If already resolved no need to process that row
 		um.Returning(psql.Quote(alarmEventRecordID)),
 	)