chore: clean up node conditions before test

Author: tanishagoyal2

health-events-analyzer/go.mod

@@ -28,7 +28,7 @@ require (
 )
 
 require (
-	github.com/BurntSushi/toml v1.5.0
+	github.com/BurntSushi/toml v1.5.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/caarlos0/env/v11 v11.3.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect

tests/helpers/kube.go

@@ -613,7 +613,7 @@ func CreateRebootNodeCR(
 }
 
 // WaitForNodeConditionWithCheckName waits for the node to have a condition with the reason containing the specified checkName.
-func WaitForNodeConditionWithCheckName(ctx context.Context, t *testing.T, c klient.Client, nodeName, checkName string) {
+func WaitForNodeConditionWithCheckName(ctx context.Context, t *testing.T, c klient.Client, nodeName, checkName, message string) {
 	require.Eventually(t, func() bool {
 		node, err := GetNodeByName(ctx, c, nodeName)
 		if err != nil {
@@ -623,7 +623,7 @@ func WaitForNodeConditionWithCheckName(ctx context.Context, t *testing.T, c klie
 
 		// Look for a condition where the reason contains the check name
 		for _, condition := range node.Status.Conditions {
-			if condition.Status == v1.ConditionTrue && strings.Contains(condition.Reason, checkName) {
+			if condition.Status == v1.ConditionTrue && strings.Contains(condition.Reason, checkName) && strings.Contains(message, condition.Message) {
 				t.Logf("Found node condition: Type=%s, Reason=%s, Status=%s, Message=%s",
 					condition.Type, condition.Reason, condition.Status, condition.Message)
 				return true

tests/multiple_fatal_event_test.go

@@ -59,10 +59,16 @@ func TestMultipleFatalEventRule(t *testing.T) {
 
 		client, err := c.NewClient()
 		assert.NoError(t, err, "failed to create kubernetes client")
+
+		// clean up any existing node conditions
+		t.Logf("Cleaning up any existing node conditions for node %s", gpuNodeName)
+		err = helpers.SendHealthEventsToNodes(t, []string{gpuNodeName}, ERRORCODE_13, "data/non-fatal-health-event.json", "")
+		assert.NoError(t, err, "failed to send non-fatal events")
+
 		xidsToInject := []string{ERRORCODE_13, ERRORCODE_48, ERRORCODE_13, ERRORCODE_48, ERRORCODE_13}
 
 		// inject 5 fatal errors and let the remediation cycle finish
-
+		t.Logf("Injecting fatal errors to node %s", gpuNodeName)
 		for _, xid := range xidsToInject {
 			// inject XID error
 			err = helpers.SendHealthEventsToNodes(t, []string{gpuNodeName}, xid, "data/fatal-health-event.json", "")
@@ -92,7 +98,7 @@ func TestMultipleFatalEventRule(t *testing.T) {
 		assert.NoError(t, err, "failed to send fatal events")
 
 		// Check node condition for matched ruleset
-		helpers.WaitForNodeConditionWithCheckName(ctx, t, client, gpuNodeName, "MultipleFatalError")
+		helpers.WaitForNodeConditionWithCheckName(ctx, t, client, gpuNodeName, "MultipleFatalError", "ErrorCode:31 GPU:0 XID error occurred Recommended Action=CONTACT_SUPPORT;")
 
 		return ctx
 	})
@@ -143,8 +149,13 @@ func TestMultipleNonFatalEventRule(t *testing.T) {
 		assert.True(t, len(gpuNodes) > 0, "no gpu nodes found")
 		gpuNodeName := gpuNodes[rand.Intn(len(gpuNodes))]
 		ctx = context.WithValue(ctx, keyGpuNodeName, gpuNodeName)
-		t.Logf("Injecting fatal events to node %s", gpuNodeName)
 
+		// clean up any existing node conditions
+		t.Logf("Cleaning up any existing node conditions for node %s", gpuNodeName)
+		err := helpers.SendHealthEventsToNodes(t, []string{gpuNodeName}, ERRORCODE_13, "data/non-fatal-health-event.json", "")
+		assert.NoError(t, err, "failed to send non-fatal events")
+
+		t.Logf("Injecting non-fatal events to node %s", gpuNodeName)
 		for i := 0; i < 5; i++ {
 			// inject XID error
 			err := helpers.SendHealthEventsToNodes(t, []string{gpuNodeName}, ERRORCODE_13, "data/non-fatal-health-event.json", "")