fix: tone down logging and test case fixes (#269)

Author: lalitadithya

.coderabbit.yaml

@@ -0,0 +1,18 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+reviews:
+  profile: chill
+  auto_review:
+    enabled: true

health-monitors/syslog-health-monitor/pkg/syslog-monitor/syslogmonitor.go

@@ -662,14 +662,14 @@ func (sm *SyslogMonitor) processJournalEntries(journal Journal, check CheckDefin
 			}
 			// This entry (matched or not) is considered processed.
 			sm.checkLastCursors[check.Name] = currentEntryCursor // Update cursor for the next run
-			slog.Info("Check, considered processed", "name", check.Name,
+			slog.Debug("Check errored but considered processed", "name", check.Name,
 				"message", message,
 				"cursor", currentEntryCursor)
 		}
 
 		advancedNext, advErr := journal.Next()
 		if advErr == io.EOF || advancedNext == 0 { //nolint:errorlint // TODO
-			slog.Info("Check, no more", "name", check.Name, "cursor", currentEntryCursor)
+			slog.Info("Check no more", "name", check.Name, "cursor", currentEntryCursor)
 			// sm.checkLastCursors[checkName] is already set to currentEntryCursor.
 			break
 		}

health-monitors/syslog-health-monitor/pkg/xid/parser/sidecar.go

@@ -35,9 +35,12 @@ type SidecarParser struct {
 
 // NewSidecarParser creates a new sidecar parser
 func NewSidecarParser(endpoint, nodeName string) *SidecarParser {
+	c := retryablehttp.NewClient()
+	c.Logger = slog.With("http", "retryablehttp-client")
+
 	return &SidecarParser{
 		url:      fmt.Sprintf("%s/decode-xid", endpoint),
-		client:   retryablehttp.NewClient(),
+		client:   c,
 		nodeName: nodeName,
 	}
 }
@@ -74,7 +77,7 @@ func (p *SidecarParser) Parse(message string) (*Response, error) {
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		slog.Error("HTTP request failed", "statusCode", resp.StatusCode)
+		slog.Debug("HTTP request failed", "statusCode", resp.StatusCode)
 		metrics.XidProcessingErrors.WithLabelValues("http_status_error", p.nodeName).Inc()
 
 		return nil, fmt.Errorf("HTTP request failed with status code: %d", resp.StatusCode)

tests/uat/tests.sh

@@ -76,7 +76,7 @@ test_gpu_monitoring_dcgm() {
     log "========================================="
 
     local gpu_node
-    gpu_node=$(kubectl get nodes -l workload-type=gpu -o jsonpath='{.items[0].metadata.name}')
+    gpu_node=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].metadata.name}')
 
     if [[ -z "$gpu_node" ]]; then
         error "No GPU nodes found"
@@ -103,7 +103,7 @@ test_gpu_monitoring_dcgm() {
     local max_wait=30
     local waited=0
     while [[ $waited -lt $max_wait ]]; do
-        conditions_count=$(kubectl get node "$gpu_node" -o json | jq '[.status.conditions[] | select(.type == "GpuInforomWatch" or .type == "GpuPcieWatch")] | length')
+        conditions_count=$(kubectl get node "$gpu_node" -o json | jq '[.status.conditions[] | select(.status == "True" and (.type == "GpuInforomWatch" or .type == "GpuPcieWatch"))] | length')
         if [[ "$conditions_count" -ge 2 ]]; then
             log "Found $conditions_count node conditions"
             break
@@ -145,7 +145,7 @@ test_xid_monitoring_syslog() {
     log "========================================="
 
     local gpu_node
-    gpu_node=$(kubectl get nodes -l workload-type=gpu -o jsonpath='{.items[0].metadata.name}')
+    gpu_node=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].metadata.name}')
 
     if [[ -z "$gpu_node" ]]; then
         error "No GPU nodes found"
@@ -179,7 +179,7 @@ test_sxid_monitoring_syslog() {
     log "========================================="
 
     local gpu_node
-    gpu_node=$(kubectl get nodes -l workload-type=gpu -o jsonpath='{.items[0].metadata.name}')
+    gpu_node=$(kubectl get nodes -l nvidia.com/gpu.present=true -o jsonpath='{.items[0].metadata.name}')
 
     if [[ -z "$gpu_node" ]]; then
         error "No GPU nodes found"
@@ -254,9 +254,6 @@ test_sxid_monitoring_syslog() {
     fi
     log "Node event verified: SysLogsSXIDError ✓"
 
-    log "Waiting for node to be quarantined and rebooted..."
-    wait_for_boot_id_change "$gpu_node" "$original_boot_id"
-
     log "Test 3 PASSED ✓"
 }