chore: automated UAT test on GCP

.github/workflows/integration-gcp.yml

@@ -0,0 +1,163 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Integration Tests - GCP
+
+on:
+  workflow_dispatch: {}  # allow manual runs for testing
+  schedule:
+    - cron: '30 14 * * *' # daily at 14:30 UTC, runs on default branch only
+  push:
+    branches:
+      - main
+      - feature/oidc-gcp
+
+permissions:
+  contents: read
+  actions: read
+  id-token: write
+
+jobs:
+  integration-test-gcp:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      CSP: "gcp"
+      PREFIX: "nvs"
+      PROJECT_ID: "nv-dgxck8s-20250306"
+      IDENTITY_PROVIDER: "projects/1015254933832/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
+      SERVICE_ACCOUNT: "github-actions-user"
+      # Terraform Vars
+      TF_VAR_deployment_id: "d${{ github.run_id }}"
+      TF_VAR_project_id: "nv-dgxck8s-20250306"
+      TF_VAR_region: "europe-west4"
+      TF_VAR_zone: "europe-west4-b"
+      TF_VAR_system_node_type: "e2-standard-4"
+      TF_VAR_system_node_count: "3"
+      TF_VAR_gpu_node_pool_name: "gpu-pool"
+      TF_VAR_gpu_machine_type: "a3-megagpu-8g"
+      TF_VAR_gpu_node_count: "1"
+      TF_VAR_gpu_reservation_project: "nv-dgxcloudprodgsc-20240206"
+      TF_VAR_gpu_reservation_name: "gsc-a3-megagpu-8g-shared-res-2"
+      TF_VAR_gpu_driver_version: "INSTALLATION_DISABLED"
+      TF_VAR_resource_labels: '{"environment":"test","team":"nvsentinel","managed_by":"terraform"}'
+      # Debug
+      SKIP_DELETE: "false"  # skip cluster deletion
+      TEST_TAG: "main-33c1d03"
+
+    steps:
+      # Checkout
+      - name: Checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+
+      # Terraform
+      - name: Terraform
+        uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd  # v3.1.2
+        with:
+          terraform_version: "1.13.5"
+
+      # Auth
+      - name: Get AuthN Token
+        id: auth
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093  # v3
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ env.IDENTITY_PROVIDER }}
+          service_account: "${{ env.SERVICE_ACCOUNT }}@${{ env.PROJECT_ID }}.iam.gserviceaccount.com"
+
+      # Gcloud
+      - name: Setup gcloud CLI
+        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db  # v3.0.1
+
+      # Cluster
+      - name: Create Cluster
+        id: cluster
+        shell: bash
+        continue-on-error: true 
+        run: |
+          set -euo pipefail
+          cd tests/uat/gcp/cluster
+          terraform init
+          terraform apply -auto-approve
+
+      # Connect
+      - name: Connect to Cluster
+        id: client
+        if: steps.cluster.outcome == 'success'
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "Installing GKE auth plugin..."
+          gcloud components install gke-gcloud-auth-plugin --quiet --project ${{ env.TF_VAR_project_id }}
+          echo "Getting cluster credentials..."
+          gcloud container clusters get-credentials "${{ env.PREFIX }}-${{ env.TF_VAR_deployment_id }}" \
+            --zone ${{ env.TF_VAR_zone }} --project ${{ env.TF_VAR_project_id }}
+
+      # Image Tag
+      - name: Compute ref name with short SHA
+        id: ref-name
+        run: |
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            SAFE_REF="${{ github.ref_name }}"
+          elif [[ "${{ github.ref_name }}" == "main" ]]; then
+            SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
+            SAFE_REF="${{ github.ref_name }}-${SHORT_SHA}"
+          else
+            SAFE_REF="${{ env.TEST_TAG }}"
+          fi
+          # Sanitize ref name: replace slashes with hyphens for Docker tag compatibility
+          SAFE_REF=$(echo "$SAFE_REF" | sed 's/\//-/g')
+          echo "value=$SAFE_REF" >> $GITHUB_OUTPUT
+
+      # Apps
+      - name: Install NVS
+        id: apps
+        if: steps.client.outcome == 'success'
+        shell: bash
+        env:
+          GCP_PROJECT_ID: "${{ env.PROJECT_ID }}"
+          GCP_ZONE: "${{ env.TF_VAR_zone }}"
+          GCP_SERVICE_ACCOUNT: "${{ env.SERVICE_ACCOUNT }}"
+          NVSENTINEL_VERSION: "${{ steps.ref-name.outputs.value }}"
+        run: |
+          set -euxo pipefail
+          tests/uat/install-apps.sh
+
+      # Test
+      - name: Run UAT Tests
+        id: tests
+        if: steps.apps.outcome == 'success'
+        shell: bash
+        run: |
+          set -euxo pipefail
+          tests/uat/tests.sh
+
+      # Teardown
+      - name: Destroy Cluster
+        if: always() && steps.cluster.outcome != 'skipped' && env.SKIP_DELETE != 'true'
+        shell: bash
+        run: |
+          set -euxo pipefail
+          cd tests/uat/gcp/cluster
+          terraform destroy -auto-approve
+
+      # Summary
+      - name: Test Summary
+        if: always()
+        run: |
+          echo "## Test Results" >> $GITHUB_STEP_SUMMARY
+          echo "- Cluster: ${{ steps.cluster.outcome }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Connection: ${{ steps.client.outcome }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Apps: ${{ steps.apps.outcome }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Tests: ${{ steps.tests.outcome }}" >> $GITHUB_STEP_SUMMARY

distros/kubernetes/nvsentinel/charts/kubernetes-object-monitor/templates/serviceaccount.yaml

@@ -17,5 +17,4 @@ kind: ServiceAccount
 metadata:
   name: {{ include "kubernetes-object-monitor.fullname" . }}
   labels:
-    {{- include "kubernetes-object-monitor.labels" . | nindent 4 }}
-
+    {{- include "kubernetes-object-monitor.labels" . | nindent 4 }}

distros/kubernetes/nvsentinel/charts/metadata-collector/templates/daemonset.yaml

@@ -92,4 +92,4 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-
+      runtimeClassName: nvidia

tests/uat/gcp/cert-manager-values.yaml

@@ -0,0 +1,54 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+installCRDs: true
+
+# Optimize for faster startup in Kind/test environments
+global:
+  leaderElection:
+    # Reduce leader election timeout for faster startup
+    leaseDuration: 30s
+    renewDeadline: 20s
+    retryPeriod: 5s
+
+# Reduce resource requests for Kind (local testing)
+resources:
+  requests:
+    cpu: 10m
+    memory: 32Mi
+
+webhook:
+  # Reduce webhook resource requirements
+  resources:
+    requests:
+      cpu: 10m
+      memory: 32Mi
+  # Faster readiness checks
+  readinessProbe:
+    initialDelaySeconds: 3
+    periodSeconds: 3
+
+cainjector:
+  # Reduce cainjector resource requirements
+  resources:
+    requests:
+      cpu: 10m
+      memory: 32Mi
+
+startupapicheck:
+  # Reduce startup check resource requirements
+  resources:
+    requests:
+      cpu: 10m
+      memory: 32Mi

tests/uat/gcp/cluster/README.md

@@ -0,0 +1,94 @@
+# GKE Cluster Terraform Configuration
+
+This Terraform configuration creates a GKE cluster with GPU nodes for NVSentinel testing.
+
+- Single zone (zonal cluster)
+- GPU nodes use specific reservation affinity
+- Service account `gke-cluster-kubernetes@PROJECT_ID.iam.gserviceaccount.com` must exist
+
+## Prerequisites
+
+- [Terraform](https://www.terraform.io/downloads.html) `>= 1.9.5`
+- [gcloud CLI](https://cloud.google.com/sdk/docs/install) configured with appropriate credentials
+- GCP project with necessary APIs enabled:
+  - Kubernetes Engine API
+  - Compute Engine API
+
+## Known Issues
+
+⚠️ **Reservation Maintenance Interval Mismatch - RESOLVED**
+
+**Previous Issue:** gSC (Google Supercomputer) reservations require instances to have `maintenanceInterval=PERIODIC`, but GKE created instances with `maintenanceInterval=MAINTENANCE_INTERVAL_UNSPECIFIED` by default.
+
+**Solution:** The configuration now includes `host_maintenance_policy` block in the GPU node pool with `maintenance_interval = "PERIODIC"`, which resolves this issue.
+
+```hcl
+host_maintenance_policy {
+  maintenance_interval = "PERIODIC"
+}
+```
+
+## Usage
+
+1. **Initialize Terraform:**
+   ```bash
+   terraform init
+   ```
+
+2. **Configure variables (optional):**
+   ```bash
+   cp terraform.tfvars.example terraform.tfvars
+   # Edit terraform.tfvars with your values
+   ```
+
+3. **Preview changes:**
+   ```bash
+   terraform plan
+   ```
+
+4. **Create the cluster:**
+   ```bash
+   terraform apply
+   ```
+
+5. **Get kubeconfig:**
+   ```bash
+   gcloud container clusters get-credentials nvs-d2 --zone europe-west4-b --project nv-dgxck8s-20250306
+   ```
+
+   Or use the output command:
+   ```bash
+   terraform output -raw kubeconfig_command | bash
+   ```
+
+6. **Destroy the cluster:**
+   ```bash
+   terraform destroy
+   ```
+
+## Configuration Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `deployment_id` | Deployment identifier for cluster naming | `d2` |
+| `project_id` | GCP project ID | `nv-dgxck8s-20250306` |
+| `zone` | GCP zone for the cluster | `europe-west4-b` |
+| `system_node_type` | Machine type for system nodes | `e2-standard-4` |
+| `system_node_count` | Number of system nodes | `3` |
+| `gpu_node_pool_name` | Name of the GPU node pool | `gpu-pool` |
+| `gpu_machine_type` | Machine type for GPU nodes | `a3-megagpu-8g` |
+| `gpu_node_count` | Number of GPU nodes | `1` |
+| `gpu_reservation_project` | Project containing GPU reservation | `nv-dgxcloudprodgsc-20240206` |
+| `gpu_reservation_name` | Name of GPU reservation | `gsc-a3-megagpu-8g-shared-res-2` |
+| `gpu_driver_version` | GPU driver installation mode | `INSTALLATION_DISABLED` |
+| `resource_labels` | Labels to apply to resources | `{}` |
+
+
+## Outputs
+
+- `cluster_name`: Name of the created cluster
+- `cluster_location`: Zone where cluster is deployed
+- `cluster_endpoint`: API endpoint (sensitive)
+- `cluster_ca_certificate`: CA certificate (sensitive)
+- `gpu_node_pool_name`: Name of GPU node pool
+- `kubeconfig_command`: Command to configure kubectl