add GPU mock infrastructure

    - Implement pkg/gpu/mockfs for NVIDIA driver filesystem mocking
    - Implement pkg/gpu/mocktopo with dgxa100 support via go-nvml
    - Add cmd/gpu-mockctl CLI tool for mock generation
    - Add Kubernetes Job for verification on kind
    - Support extensibility for future H100/H200/B200 flavors

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>

Author: ArangoGutierrez

cmd/gpu-mockctl/main.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli/v3"
+
+	mockfs "github.com/NVIDIA/k8s-test-infra/pkg/gpu/mockfs"
+	mocktopo "github.com/NVIDIA/k8s-test-infra/pkg/gpu/mocktopo"
+)
+
+func main() {
+	cmd := &cli.Command{
+		Name:  "gpu-mockctl",
+		Usage: "Generate mock NVIDIA driver filesystem for testing",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "base",
+				Value: "/run/nvidia/driver",
+				Usage: "mock driver root directory",
+			},
+			&cli.StringFlag{
+				Name: "machine",
+				Value: func() string {
+					if v := os.Getenv("MACHINE_TYPE"); v != "" {
+						return v
+					}
+					return "dgxa100"
+				}(),
+				Usage: "machine type (only dgxa100 supported)",
+			},
+		},
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return run(
+				cmd.String("base"),
+				cmd.String("machine"),
+			)
+		},
+	}
+
+	if err := cmd.Run(context.Background(), os.Args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run(base, machine string) error {
+	topo, err := mocktopo.New(machine)
+	if err != nil {
+		if os.Getenv("ALLOW_UNSUPPORTED") == "true" {
+			log.Printf("unsupported machine %q, using fallback", machine)
+			topo = mocktopo.NewFallback(8, "NVIDIA A100-SXM4-40GB")
+		} else {
+			return fmt.Errorf("failed to create topology: %w", err)
+		}
+	}
+
+	layout := mockfs.Layout{Base: filepath.Clean(base)}
+	for _, g := range topo.GPUs {
+		layout.GPUs = append(layout.GPUs, mockfs.GPU{
+			PCI:   mockfs.NormPCI(g.PCI),
+			UUID:  g.UUID,
+			Model: g.Model,
+		})
+	}
+
+	if err := layout.Write(); err != nil {
+		return fmt.Errorf("failed to write mock filesystem: %w", err)
+	}
+
+	log.Printf(
+		"mock filesystem written under %s (%d GPUs)\n",
+		layout.Base,
+		len(layout.GPUs),
+	)
+	return nil
+}

deployments/devel/gpu-mock/00-namespace.yaml

@@ -0,0 +1,18 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gpu-mock
+

deployments/devel/gpu-mock/10-configmap-verify-script.yaml

@@ -0,0 +1,41 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gpu-mock-scripts
+  namespace: gpu-mock
+immutable: true
+data:
+  verify_gpu_mock.sh: |
+    #!/bin/sh
+    set -euo pipefail
+    BASE="${1:-/run/nvidia/driver}"
+    DEV="$BASE/dev"
+    PROC="$BASE/proc/driver/nvidia"
+    require() { test -e "$1" || { echo "MISSING: $1" >&2; exit 1; }; }
+    require_char() { [ -c "$1" ] || { echo "NOT-CHAR: $1" >&2; exit 1; }; }
+    COUNT=$(find "$PROC/gpus" -maxdepth 2 -type f -name information | wc -l | tr -d ' ')
+    [ "$COUNT" -gt 0 ] || { echo "No GPUs found in proc mock" >&2; exit 1; }
+    for i in $(seq 0 $((COUNT-1))); do require_char "$DEV/nvidia$i"; done
+    require_char "$DEV/nvidiactl"
+    require_char "$DEV/nvidia-uvm"
+    require_char "$DEV/nvidia-uvm-tools"
+    require "$PROC/version" && grep -q "NVRM version:" "$PROC/version"
+    for f in $(find "$PROC/gpus" -maxdepth 2 -type f -name information | sort); do
+      grep -q "Model:" "$f" && grep -q "GPU UUID:" "$f" && grep -q "Bus Location:" "$f" || { echo "Bad info in $f" >&2; exit 1; }
+    done
+    (cd "$BASE" && find . -maxdepth 4 -print | sort)
+    echo "ALL CHECKS PASSED"
+

deployments/devel/gpu-mock/20-job-gpu-mock-verify.yaml

@@ -0,0 +1,58 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: gpu-mock-verify
+  namespace: gpu-mock
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: gpu-mock-verify
+    spec:
+      restartPolicy: Never
+      volumes:
+        - name: mock-driver
+          emptyDir: {}
+        - name: scripts
+          configMap:
+            name: gpu-mock-scripts
+            defaultMode: 0755
+      initContainers:
+        - name: setup-mock
+          image: local/gpu-mockctl:dev
+          env:
+            - name: MACHINE_TYPE
+              value: "dgxa100"
+          securityContext:
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - name: mock-driver
+              mountPath: /run/nvidia/driver
+          command: ["/usr/local/bin/gpu-mockctl","-base","/run/nvidia/driver"]
+      containers:
+        - name: verify
+          image: alpine:3.20
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            - name: mock-driver
+              mountPath: /run/nvidia/driver
+            - name: scripts
+              mountPath: /opt/mock/bin
+          command: ["/bin/sh","-c","/opt/mock/bin/verify_gpu_mock.sh /run/nvidia/driver"]
+

deployments/devel/gpu-mock/Dockerfile

@@ -0,0 +1,29 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# builder
+FROM golang:1.25 AS builder
+WORKDIR /src
+COPY . .
+RUN --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=1 go build -trimpath -ldflags='-s -w' \
+    -o /out/gpu-mockctl ./cmd/gpu-mockctl
+
+# runtime
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash coreutils util-linux && \
+    rm -rf /var/lib/apt/lists/*
+COPY --from=builder /out/gpu-mockctl /usr/local/bin/gpu-mockctl
+ENTRYPOINT ["/usr/local/bin/gpu-mockctl"]
+

deployments/devel/gpu-mock/Makefile

@@ -0,0 +1,54 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHELL := /bin/bash
+APP := gpu-mockctl
+IMAGE ?= local/$(APP):dev
+CLUSTER ?= gpu-mock
+KIND_IMAGE ?= kindest/node:v1.30.0
+KUBECTL ?= kubectl
+REPO_ROOT := $(shell cd ../../.. && pwd)
+
+.PHONY: all kind-up image load apply wait logs clean re vendor
+
+all: kind-up image load apply wait logs
+
+vendor:
+	cd $(REPO_ROOT) && go mod tidy && go mod vendor
+
+image: vendor
+	cd $(REPO_ROOT) && docker build -t $(IMAGE) -f deployments/devel/gpu-mock/Dockerfile .
+
+load:
+	kind load docker-image $(IMAGE) --name $(CLUSTER)
+
+kind-up:
+	kind create cluster --name $(CLUSTER) --image $(KIND_IMAGE) || true
+
+apply:
+	$(KUBECTL) apply -f 00-namespace.yaml
+	$(KUBECTL) -n gpu-mock apply -f 10-configmap-verify-script.yaml
+	$(KUBECTL) -n gpu-mock apply -f 20-job-gpu-mock-verify.yaml
+
+wait:
+	$(KUBECTL) -n gpu-mock wait --for=condition=complete job/gpu-mock-verify --timeout=180s
+
+logs:
+	@pod=$$($(KUBECTL) -n gpu-mock get pods -l job-name=gpu-mock-verify -o jsonpath='{.items[0].metadata.name}'); \
+	$(KUBECTL) -n gpu-mock logs $$pod
+
+clean:
+	kind delete cluster --name $(CLUSTER) || true
+
+re: clean all
+