diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index a8ea4c2..25c20f5 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -15,6 +15,7 @@ env: REGISTRY: ghcr.io IMAGE_NAME_UBUNTU: ghcr.io/mellanox/ovn-kubernetes-dpf IMAGE_NAME_FEDORA: ghcr.io/mellanox/ovn-kubernetes-dpf-fedora + IMAGE_NAME_DPF_UTILS: ghcr.io/mellanox/ovn-kubernetes-dpf-utils jobs: build-ubuntu: @@ -135,3 +136,61 @@ jobs: cache-to: type=gha,mode=max provenance: false + build-dpf-utils: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Determine tag + id: tag + run: | + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + TAG=${GITHUB_REF#refs/tags/} + else + TAG=v25.7.1-${GITHUB_SHA::7} + fi + echo "tag=${TAG}" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (labels only) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME_DPF_UTILS }} + + - name: Build and push DPF Utils image + uses: docker/build-push-action@v5 + with: + context: ./dpf-utils + file: ./dpf-utils/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ env.IMAGE_NAME_DPF_UTILS }}:${{ steps.tag.outputs.tag }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + builder_image=quay.io/projectquay/golang:1.24 + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: false + diff --git a/.github/workflows/test-and-lint.yaml b/.github/workflows/test-and-lint.yaml new file mode 100644 index 0000000..db9ca4d --- /dev/null +++ b/.github/workflows/test-and-lint.yaml @@ -0,0 +1,45 @@ +name: Test and Lint DPF Utils + +on: + push: + branches: + - main + paths: + - 'dpf-utils/**' + - '.github/workflows/test-dpf-utils.yaml' + pull_request: + branches: + - main + paths: + - 'dpf-utils/**' + - '.github/workflows/test-dpf-utils.yaml' + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: Run tests + run: make test + + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: Run linter + run: make lint + diff --git a/Makefile b/Makefile index 9ed08f7..14e43c0 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ help: ## Display this help. REGISTRY ?= example.com OVNKUBERNETES_IMAGE ?= $(REGISTRY)/ovn-kubernetes-dpf +DPF_UTILS_IMAGE ?= $(REGISTRY)/ovn-kubernetes-dpf-utils .PHONY: docker-build-ubuntu docker-build-ubuntu: @@ -55,6 +56,15 @@ docker-build-fedora: --load \ -f Dockerfile.ovn-kubernetes.fedora . +.PHONY: docker-build-dpf-utils +docker-build-dpf-utils: ## Build DPF utilities image + docker buildx build \ + --build-arg builder_image=${GO_IMAGE} \ + -t $(DPF_UTILS_IMAGE):$(TAG) \ + --load \ + -f dpf-utils/Dockerfile \ + dpf-utils/ + .PHONY: docker-push-ubuntu docker-push-ubuntu: ## Push Ubuntu image to registry docker push $(OVNKUBERNETES_IMAGE):$(TAG) @@ -63,6 +73,22 @@ docker-push-ubuntu: ## Push Ubuntu image to registry docker-push-fedora: ## Push Fedora image to registry docker push $(OVNKUBERNETES_IMAGE):$(TAG)-fedora +.PHONY: docker-push-dpf-utils +docker-push-dpf-utils: ## Push DPF utilities image to registry + docker push $(DPF_UTILS_IMAGE):$(TAG) + +##@ DPF Utils Targets + +DPF_UTILS_DIR = dpf-utils + +.PHONY: lint +lint: golangci-lint ## Run linter for DPF utilities + cd $(DPF_UTILS_DIR) && $(GOLANGCI_LINT) run --timeout=5m ./... + +.PHONY: test +test: ## Run tests for DPF utilities + cd $(DPF_UTILS_DIR) && go test -v -coverprofile=coverage.out -covermode=atomic ./... + ##@ Helm Chart Targets HELM_CHART_DIR ?= helm/ovn-kubernetes-dpf @@ -72,7 +98,7 @@ HELM_OUTPUT_DIR ?= _output/helm helm-build: yq @mkdir -p $(HELM_OUTPUT_DIR) @cp $(HELM_CHART_DIR)/values.yaml.tmpl $(HELM_CHART_DIR)/values.yaml - @$(YQ) eval -i '.ovn-kubernetes-resource-injector.controllerManager.webhook.image.repository = "$(OVNKUBERNETES_IMAGE)"' $(HELM_CHART_DIR)/values.yaml + @$(YQ) eval -i '.ovn-kubernetes-resource-injector.controllerManager.webhook.image.repository = "$(DPF_UTILS_IMAGE)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.ovn-kubernetes-resource-injector.controllerManager.webhook.image.tag = "$(TAG)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.nodeWithDPUManifests.image.repository = "$(OVNKUBERNETES_IMAGE)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.nodeWithDPUManifests.image.tag = "$(TAG)"' $(HELM_CHART_DIR)/values.yaml @@ -80,7 +106,7 @@ helm-build: yq @$(YQ) eval -i '.nodeWithoutDPUManifests.image.tag = "$(TAG)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.dpuManifests.image.repository = "$(OVNKUBERNETES_IMAGE)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.dpuManifests.image.tag = "$(TAG)"' $(HELM_CHART_DIR)/values.yaml - @$(YQ) eval -i '.dpuManifests.imagedpf.repository = "$(OVNKUBERNETES_IMAGE)"' $(HELM_CHART_DIR)/values.yaml + @$(YQ) eval -i '.dpuManifests.imagedpf.repository = "$(DPF_UTILS_IMAGE)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.dpuManifests.imagedpf.tag = "$(TAG)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.controlPlaneManifests.image.repository = "$(OVNKUBERNETES_IMAGE)"' $(HELM_CHART_DIR)/values.yaml @$(YQ) eval -i '.controlPlaneManifests.image.tag = "$(TAG)"' $(HELM_CHART_DIR)/values.yaml @@ -103,6 +129,8 @@ helm-clean: TOOLSDIR ?= $(CURDIR)/hack/tools/bin YQ_VERSION ?= v4.45.1 export YQ ?= $(TOOLSDIR)/yq-$(YQ_VERSION) +GOLANGCI_LINT_VERSION ?= v1.62.2 +export GOLANGCI_LINT ?= $(TOOLSDIR)/golangci-lint-$(GOLANGCI_LINT_VERSION) define go-install-tool @[ -f $(1) ] || { \ @@ -120,4 +148,9 @@ $(TOOLSDIR): .PHONY: yq yq: $(YQ) ## Download yq locally if necessary $(YQ): | $(TOOLSDIR) - $(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4,$(YQ_VERSION)) \ No newline at end of file + $(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4,$(YQ_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary +$(GOLANGCI_LINT): | $(TOOLSDIR) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) \ No newline at end of file diff --git a/dpf-utils/.gitignore b/dpf-utils/.gitignore new file mode 100644 index 0000000..492e9a4 --- /dev/null +++ b/dpf-utils/.gitignore @@ -0,0 +1,3 @@ +/ovnkubernetesresourceinjector +/ipallocator +/dpucniprovisioner diff --git a/dpf-utils/Dockerfile b/dpf-utils/Dockerfile new file mode 100644 index 0000000..72d22dd --- /dev/null +++ b/dpf-utils/Dockerfile @@ -0,0 +1,113 @@ +ARG builder_image + +FROM --platform=${BUILDPLATFORM} ${builder_image} AS builder + +ARG TARGETARCH +ARG gcflags +ARG ldflags + +WORKDIR /workspace + +ARG ipallocator_dir +COPY ./ ./ +COPY go.mod go.mod +COPY go.sum go.sum + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath \ + -ldflags="${ldflags}" \ + -gcflags="${gcflags}" \ + -o ipallocator ./cmd/ipallocator + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath \ + -ldflags="${ldflags}" \ + -gcflags="${gcflags}" \ + -o dpucniprovisioner ./cmd/dpucniprovisioner + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath \ + -ldflags="${ldflags}" \ + -gcflags="${gcflags}" \ + -o ovnkubernetesresourceinjector ./cmd/ovnkubernetesresourceinjector + +# Create source code archive excluding .gocache, and test files. +# Skipping `.gocache` since it contains pre-compiled versions of packages and other build artifacts for speeding up subsequent builds +RUN mkdir src && \ + find . -name '*.go' \ + -not -path "./hack/*" \ + -not -path "./.gocache/*" \ + -not -name "*_test.go" \ + -exec cp --parents {} src/ \; && \ + tar -czf source-code.tar.gz src + +# Build the final image +FROM nvcr.io/nvidia/doca/canonical:ubuntu24.04 + +ARG TARGETARCH + +USER root + +ARG ubuntu_mirror=http://archive.ubuntu.com/ubuntu/ + +# Dependencies for installing OVN (Netplan, systemd and udev required by dpucniprovisioner). +ARG PACKAGES="openvswitch-switch netplan.io udev systemd dnsmasq" + +RUN dpkg -l | awk '/^ii/{print $2"="$3}' | sort > /initial-dpkg-list.txt + +RUN find /etc/apt/sources.list* -type f -exec sed -i \ + -e "s|http://archive.ubuntu.com/ubuntu/|${ubuntu_mirror}|g" \ + -e "s|http://security.ubuntu.com/ubuntu/|${ubuntu_mirror}|g" '{}' \; + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ${PACKAGES} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN dpkg -l | awk '/^ii/{print $2"="$3}' | sort > /after-ovn-dpkg-list.txt + +RUN mkdir -p /var/run/openvswitch + +RUN mkdir -p /usr/libexec/cni/ +COPY --from=builder /workspace/ipallocator /ipallocator +COPY --from=builder /workspace/dpucniprovisioner /cniprovisioner +COPY --from=builder /workspace/ovnkubernetesresourceinjector /ovnkubernetesresourceinjector + +# Get all the source code +RUN mkdir -p /src +WORKDIR /src + +# Copy source code from builder stage +COPY --from=builder /workspace/source-code.tar.gz ovn-kubernetes-source-code.tar.gz + +# Download source code for apt packages. +# Starting from Ubuntu 24.04 shifted to the new deb822 format for source management. +# Enable `deb-src` to be able to fetch sources using `apt-get source` +ARG PACKAGE_SOURCES +RUN test "${PACKAGE_SOURCES}" = "false" || ( \ + sed -i 's/^# deb-src/deb-src/g' /etc/apt/sources.list /etc/apt/sources.list.d/* && \ + sed -i 's/^Types: deb$/Types: deb deb-src/g' /etc/apt/sources.list.d/*.sources && \ + apt-get update && \ + apt-get source --download-only ${PACKAGES} && \ + comm -23 /after-ovn-dpkg-list.txt /initial-dpkg-list.txt | xargs -r apt-get source --download-only && \ + apt-get clean && \ + rm -f /initial-dpkg-list.txt /after-ovn-dpkg-list.txt && \ + rm -rf /var/lib/apt/lists/* && \ + cd / && \ + tar -cf source-code.tar /src && \ + rm -rf /src \ + ) + +LABEL io.k8s.display-name="ovn-kubernetes dpf utilities" \ + io.k8s.description="ovn-kubernetes dpf utilities ubuntu image" + +WORKDIR /root diff --git a/dpf-utils/cmd/dpucniprovisioner/main.go b/dpf-utils/cmd/dpucniprovisioner/main.go new file mode 100644 index 0000000..bc82337 --- /dev/null +++ b/dpf-utils/cmd/dpucniprovisioner/main.go @@ -0,0 +1,294 @@ +//go:build linux + +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "os/signal" + "strconv" + "sync" + + "github.com/nvidia/doca-platform/pkg/ipallocator" + "github.com/nvidia/doca-platform/pkg/utils/networkhelper" + dpucniprovisioner "github.com/nvidia/ovn-kubernetes-components/internal/cniprovisioner/dpu" + "github.com/nvidia/ovn-kubernetes-components/internal/readyz" + "github.com/nvidia/ovn-kubernetes-components/internal/utils/ovsclient" + + "github.com/vishvananda/netlink" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/utils/clock" + kexec "k8s.io/utils/exec" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +const ( + // vtepIPAllocationFilePath is the path to the file that contains the VTEP IP allocation done by the IP Allocator. + // We should ensure that the IP Allocation request name is vtep to have this file created correctly. + vtepIPAllocationFilePath = "/tmp/ips/vtep" + // pfIPAllocationFilePath is the path to the file that contains the PF IP allocation done by the IP Allocator. + // We should ensure that the IP Allocation request name is pf to have this file created correctly. + pfIPAllocationFilePath = "/tmp/ips/pf" +) + +func main() { + if len(os.Args) != 2 { + klog.Fatal("expecting mode to be specified via args") + } + + modeRaw := os.Args[1] + mode, err := parseMode(modeRaw) + if err != nil { + klog.Fatalf("error while parsing mode: %s", err.Error()) + } + + klog.Info("Starting DPU CNI Provisioner") + + node := os.Getenv("NODE_NAME") + if node == "" { + klog.Fatal("NODE_NAME environment variable is not found. This is supposed to be configured via Kubernetes Downward API in production") + } + + var vtepIPNet *net.IPNet + var gateway net.IP + var pfIPNet *net.IPNet + var vtepCIDR *net.IPNet + var ovnMTU int + var gatewayDiscoveryNetwork *net.IPNet + if mode == dpucniprovisioner.InternalIPAM { + vtepIPNet, gateway, err = getInfoFromVTEPIPAllocation() + if err != nil { + klog.Fatalf("error while parsing info from the VTEP IP allocation file: %s", err.Error()) + } + + pfIPNet, err = getPFIP() + if err != nil { + klog.Fatalf("error while the PF IP from the allocation file: %s", err.Error()) + } + + ovnMTU, err = getOVNMTU() + if err != nil { + klog.Fatalf("error while parsing MTU %s", err.Error()) + } + } else { + gatewayDiscoveryNetwork, err = getGatewayDiscoveryNetwork() + if err != nil { + klog.Fatalf("error while parsing the Gateway Discovery Network: %s", err.Error()) + } + } + + vtepCIDR, err = getVTEPCIDR() + if err != nil { + klog.Fatalf("error while parsing VTEP CIDR: %s", err.Error()) + } + + hostCIDR, err := getHostCIDR() + if err != nil { + klog.Fatalf("error while parsing Host CIDR %s", err.Error()) + } + + exec := kexec.New() + + ovsClient, err := ovsclient.New(exec) + if err != nil { + klog.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + c := clock.RealClock{} + + config, err := config.GetConfig() + if err != nil { + klog.Fatal(err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatal(err) + } + + provisioner := dpucniprovisioner.New(ctx, mode, c, ovsClient, networkhelper.New(), exec, clientset, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, node, gatewayDiscoveryNetwork, ovnMTU) + + err = provisioner.RunOnce() + if err != nil { + klog.Fatal(err) + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + provisioner.EnsureConfiguration() + }() + + err = readyz.ReportReady() + if err != nil { + klog.Fatal(err) + } + + klog.Info("DPU CNI Provisioner is ready") + + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt) + <-ch + klog.Info("Received termination signal, terminating.") + cancel() + provisioner.Stop() + wg.Wait() +} + +// getInfoFromVTEPIPAllocation returns the VTEP IP and gateway from a file that contains the VTEP IP allocation done +// by the IP Allocator component. +func getInfoFromVTEPIPAllocation() (*net.IPNet, net.IP, error) { + content, err := os.ReadFile(vtepIPAllocationFilePath) + if err != nil { + return nil, nil, fmt.Errorf("error while reading file %s: %w", vtepIPAllocationFilePath, err) + } + + results := []ipallocator.NVIPAMIPAllocatorResult{} + if err := json.Unmarshal(content, &results); err != nil { + return nil, nil, fmt.Errorf("error while unmarshalling IP Allocator results: %w", err) + } + + if len(results) != 1 { + return nil, nil, fmt.Errorf("expecting exactly 1 IP allocation for VTEP") + } + + vtepIPRaw := results[0].IP + vtepIP, err := netlink.ParseIPNet(vtepIPRaw) + if err != nil { + return nil, nil, fmt.Errorf("error while parsing VTEP IP to net.IPNet: %w", err) + } + + gatewayRaw := results[0].Gateway + gateway := net.ParseIP(gatewayRaw) + if gateway == nil { + return nil, nil, errors.New("error while parsing Gateway IP to net.IP: input is not valid") + } + + return vtepIP, gateway, nil +} + +// getPFIP() returns the PF IP from a file that contains the PF IP allocation done by the IP Allocator +// component. +func getPFIP() (*net.IPNet, error) { + content, err := os.ReadFile(pfIPAllocationFilePath) + if err != nil { + return nil, fmt.Errorf("error while reading file %s: %w", vtepIPAllocationFilePath, err) + } + + results := []ipallocator.NVIPAMIPAllocatorResult{} + if err := json.Unmarshal(content, &results); err != nil { + return nil, fmt.Errorf("error while unmarshalling IP Allocator results: %w", err) + } + + if len(results) != 1 { + return nil, fmt.Errorf("expecting exactly 1 IP allocation for PF") + } + + pfIPRaw := results[0].IP + pfIP, err := netlink.ParseIPNet(pfIPRaw) + if err != nil { + return nil, fmt.Errorf("error while parsing PF IP to net.IPNet: %w", err) + } + + return pfIP, nil +} + +// getVTEPCIDR returns the VTEP CIDR to be used by the provisioner +func getVTEPCIDR() (*net.IPNet, error) { + vtepCIDRRaw := os.Getenv("VTEP_CIDR") + if vtepCIDRRaw == "" { + return nil, errors.New("required VTEP_CIDR environment variable is not set") + } + + _, vtepCIDR, err := net.ParseCIDR(vtepCIDRRaw) + if err != nil { + klog.Fatalf("error while parsing VTEP CIDR %s as net.IPNet: %s", vtepCIDRRaw, err.Error()) + } + + return vtepCIDR, nil +} + +// getHostCIDR returns the Host CIDR to be used by the provisioner +func getHostCIDR() (*net.IPNet, error) { + hostCIDRRaw := os.Getenv("HOST_CIDR") + if hostCIDRRaw == "" { + return nil, errors.New("required HOST_CIDR environment variable is not set") + } + + _, hostCIDR, err := net.ParseCIDR(hostCIDRRaw) + if err != nil { + klog.Fatalf("error while parsing Host CIDR %s as net.IPNet: %s", hostCIDRRaw, err.Error()) + } + + return hostCIDR, nil +} + +// getGatewayDiscoveryNetwork returns the Network to be used by the provisioner to discover the gateway +func getGatewayDiscoveryNetwork() (*net.IPNet, error) { + gatewayDiscoveryNetworkRaw := os.Getenv("GATEWAY_DISCOVERY_NETWORK") + if gatewayDiscoveryNetworkRaw == "" { + return nil, errors.New("required GATEWAY_DISCOVERY_NETWORK environment variable is not set") + } + + _, gatewayDiscoveryNetwork, err := net.ParseCIDR(gatewayDiscoveryNetworkRaw) + if err != nil { + klog.Fatalf("error while parsing Gateway Discovery Network %s as net.IPNet: %s", gatewayDiscoveryNetwork, err.Error()) + } + + return gatewayDiscoveryNetwork, nil +} + +// getOVNMTU returns the PF MTU to be used by the provisioner +func getOVNMTU() (int, error) { + mtuString := os.Getenv("OVN_MTU") + if mtuString == "" { + return 0, errors.New("required OVN_MTU environment variable is not set") + } + + mtu, err := strconv.Atoi(mtuString) + if err != nil { + return 0, fmt.Errorf("parse environment variable OVN_MTU %s as int: %v", mtuString, err) + } + + if mtu == 0 { + return 0, errors.New("invalid OVN_MTU value: 0") + } + + return mtu, nil +} + +// parseMode parses the mode in which the binary should be started +func parseMode(mode string) (dpucniprovisioner.Mode, error) { + m := map[dpucniprovisioner.Mode]struct{}{ + dpucniprovisioner.InternalIPAM: {}, + dpucniprovisioner.ExternalIPAM: {}, + } + modeTyped := dpucniprovisioner.Mode(mode) + if _, ok := m[modeTyped]; !ok { + return "", errors.New("unknown mode") + } + + return modeTyped, nil +} diff --git a/dpf-utils/cmd/ipallocator/main.go b/dpf-utils/cmd/ipallocator/main.go new file mode 100644 index 0000000..f203789 --- /dev/null +++ b/dpf-utils/cmd/ipallocator/main.go @@ -0,0 +1,167 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + + "github.com/nvidia/doca-platform/pkg/ipallocator" + "github.com/nvidia/ovn-kubernetes-components/internal/readyz" + + "github.com/containernetworking/cni/libcni" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" +) + +// requiredEnvVariables are environment variables that the program needs in order to run. Boolean indicates if this env +// variable should be configured by Kubernetes downward API (easier for user to understand the mistake). +var requiredEnvVariables = map[string]bool{ + "IP_ALLOCATOR_REQUESTS": true, + "K8S_POD_NAME": true, + "K8S_POD_NAMESPACE": true, + "K8S_POD_UID": true, +} + +type Mode string + +// String returns the string representation of the mode +func (m Mode) String() string { + return string(m) +} + +const ( + Allocator Mode = "allocator" + Deallocator Mode = "deallocator" +) + +func main() { + if len(os.Args) != 2 { + klog.Fatal("expecting mode to be specified via args") + } + + modeRaw := os.Args[1] + mode, err := parseMode(modeRaw) + if err != nil { + klog.Fatalf("error while parsing mode: %s", err.Error()) + } + + klog.Info("Starting IP Allocator") + env, err := parseEnv() + if err != nil { + klog.Fatalf("error while parsing environment: %s", err.Error()) + } + + allocator := ipallocator.New( + libcni.NewCNIConfig([]string{ipallocator.CNIBinDir}, nil), + env["K8S_POD_NAME"], + env["K8S_POD_NAMESPACE"], + env["K8S_POD_UID"], + ) + + reqs, err := allocator.ParseRequests(env["IP_ALLOCATOR_REQUESTS"]) + if err != nil { + klog.Fatalf("error while parsing IP requests: %s", err.Error()) + } + + switch mode { + case Allocator: + if err := runInAllocatorMode(allocator, reqs); err != nil { + klog.Fatal(err) + } + case Deallocator: + if err := runInDeallocatorMode(allocator, reqs); err != nil { + klog.Fatal(err) + } + } +} + +// parseMode parses the mode in which the binary should be started +func parseMode(mode string) (Mode, error) { + m := map[Mode]struct{}{ + Allocator: {}, + Deallocator: {}, + } + modeTyped := Mode(mode) + if _, ok := m[modeTyped]; !ok { + return "", errors.New("unknown mode") + } + + return modeTyped, nil +} + +// runInAllocatorMode runs the allocator in Allocator mode +func runInAllocatorMode(a *ipallocator.NVIPAMIPAllocator, reqs []ipallocator.NVIPAMIPAllocatorRequest) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, req := range reqs { + if err := a.Allocate(ctx, req); err != nil { + return err + } + } + + if err := readyz.ReportReady(); err != nil { + return err + } + + klog.Info("IP allocation is done") + + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt) + <-ch + klog.Info("Received termination signal, terminating.") + + return nil +} + +// runInDeallocatorMode runs the allocator in Deallocator mode +func runInDeallocatorMode(a *ipallocator.NVIPAMIPAllocator, reqs []ipallocator.NVIPAMIPAllocatorRequest) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, req := range reqs { + if err := a.Deallocate(ctx, req); err != nil { + return err + } + } + + klog.Info("IP deallocation is done") + return nil +} + +// parseEnv parses the required environment variables +func parseEnv() (map[string]string, error) { + var errs []error + m := make(map[string]string) + for key, expectedViaDownwardAPI := range requiredEnvVariables { + value := os.Getenv(key) + if value == "" { + err := fmt.Errorf("required %s environment variable is not found", key) + if expectedViaDownwardAPI { + err = errors.Join(err, errors.New("this is supposed to be configured via Kubernetes Downward API in production")) + } + errs = append(errs, err) + } + m[key] = value + } + + return m, kerrors.NewAggregate(errs) +} diff --git a/dpf-utils/cmd/ovnkubernetesresourceinjector/main.go b/dpf-utils/cmd/ovnkubernetesresourceinjector/main.go new file mode 100644 index 0000000..3dfc7e3 --- /dev/null +++ b/dpf-utils/cmd/ovnkubernetesresourceinjector/main.go @@ -0,0 +1,158 @@ +/* +COPYRIGHT 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "flag" + "os" + "time" + + "github.com/nvidia/ovn-kubernetes-components/internal/ovnkubernetesresourceinjector/webhooks" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var syncPeriod time.Duration + var nadName string + var nadNamespace string + + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", false, + "If set the metrics endpoint is served securely") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, + "The minimum interval at which watched resources are reconciled.") + flag.StringVar(&nadName, "nad-name", "dpf-ovn-kubernetes", + "The name of the NetworkAttachmentDefinition the VF injector should use") + flag.StringVar(&nadNamespace, "nad-namespace", "ovn-kubernetes", + "The namespace of the NetworkAttachmentDefinition the VF injector should use") + + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancelation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + tlsOpts := []func(*tls.Config){} + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: tlsOpts, + }) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + }, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + Cache: cache.Options{ + SyncPeriod: &syncPeriod, + }, + LeaderElection: enableLeaderElection, + LeaderElectionID: "8a3114c5.dpu.nvidia.com", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if err = (&webhooks.NetworkInjector{ + Client: mgr.GetClient(), + Settings: webhooks.NetworkInjectorSettings{ + NADName: nadName, + NADNamespace: nadNamespace, + }, + }).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DPFOperatorConfig") + os.Exit(1) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/dpf-utils/go.mod b/dpf-utils/go.mod new file mode 100644 index 0000000..c39a9c2 --- /dev/null +++ b/dpf-utils/go.mod @@ -0,0 +1,76 @@ +module github.com/nvidia/ovn-kubernetes-components + +go 1.24.0 + +require ( + github.com/containernetworking/cni v1.2.3 + github.com/nvidia/doca-platform v0.0.0-20251016072527-fc9761393d62 + github.com/onsi/ginkgo/v2 v2.23.4 + github.com/onsi/gomega v1.36.3 + github.com/vishvananda/netlink v1.3.0 + go.uber.org/mock v0.5.0 + k8s.io/api v0.33.0 + k8s.io/apimachinery v0.33.0 + k8s.io/client-go v0.33.0 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20241210054802-24370beab758 + sigs.k8s.io/controller-runtime v0.21.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/pflag v1.0.6-0.20201009195203-85dd5c8bc61c // indirect + github.com/vishvananda/netns v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/automaxprocs v1.6.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.31.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.33.0 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/dpf-utils/go.sum b/dpf-utils/go.sum new file mode 100644 index 0000000..3386851 --- /dev/null +++ b/dpf-utils/go.sum @@ -0,0 +1,204 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8FuJbEslXM= +github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nvidia/doca-platform v0.0.0-20251016072527-fc9761393d62 h1:pC6+1lK6LUiM3hQn0GSjpFnF+h6w+jc0ly1CFItWbk8= +github.com/nvidia/doca-platform v0.0.0-20251016072527-fc9761393d62/go.mod h1:kpkfdIqTf7Utu/MUYlqAxa4M4jYCRK/izj8OGpbfQ/c= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= +github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.6-0.20201009195203-85dd5c8bc61c h1:zqmyTlQyufRC65JnImJ6H1Sf7BDj8bG31EV919NVEQc= +github.com/spf13/pflag v1.0.6-0.20201009195203-85dd5c8bc61c/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.33.0 h1:yTgZVn1XEe6opVpP1FylmNrIFWuDqe2H0V8CT5gxfIU= +k8s.io/api v0.33.0/go.mod h1:CTO61ECK/KU7haa3qq8sarQ0biLq2ju405IZAd9zsiM= +k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= +k8s.io/apiextensions-apiserver v0.33.0/go.mod h1:VeJ8u9dEEN+tbETo+lFkwaaZPg6uFKLGj5vyNEwwSzc= +k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ= +k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.0 h1:UASR0sAYVUzs2kYuKn/ZakZlcs2bEHaizrrHUZg0G98= +k8s.io/client-go v0.33.0/go.mod h1:kGkd+l/gNGg8GYWAPr0xF1rRKvVWvzh9vmZAMXtaKOg= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/dpf-utils/internal/cniprovisioner/dpu/dpu_suite_test.go b/dpf-utils/internal/cniprovisioner/dpu/dpu_suite_test.go new file mode 100644 index 0000000..61b8c30 --- /dev/null +++ b/dpf-utils/internal/cniprovisioner/dpu/dpu_suite_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dpucniprovisioner_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDPUCNIProvisioner(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DPU CNI Provisioner Suite") +} diff --git a/dpf-utils/internal/cniprovisioner/dpu/provisioner.go b/dpf-utils/internal/cniprovisioner/dpu/provisioner.go new file mode 100644 index 0000000..9b0b956 --- /dev/null +++ b/dpf-utils/internal/cniprovisioner/dpu/provisioner.go @@ -0,0 +1,590 @@ +/* +Copyright 2024 NVIDIA. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dpucniprovisioner + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/nvidia/doca-platform/pkg/utils/networkhelper" + "github.com/nvidia/ovn-kubernetes-components/internal/constants" + "github.com/nvidia/ovn-kubernetes-components/internal/utils/ovsclient" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/utils/clock" + kexec "k8s.io/utils/exec" + "k8s.io/utils/ptr" +) + +type Mode string + +// String returns the string representation of the mode +func (m Mode) String() string { + return string(m) +} + +const ( + // InternalIPAM is the mode where IPAM is managed by DPUServiceIPAM objects and we have to provide the IPs to br-ovn + // and the PF on the host + InternalIPAM Mode = "internal-ipam" + // ExternalIPAM is the mode where an external DHCP server provides the IPs to the br-ovn and PF on the host + ExternalIPAM Mode = "external-ipam" + // geneveHeaderSize is the size of the geneve header, which is 60 bytes. + geneveHeaderSize = 60 + // maxMTUSize is the maximum MTU size that can be set on a network interface. + maxMTUSize = 9216 + // flannelInterface is the flannel interface that has an IP address in the network where the pods on the local node + // will get IPs from. + flannelInterface = "cni0" + // oobInterface is the out of band interface used by the DPU. This interface is hardcoded to the bridge name used in + // the provisioning controller. + // TODO: Consider discovering that in the future + oobInterface = "br-comm-ch" + // sourceRoutingTable is the route table used for source routing + sourceRoutingTable = 60 +) + +const ( + // brOVN is the name of the bridge that is used by OVN as the external bridge (br-ex). This is the bridge that is + // later connected with br-sfc. In the current OVN IC w/ DPU implementation, the internal port of this bridge acts + // as the VTEP. + brOVN = "br-ovn" + brDPU = "br-dpu" + + // ovnkInputPath is the path to the file in which ovnkube-controller expects the additional gateway opts + ovnkInputPath = "/etc/openvswitch/ovn_k8s.conf" + // brOVNNetplanConfigPath is the path to the file which contains the netplan configuration for br-ovn + brOVNNetplanConfigPath = "/etc/netplan/80-br-ovn.yaml" + // netplanApplyDonePath is a file that indicates that a netplan apply has already ran and was successful. The content + // of the file is used to create a cooldown period before re-issuing a subsequent netplan apply command. + netplanApplyDonePath = "/etc/netplan/.dpucniprovisioner.done" + // netplanApplyCooldownDuration determines the cooldown period of a successful netplan apply command before a + // subsequent netplan apply command is executed. + netplanApplyCooldownDuration = time.Minute * 2 +) + +type DPUCNIProvisioner struct { + ctx context.Context + clock clock.Clock + ensureConfigurationTicker clock.Ticker + ovsClient ovsclient.OVSClient + networkHelper networkhelper.NetworkHelper + exec kexec.Interface + kubernetesClient kubernetes.Interface + + // FileSystemRoot controls the file system root. It's used for enabling easier testing of the package. Defaults to + // empty. + FileSystemRoot string + + // vtepIPNet is the IP that should be added to the VTEP interface. + vtepIPNet *net.IPNet + // gateway is the gateway IP that is configured on the routes related to OVN Kubernetes reaching its peer nodes + // when traffic needs to go from one Pod running on Node A to another Pod running on Node B. + gateway net.IP + // vtepCIDR is the CIDR in which all the VTEP IPs of all the DPUs in the DPU cluster belong to. This CIDR is + // configured on the routes related to traffic that needs to go from one Pod running on worker Node A to another Pod + // running on worker Node B. + vtepCIDR *net.IPNet + // hostCIDR is the CIDR of the host machines that is configured on the routes related to OVN Kubernetes reaching + // its peer nodes when traffic needs to go from one Pod running on worker Node A to another Pod running on control + // plane A (and vice versa). + hostCIDR *net.IPNet + // pfIP is the IP that should be added to the PF on the host + pfIP *net.IPNet + // dpuHostName is the name of the DPU. + dpuHostName string + // gatewayDiscoveryNetwork is the network from which the DPUCNIProvisioner discovers the gateway that it should be + // on relevant underlying systems. + gatewayDiscoveryNetwork *net.IPNet + + // dhcpCmd is the struct that holds information about the DHCP Server process + dhcpCmd kexec.Cmd + // mode is the mode in which the CNI provisioner is running + mode Mode + // ovnMTU is the MTU that is configured for OVN + ovnMTU int +} + +// New creates a DPUCNIProvisioner that can configure the system +func New(ctx context.Context, + mode Mode, + clock clock.WithTicker, + ovsClient ovsclient.OVSClient, + networkHelper networkhelper.NetworkHelper, + exec kexec.Interface, + kubernetesClient kubernetes.Interface, + vtepIPNet *net.IPNet, + gateway net.IP, + vtepCIDR *net.IPNet, + hostCIDR *net.IPNet, + pfIP *net.IPNet, + dpuHostName string, + gatewayDiscoveryNetwork *net.IPNet, + ovnMTU int, +) *DPUCNIProvisioner { + return &DPUCNIProvisioner{ + ctx: ctx, + clock: clock, + ensureConfigurationTicker: clock.NewTicker(30 * time.Second), + ovsClient: ovsClient, + networkHelper: networkHelper, + exec: exec, + kubernetesClient: kubernetesClient, + FileSystemRoot: "", + vtepIPNet: vtepIPNet, + gateway: gateway, + vtepCIDR: vtepCIDR, + hostCIDR: hostCIDR, + pfIP: pfIP, + dpuHostName: dpuHostName, + mode: mode, + gatewayDiscoveryNetwork: gatewayDiscoveryNetwork, + ovnMTU: ovnMTU, + } +} + +// RunOnce runs the provisioning flow once and exits +func (p *DPUCNIProvisioner) RunOnce() error { + if err := p.configure(); err != nil { + return err + } + klog.Info("Configuration complete.") + if p.mode == InternalIPAM { + if err := p.startDHCPServer(); err != nil { + return fmt.Errorf("error while starting DHCP server: %w", err) + } + klog.Info("DHCP Server started.") + } + + return nil +} + +// Stop stops the provisioner +func (p *DPUCNIProvisioner) Stop() { + if p.mode == InternalIPAM { + p.dhcpCmd.Stop() + } + + klog.Info("Provisioner stopped") +} + +// EnsureConfiguration ensures that particular configuration is in place. This is a blocking function. +func (p *DPUCNIProvisioner) EnsureConfiguration() { + for { + select { + case <-p.ctx.Done(): + return + case <-p.ensureConfigurationTicker.C(): + if err := p.configure(); err != nil { + klog.Errorf("failed to ensure configuration: %s", err.Error()) + } + } + } +} + +// configure runs the provisioning flow once +func (p *DPUCNIProvisioner) configure() error { + klog.Info("Configuring Kubernetes host name in OVS") + if err := p.findAndSetKubernetesHostNameInOVS(); err != nil { + return fmt.Errorf("error while setting the Kubernetes Host Name in OVS: %w", err) + } + + if p.mode == ExternalIPAM { + klog.Info("Configuring br-ovn") + if err := p.configureBROVN(); err != nil { + return fmt.Errorf("error while configuring br-ovn: %w", err) + } + } + + klog.Info("Configuring system to enable pod to pod on different node connectivity") + if err := p.configurePodToPodOnDifferentNodeConnectivity(); err != nil { + return err + } + + klog.Info("Writing OVN Kubernetes expected input files") + if err := p.writeFilesForOVN(); err != nil { + return err + } + + klog.Info("Configuring symmetric routing") + if err := p.configureSymmetricRouting(); err != nil { + return err + } + + return nil +} + +// findAndSetKubernetesHostNameInOVS discovers and sets the Kubernetes Host Name in OVS +func (p *DPUCNIProvisioner) findAndSetKubernetesHostNameInOVS() error { + nodeClient := p.kubernetesClient.CoreV1().Nodes() + n, err := nodeClient.Get(p.ctx, p.dpuHostName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error while getting Kubernetes Node: %w", err) + } + hostName, ok := n.Labels[constants.HostNameDPULabelKey] + if !ok { + return fmt.Errorf("required label %s is not set on node %s in the DPU cluster", constants.HostNameDPULabelKey, p.dpuHostName) + } + + if err := p.ovsClient.SetKubernetesHostNodeName(hostName); err != nil { + return fmt.Errorf("error while setting the Kubernetes Host Name in OVS: %w", err) + } + return nil +} + +// configurePodToPodOnDifferentNodeConnectivity configures the VTEP interface (br-ovn) and the ovn-encap-ip external ID +// so that traffic going through the geneve tunnels can function as expected. +func (p *DPUCNIProvisioner) configurePodToPodOnDifferentNodeConnectivity() error { + if p.mode == InternalIPAM { + if err := p.setLinkIPAddressIfNotSet(brOVN, p.vtepIPNet); err != nil { + return fmt.Errorf("error while setting VTEP IP: %w", err) + } + if err := p.networkHelper.SetLinkUp(brOVN); err != nil { + return fmt.Errorf("error while setting link %s up: %w", brOVN, err) + } + + _, vtepNetwork, err := net.ParseCIDR(p.vtepIPNet.String()) + if err != nil { + return fmt.Errorf("error while parsing network from VTEP IP %s: %w", p.vtepIPNet.String(), err) + } + + if vtepNetwork.String() != p.vtepCIDR.String() { + // Add route related to traffic that needs to go from one Pod running on worker Node A to another Pod running + // on worker Node B. + if err := p.addRouteIfNotExists(p.vtepCIDR, p.gateway, brOVN, nil, nil); err != nil { + return fmt.Errorf("error while adding route %s %s %s: %w", p.vtepCIDR, p.gateway.String(), brOVN, err) + } + } + } + + // Add route related to traffic that needs to go from one Pod running on worker Node A to another Pod running on + // control plane A (and vice versa). + // + // In our setup, we will already have a route pointing to the same CIDR via the SF designated for kubelet traffic + // which gets a DHCP IP in that CIDR. Given that, we need to set the metric of this route to something very high + // so that it's the last preferred route in the route table for that CIDR. The reason for that is this OVS bug that + // selects the route with the highest prio - see issue 3871067. + if err := p.addRouteIfNotExists(p.hostCIDR, p.gateway, brOVN, ptr.To[int](10000), nil); err != nil { + return fmt.Errorf("error while adding route %s %s %s: %w", p.hostCIDR, p.gateway.String(), brOVN, err) + } + + if err := p.ovsClient.SetOVNEncapIP(p.vtepIPNet.IP); err != nil { + return fmt.Errorf("error while setting the OVN Encap IP: %w", err) + } + + return nil +} + +// setLinkIPAddressIfNotSet sets an IP address to a link if it's not already set +func (p *DPUCNIProvisioner) setLinkIPAddressIfNotSet(link string, ipNet *net.IPNet) error { + hasIP, err := p.networkHelper.LinkIPAddressExists(link, ipNet) + if err != nil { + return fmt.Errorf("error checking whether IP exists: %w", err) + } + if hasIP { + klog.Infof("Link %s has IP %s, skipping configuration", link, ipNet) + return nil + } + if err := p.networkHelper.SetLinkIPAddress(link, ipNet); err != nil { + return fmt.Errorf("error setting IP address: %w", err) + } + return nil +} + +// addRouteIfNotExists adds a route if it doesn't already exist +func (p *DPUCNIProvisioner) addRouteIfNotExists(network *net.IPNet, gateway net.IP, device string, metric *int, table *int) error { + hasRoute, err := p.networkHelper.RouteExists(network, gateway, device, table) + if err != nil { + return fmt.Errorf("error checking whether route exists: %w", err) + } + if hasRoute { + klog.Infof("Route %s %s %s with metric %v in table %v exists, skipping configuration", network, gateway, device, metric, table) + return nil + } + if err := p.networkHelper.AddRoute(network, gateway, device, metric, table); err != nil { + return fmt.Errorf("error adding route: %w", err) + } + return nil +} + +// addRuleIfNotExists adds a rule if it doesn't already exist +func (p *DPUCNIProvisioner) addRuleIfNotExists(network *net.IPNet, table int, priority int) error { + hasRule, err := p.networkHelper.RuleExists(network, table, priority) + if err != nil { + return fmt.Errorf("error checking whether rule exists: %w", err) + } + if hasRule { + klog.Infof("Rule %v %d %d exists, skipping configuration", network, table, priority) + return nil + } + if err := p.networkHelper.AddRule(network, table, priority); err != nil { + return fmt.Errorf("error adding rule: %w", err) + } + return nil +} + +// writeFilesForOVN writes the input files that the ovnkube-controller expects +func (p *DPUCNIProvisioner) writeFilesForOVN() error { + configPath := filepath.Join(p.FileSystemRoot, ovnkInputPath) + + // Build the complete content in one operation + content := "[Gateway]\n" + content += p.writeOVNInputGatewayOptsFile() + + routerSubnetContent, err := p.writeOVNInputRouterSubnetPath() + if err != nil { + return fmt.Errorf("error while getting the gateway router subnet content: %w", err) + } + content += routerSubnetContent + + // Write the complete content to the file in one operation + err = os.WriteFile(configPath, []byte(content), 0644) + if err != nil { + return fmt.Errorf("error writing to file %s: %w", configPath, err) + } + + return nil +} + +// configureBROVN requests an IP via DHCP for br-ovn and mutates the relevant fields of the DPUCNIProvisioner objects +func (p *DPUCNIProvisioner) configureBROVN() error { + if err := p.writeNetplanFileForBROVN(); err != nil { + return fmt.Errorf("error while br-ovn netplan: %w", err) + } + + addrs, err := p.networkHelper.GetLinkIPAddresses(brOVN) + if err != nil { + return fmt.Errorf("error while getting IP addresses for link %s: %w", brOVN, err) + } + + if len(addrs) != 1 { + if err := p.runNetplanApply(); err != nil { + return fmt.Errorf("error running netplan apply: %w", err) + } + + return fmt.Errorf("exactly 1 IP is expected in %s, but found %d", brOVN, len(addrs)) + } + + p.vtepIPNet = addrs[0] + + gateway, err := p.networkHelper.GetGateway(p.gatewayDiscoveryNetwork) + if err != nil { + return fmt.Errorf("error while parsing gateway from gateway discovery network %s: %w", p.gatewayDiscoveryNetwork.String(), err) + } + + p.gateway = gateway + return nil +} + +// writeOVNInputGatewayOptsFile returns the gateway options content +// that ovnkube-controller reads from. +func (p *DPUCNIProvisioner) writeOVNInputGatewayOptsFile() string { + return "next-hop=" + p.gateway.String() + "\n" +} + +// writeOVNInputRouterSubnetPath returns the Gateway Router Subnet content +// that kubeovn-controller reads. +func (p *DPUCNIProvisioner) writeOVNInputRouterSubnetPath() (string, error) { + _, vtepNetwork, err := net.ParseCIDR(p.vtepIPNet.String()) + if err != nil { + return "", fmt.Errorf("error while parsing network from VTEP IP %s: %w", p.vtepIPNet.String(), err) + } + return "router-subnet=" + vtepNetwork.String() + "\n", nil +} + +// writeNetplanFileForBROVN writes a netplan file for br-ovn to request dhcp +func (p *DPUCNIProvisioner) writeNetplanFileForBROVN() error { + configPath := filepath.Join(p.FileSystemRoot, brOVNNetplanConfigPath) + content := fmt.Sprintf(` +network: + renderer: networkd + version: 2 + bridges: + %s: + dhcp4: yes + dhcp4-overrides: + use-dns: no + openvswitch: {} +`, brOVN) + if err := os.WriteFile(configPath, []byte(content), 0600); err != nil { + return fmt.Errorf("error while writing file %s: %w", configPath, err) + } + + return nil +} + +// runNetplanApply runs netplan apply while respecting the cooldown period after a successful netplan apply +func (p *DPUCNIProvisioner) runNetplanApply() error { + applyDonePath := filepath.Join(p.FileSystemRoot, netplanApplyDonePath) + lastSuccessfulRunTimestampRaw, err := os.ReadFile(applyDonePath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("error while reading %s: %w", applyDonePath, err) + } + + var lastSuccessfulRunTimestampRawInt int + if string(lastSuccessfulRunTimestampRaw) != "" { + lastSuccessfulRunTimestampRawInt, err = strconv.Atoi(string(lastSuccessfulRunTimestampRaw)) + if err != nil { + return fmt.Errorf("error parsing timestamp from string %s: %w", lastSuccessfulRunTimestampRaw, err) + } + } + + lastSuccessfulRunTimestamp := time.Unix(int64(lastSuccessfulRunTimestampRawInt), 0) + if lastSuccessfulRunTimestamp.Add(netplanApplyCooldownDuration).After(p.clock.Now()) { + klog.Info("netplan apply is in cool down period, skipping apply") + return nil + } + + cmd := p.exec.Command("netplan", "apply") + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.SetStdout(&stdout) + cmd.SetStderr(&stderr) + if err := cmd.Run(); err != nil { + return fmt.Errorf("error running netplan: stdout='%s' stderr='%s': %w", stdout.String(), stderr.String(), err) + } + + if err = os.WriteFile(applyDonePath, []byte(strconv.Itoa(int(p.clock.Now().Unix()))), 0644); err != nil { + return fmt.Errorf("error writing file %s: %w", applyDonePath, err) + } + + return nil +} + +// startDHCPServer starts a DHCP Server to enable the PF on the host to get an IP. +func (p *DPUCNIProvisioner) startDHCPServer() error { + if p.dhcpCmd != nil { + klog.Warning("DHCP Server already running") + return nil + } + + _, vtepNetwork, err := net.ParseCIDR(p.vtepIPNet.String()) + if err != nil { + return fmt.Errorf("error while parsing network from VTEP IP %s: %w", p.vtepIPNet.String(), err) + } + + mac, err := p.networkHelper.GetPFRepMACAddress("pf0hpf") + if err != nil { + return fmt.Errorf("error while parsing MAC address of the PF on the host: %w", err) + } + + // Add the geneve header size to the MTU. + pfMTU := p.ovnMTU + geneveHeaderSize + + if pfMTU == geneveHeaderSize || pfMTU > maxMTUSize { + return errors.New("invalid PF MTU: it must be greater than 60 and less than or equal to 9216") + } + + args := []string{ + "--keep-in-foreground", + "--port=0", // Disable DNS Server + "--log-facility=-", // Log to stderr + fmt.Sprintf("--interface=%s", brOVN), + "--dhcp-option=option:router", + fmt.Sprintf("--dhcp-option=option:mtu,%d", pfMTU), + fmt.Sprintf("--dhcp-range=%s,static", vtepNetwork.IP.String()), + fmt.Sprintf("--dhcp-host=%s,%s", mac, p.pfIP.IP.String()), + } + + if vtepNetwork.String() != p.vtepCIDR.String() { + args = append(args, fmt.Sprintf("--dhcp-option=option:classless-static-route,%s,%s", p.vtepCIDR.String(), p.gateway.String())) + } + + cmd := p.exec.Command("dnsmasq", args...) + + cmd.SetStdout(os.Stdout) + cmd.SetStderr(os.Stderr) + if err := cmd.Start(); err != nil { + return fmt.Errorf("error while starting the DHCP server: %w", err) + } + + p.dhcpCmd = cmd + return nil +} + +// configureSymmetricRouting configures source routing to avoid asymmetric routing for the following 2 flows: +// * When source address is an IP belonging to the DPU (OOB), traffic should always go back via the OOB +// * When source address is a primary CNI address of a Pod on the DPUCluster, traffic should always go back via the OOB +// This feature is essential so that DPUService ConfigPorts feature is working as expected. +func (p *DPUCNIProvisioner) configureSymmetricRouting() error { + // When source address is a Pod on the DPUCluster, traffic should always go back via the OOB + flannelInterfaceIPs, err := p.networkHelper.GetLinkIPAddresses(flannelInterface) + if err != nil { + return fmt.Errorf("error while getting IP addresses for link %s: %w", flannelInterface, err) + } + + if len(flannelInterfaceIPs) != 1 { + return fmt.Errorf("flannel interface %s is expected to have a single address", flannelInterface) + } + + _, flannelNetwork, err := net.ParseCIDR(flannelInterfaceIPs[0].String()) + if err != nil { + return fmt.Errorf("error while parsing the local CNI network: %w", err) + } + + // ip rule a prio 31000 from 10.244.6.0/24 lookup 60 + if err := p.addRuleIfNotExists(flannelNetwork, sourceRoutingTable, 31000); err != nil { + return fmt.Errorf("error while adding rule: %w", err) + } + + // When source address is an IP belonging to the DPU (OOB), traffic should always go back via the OOB + oobInterfaceIPs, err := p.networkHelper.GetLinkIPAddresses(oobInterface) + if err != nil { + return fmt.Errorf("error while getting IP addresses for link %s: %w", oobInterface, err) + } + + if len(oobInterfaceIPs) != 1 { + return fmt.Errorf("oob interface %s is expected to have a single address", oobInterface) + } + + oobInterfaceIP := &net.IPNet{ + IP: oobInterfaceIPs[0].IP, + Mask: net.CIDRMask(32, 32), + } + + // ip rule a prio 32000 from 10.0.110.70/32 lookup 60 + if err := p.addRuleIfNotExists(oobInterfaceIP, sourceRoutingTable, 32000); err != nil { + return fmt.Errorf("error while adding rule: %w", err) + } + + _, defaultRouteNetwork, err := net.ParseCIDR("0.0.0.0/0") + if err != nil { + return fmt.Errorf("error while parsing 0.0.0.0/0 as network: %w", err) + } + + defaultGateway, err := p.networkHelper.GetGateway(defaultRouteNetwork) + if err != nil { + return fmt.Errorf("error while parsing gateway %s: %w", defaultRouteNetwork.String(), err) + } + + // Add route referenced by the above rules + // ip route a table 60 10.0.120.0/22 via 10.0.110.254 dev br-comm-ch + if err := p.addRouteIfNotExists(p.vtepCIDR, defaultGateway, oobInterface, nil, ptr.To(sourceRoutingTable)); err != nil { + return fmt.Errorf("error while adding rule: %w", err) + } + + return nil +} diff --git a/dpf-utils/internal/cniprovisioner/dpu/provisioner_test.go b/dpf-utils/internal/cniprovisioner/dpu/provisioner_test.go new file mode 100644 index 0000000..14daa98 --- /dev/null +++ b/dpf-utils/internal/cniprovisioner/dpu/provisioner_test.go @@ -0,0 +1,837 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dpucniprovisioner_test + +import ( + "context" + "encoding/json" + "net" + "os" + "path/filepath" + "time" + + networkhelperMock "github.com/nvidia/doca-platform/pkg/utils/networkhelper/mock" + dpucniprovisioner "github.com/nvidia/ovn-kubernetes-components/internal/cniprovisioner/dpu" + ovsclientMock "github.com/nvidia/ovn-kubernetes-components/internal/utils/ovsclient/mock" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + testclient "k8s.io/client-go/kubernetes/fake" + clock "k8s.io/utils/clock/testing" + kexec "k8s.io/utils/exec" + kexecTesting "k8s.io/utils/exec/testing" + "k8s.io/utils/ptr" +) + +var _ = Describe("DPU CNI Provisioner in Internal mode", func() { + Context("When it runs once for the first time", func() { + It("should configure the system fully when different subnets per DPU", func() { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + vtepIPNet, err := netlink.ParseIPNet("192.168.1.1/24") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.10") + vtepCIDR, err := netlink.ParseIPNet("192.168.1.0/23") + Expect(err).ToNot(HaveOccurred()) + hostCIDR, err := netlink.ParseIPNet("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + pfIPNet, err := netlink.ParseIPNet("192.168.1.2/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNet, err := netlink.ParseIPNet("10.0.100.100/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNetWith32Mask, err := netlink.ParseIPNet("10.0.100.100/32") + Expect(err).ToNot(HaveOccurred()) + flannelIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + _, defaultRouteNetwork, err := net.ParseCIDR("0.0.0.0/0") + Expect(err).ToNot(HaveOccurred()) + defaultGateway := net.ParseIP("10.0.100.254") + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset() + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.InternalIPAM, clock.NewFakeClock(time.Now()), ovsClient, networkhelper, fakeExec, kubernetesClient, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, fakeNode.Name, nil, 8940) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + ovnInputPath := filepath.Join(ovnInputDirPath, "ovn_k8s.conf") + + mac, _ := net.ParseMAC("00:00:00:00:00:01") + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + Expect(cmd).To(Equal("dnsmasq")) + Expect(args).To(Equal([]string{ + "--keep-in-foreground", + "--port=0", + "--log-facility=-", + "--interface=br-ovn", + "--dhcp-option=option:router", + "--dhcp-option=option:mtu,9000", + "--dhcp-range=192.168.1.0,static", + "--dhcp-host=00:00:00:00:00:01,192.168.1.2", + "--dhcp-option=option:classless-static-route,192.168.1.0/23,192.168.1.10", + })) + + return kexec.New().Command("echo") + })) + + networkhelper.EXPECT().LinkIPAddressExists("br-ovn", vtepIPNet) + networkhelper.EXPECT().SetLinkIPAddress("br-ovn", vtepIPNet) + networkhelper.EXPECT().SetLinkUp("br-ovn") + networkhelper.EXPECT().RouteExists(vtepCIDR, gateway, "br-ovn", nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, gateway, "br-ovn", nil, nil) + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil) + networkhelper.EXPECT().AddRoute(hostCIDR, gateway, "br-ovn", ptr.To[int](10000), nil) + networkhelper.EXPECT().GetPFRepMACAddress("pf0hpf").Return(mac, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + _, flannelIPNet, err := net.ParseCIDR(flannelIP.String()) + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(false, nil) + networkhelper.EXPECT().AddRule(flannelIPNet, 60, 31000).Return(nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(false, nil) + networkhelper.EXPECT().AddRule(oobIPNetWith32Mask, 60, 32000).Return(nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(false, nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, defaultGateway, "br-comm-ch", nil, ptr.To(60)).Return(nil) + + ovsClient.EXPECT().SetOVNEncapIP(net.ParseIP("192.168.1.1")) + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + + fakeNode.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Node")) + fakeNode.SetManagedFields(nil) + data, err := json.Marshal(fakeNode) + Expect(err).ToNot(HaveOccurred()) + _, err = kubernetesClient.CoreV1().Nodes().Patch(context.Background(), fakeNode.Name, types.ApplyPatchType, data, metav1.PatchOptions{ + FieldManager: "somemanager", + Force: ptr.To[bool](true), + }) + Expect(err).NotTo(HaveOccurred()) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + ovnInputGatewayOpts, err := os.ReadFile(ovnInputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(ovnInputGatewayOpts)).To(Equal("[Gateway]\nnext-hop=192.168.1.10\nrouter-subnet=192.168.1.0/24\n")) + + Expect(fakeExec.CommandCalls).To(Equal(1)) + }) + It("should configure the system fully when same subnet across DPUs", func() { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + vtepIPNet, err := netlink.ParseIPNet("192.168.1.1/24") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.10") + _, vtepCIDR, err := net.ParseCIDR("192.168.1.0/24") + Expect(err).ToNot(HaveOccurred()) + _, hostCIDR, err := net.ParseCIDR("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + pfIPNet, err := netlink.ParseIPNet("192.168.1.2/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNet, err := netlink.ParseIPNet("10.0.100.100/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNetWith32Mask, err := netlink.ParseIPNet("10.0.100.100/32") + Expect(err).ToNot(HaveOccurred()) + flannelIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + _, defaultRouteNetwork, err := net.ParseCIDR("0.0.0.0/0") + Expect(err).ToNot(HaveOccurred()) + defaultGateway := net.ParseIP("10.0.100.254") + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset() + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.InternalIPAM, clock.NewFakeClock(time.Now()), ovsClient, networkhelper, fakeExec, kubernetesClient, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, fakeNode.Name, nil, 1440) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + ovnInputPath := filepath.Join(ovnInputDirPath, "ovn_k8s.conf") + + mac, _ := net.ParseMAC("00:00:00:00:00:01") + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + Expect(cmd).To(Equal("dnsmasq")) + Expect(args).To(Equal([]string{ + "--keep-in-foreground", + "--port=0", + "--log-facility=-", + "--interface=br-ovn", + "--dhcp-option=option:router", + "--dhcp-option=option:mtu,1500", + "--dhcp-range=192.168.1.0,static", + "--dhcp-host=00:00:00:00:00:01,192.168.1.2", + })) + + return kexec.New().Command("echo") + })) + + Expect(vtepIPNet.String()).To(Equal("192.168.1.1/24")) + _, vtepNetwork, _ := net.ParseCIDR(vtepIPNet.String()) + Expect(vtepNetwork.String()).To(Equal("192.168.1.0/24")) + Expect(vtepCIDR).To(Equal(vtepNetwork)) + networkhelper.EXPECT().LinkIPAddressExists("br-ovn", vtepIPNet) + networkhelper.EXPECT().SetLinkIPAddress("br-ovn", vtepIPNet) + networkhelper.EXPECT().SetLinkUp("br-ovn") + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil) + networkhelper.EXPECT().AddRoute(hostCIDR, gateway, "br-ovn", ptr.To[int](10000), nil) + networkhelper.EXPECT().GetPFRepMACAddress("pf0hpf").Return(mac, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + _, flannelIPNet, err := net.ParseCIDR(flannelIP.String()) + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(false, nil) + networkhelper.EXPECT().AddRule(flannelIPNet, 60, 31000).Return(nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(false, nil) + networkhelper.EXPECT().AddRule(oobIPNetWith32Mask, 60, 32000).Return(nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(false, nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, defaultGateway, "br-comm-ch", nil, ptr.To(60)).Return(nil) + + ovsClient.EXPECT().SetOVNEncapIP(net.ParseIP("192.168.1.1")) + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + + fakeNode.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Node")) + fakeNode.SetManagedFields(nil) + data, err := json.Marshal(fakeNode) + Expect(err).ToNot(HaveOccurred()) + _, err = kubernetesClient.CoreV1().Nodes().Patch(context.Background(), fakeNode.Name, types.ApplyPatchType, data, metav1.PatchOptions{ + FieldManager: "somemanager", + Force: ptr.To[bool](true), + }) + Expect(err).NotTo(HaveOccurred()) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + ovnInputGatewayOpts, err := os.ReadFile(ovnInputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(ovnInputGatewayOpts)).To(Equal("[Gateway]\nnext-hop=192.168.1.10\nrouter-subnet=192.168.1.0/24\n")) + }) + }) + Context("When checking for idempotency", func() { + It("should not error out on subsequent runs when network calls and OVS calls are fully mocked", func(ctx context.Context) { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + vtepIPNet, err := netlink.ParseIPNet("192.168.1.1/24") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.10") + vtepCIDR, err := netlink.ParseIPNet("192.168.1.0/23") + Expect(err).ToNot(HaveOccurred()) + hostCIDR, err := netlink.ParseIPNet("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + pfIPNet, err := netlink.ParseIPNet("192.168.1.2/24") + Expect(err).ToNot(HaveOccurred()) + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset(fakeNode) + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.InternalIPAM, clock.NewFakeClock(time.Now()), ovsClient, networkhelper, fakeExec, kubernetesClient, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, fakeNode.Name, nil, 1500) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + return kexec.New().Command("echo") + })) + + // These are needed because of checks we have specific to num of IPs belonging to each interface, we can't + // mock them with gomock.Any() + dummyIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{dummyIP}, nil) + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{dummyIP}, nil) + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{dummyIP}, nil) + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{dummyIP}, nil) + + networkHelperMockAll(networkhelper) + ovsClientMockAll(ovsClient) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + }) + It("should not error out when network and ovs clients are mocked like in the real world", func(ctx context.Context) { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + vtepIPNet, err := netlink.ParseIPNet("192.168.1.1/24") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.10") + vtepCIDR, err := netlink.ParseIPNet("192.168.1.0/23") + Expect(err).ToNot(HaveOccurred()) + hostCIDR, err := netlink.ParseIPNet("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + pfIPNet, err := netlink.ParseIPNet("192.168.1.2/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNet, err := netlink.ParseIPNet("10.0.100.100/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNetWith32Mask, err := netlink.ParseIPNet("10.0.100.100/32") + Expect(err).ToNot(HaveOccurred()) + flannelIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + _, defaultRouteNetwork, err := net.ParseCIDR("0.0.0.0/0") + Expect(err).ToNot(HaveOccurred()) + defaultGateway := net.ParseIP("10.0.100.254") + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset(fakeNode) + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.InternalIPAM, clock.NewFakeClock(time.Now()), ovsClient, networkhelper, fakeExec, kubernetesClient, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, fakeNode.Name, nil, 1500) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + return kexec.New().Command("echo") + })) + + By("Checking the first run") + networkhelper.EXPECT().LinkIPAddressExists("br-ovn", vtepIPNet) + networkhelper.EXPECT().SetLinkIPAddress("br-ovn", vtepIPNet) + networkhelper.EXPECT().SetLinkUp("br-ovn") + networkhelper.EXPECT().RouteExists(vtepCIDR, gateway, "br-ovn", nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, gateway, "br-ovn", nil, nil) + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil) + networkhelper.EXPECT().AddRoute(hostCIDR, gateway, "br-ovn", ptr.To[int](10000), nil) + mac, _ := net.ParseMAC("00:00:00:00:00:01") + networkhelper.EXPECT().GetPFRepMACAddress("pf0hpf").Return(mac, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + _, flannelIPNet, err := net.ParseCIDR(flannelIP.String()) + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(false, nil) + networkhelper.EXPECT().AddRule(flannelIPNet, 60, 31000).Return(nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(false, nil) + networkhelper.EXPECT().AddRule(oobIPNetWith32Mask, 60, 32000).Return(nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(false, nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, defaultGateway, "br-comm-ch", nil, ptr.To(60)).Return(nil) + + ovsClient.EXPECT().SetOVNEncapIP(net.ParseIP("192.168.1.1")) + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + By("Checking the second run") + networkhelper.EXPECT().LinkIPAddressExists("br-ovn", vtepIPNet).Return(true, nil) + networkhelper.EXPECT().SetLinkUp("br-ovn") + networkhelper.EXPECT().RouteExists(vtepCIDR, gateway, "br-ovn", nil).Return(true, nil) + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil).Return(true, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(true, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(true, nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(true, nil) + + ovsClient.EXPECT().SetOVNEncapIP(net.ParseIP("192.168.1.1")) + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + }) + It("should not start another dnsmasq if dnsmasq already running", func(ctx context.Context) { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + vtepIPNet, err := netlink.ParseIPNet("192.168.1.1/24") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.10") + vtepCIDR, err := netlink.ParseIPNet("192.168.1.0/23") + Expect(err).ToNot(HaveOccurred()) + hostCIDR, err := netlink.ParseIPNet("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + pfIPNet, err := netlink.ParseIPNet("192.168.1.2/24") + Expect(err).ToNot(HaveOccurred()) + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset(fakeNode) + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.InternalIPAM, clock.NewFakeClock(time.Now()), ovsClient, networkhelper, fakeExec, kubernetesClient, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, fakeNode.Name, nil, 1500) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + return kexec.New().Command("echo") + })) + + // These are needed because of checks we have specific to num of IPs belonging to each interface, we can't + // mock them with gomock.Any() + dummyIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{dummyIP}, nil) + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{dummyIP}, nil) + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{dummyIP}, nil) + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{dummyIP}, nil) + + networkHelperMockAll(networkhelper) + ovsClientMockAll(ovsClient) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + Expect(fakeExec.CommandCalls).To(Equal(1)) + }) + + }) +}) + +var _ = Describe("DPU CNI Provisioner in External mode", func() { + Context("When it runs once for the first time", func() { + It("should configure the system fully when same subnet across DPUs", func() { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + _, hostCIDR, err := net.ParseCIDR("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + _, gatewayDiscoveryNetwork, err := net.ParseCIDR("169.254.99.100/32") + Expect(err).ToNot(HaveOccurred()) + vtepCIDR, err := netlink.ParseIPNet("192.168.1.0/23") + Expect(err).ToNot(HaveOccurred()) + oobIPNet, err := netlink.ParseIPNet("10.0.100.100/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNetWith32Mask, err := netlink.ParseIPNet("10.0.100.100/32") + Expect(err).ToNot(HaveOccurred()) + flannelIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + _, defaultRouteNetwork, err := net.ParseCIDR("0.0.0.0/0") + Expect(err).ToNot(HaveOccurred()) + defaultGateway := net.ParseIP("10.0.100.254") + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset(fakeNode) + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.ExternalIPAM, clock.NewFakeClock(time.Now()), ovsClient, networkhelper, fakeExec, kubernetesClient, nil, nil, vtepCIDR, hostCIDR, nil, fakeNode.Name, gatewayDiscoveryNetwork, 0) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + netplanDirPath := filepath.Join(tmpDir, "/etc/netplan") + Expect(os.MkdirAll(netplanDirPath, 0755)).To(Succeed()) + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + ovnInputPath := filepath.Join(ovnInputDirPath, "ovn_k8s.conf") + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + Expect(cmd).To(Equal("netplan")) + Expect(args).To(Equal([]string{"apply"})) + return kexec.New().Command("echo") + })) + + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + brOVNAddress, err := netlink.ParseIPNet("192.168.0.3/23") + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{brOVNAddress}, nil) + _, fakeNetwork, err := net.ParseCIDR("169.254.99.100/32") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.254") + networkhelper.EXPECT().GetGateway(fakeNetwork).Return(gateway, nil) + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil) + networkhelper.EXPECT().AddRoute(hostCIDR, gateway, "br-ovn", ptr.To[int](10000), nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + _, flannelIPNet, err := net.ParseCIDR(flannelIP.String()) + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(false, nil) + networkhelper.EXPECT().AddRule(flannelIPNet, 60, 31000).Return(nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(false, nil) + networkhelper.EXPECT().AddRule(oobIPNetWith32Mask, 60, 32000).Return(nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(false, nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, defaultGateway, "br-comm-ch", nil, ptr.To(60)).Return(nil) + + ovsClient.EXPECT().SetOVNEncapIP(brOVNAddress.IP) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + ovnInputGatewayOpts, err := os.ReadFile(ovnInputPath) + Expect(err).ToNot(HaveOccurred()) + Expect(string(ovnInputGatewayOpts)).To(Equal("[Gateway]\nnext-hop=192.168.1.254\nrouter-subnet=192.168.0.0/23\n")) + + netplanFileContent, err := os.ReadFile(filepath.Join(netplanDirPath, "80-br-ovn.yaml")) + Expect(err).ToNot(HaveOccurred()) + Expect(string(netplanFileContent)).To(Equal(` +network: + renderer: networkd + version: 2 + bridges: + br-ovn: + dhcp4: yes + dhcp4-overrides: + use-dns: no + openvswitch: {} +`)) + + }) + }) + Context("When checking for idempotency", func() { + It("should not error out when network and ovs clients are mocked like in the real world", func(ctx context.Context) { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + _, hostCIDR, err := net.ParseCIDR("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + _, gatewayDiscoveryNetwork, err := net.ParseCIDR("169.254.99.100/32") + Expect(err).ToNot(HaveOccurred()) + vtepCIDR, err := netlink.ParseIPNet("192.168.1.0/23") + Expect(err).ToNot(HaveOccurred()) + oobIPNet, err := netlink.ParseIPNet("10.0.100.100/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNetWith32Mask, err := netlink.ParseIPNet("10.0.100.100/32") + Expect(err).ToNot(HaveOccurred()) + flannelIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + _, defaultRouteNetwork, err := net.ParseCIDR("0.0.0.0/0") + Expect(err).ToNot(HaveOccurred()) + defaultGateway := net.ParseIP("10.0.100.254") + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset(fakeNode) + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.ExternalIPAM, clock.NewFakeClock(time.Now()), ovsClient, networkhelper, fakeExec, kubernetesClient, nil, nil, vtepCIDR, hostCIDR, nil, fakeNode.Name, gatewayDiscoveryNetwork, 0) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + netplanDirPath := filepath.Join(tmpDir, "/etc/netplan") + Expect(os.MkdirAll(netplanDirPath, 0755)).To(Succeed()) + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + Expect(cmd).To(Equal("netplan")) + Expect(args).To(Equal([]string{"apply"})) + return kexec.New().Command("echo") + })) + + brOVNAddress, err := netlink.ParseIPNet("192.168.0.3/23") + Expect(err).ToNot(HaveOccurred()) + _, fakeNetwork, err := net.ParseCIDR("169.254.99.100/32") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.254") + By("Checking the first run") + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{}, nil) + + err = provisioner.RunOnce() + Expect(err).To(HaveOccurred()) + + By("Checking the second run") + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{brOVNAddress}, nil) + networkhelper.EXPECT().GetGateway(fakeNetwork).Return(gateway, nil) + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil).Return(true, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + _, flannelIPNet, err := net.ParseCIDR(flannelIP.String()) + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(false, nil) + networkhelper.EXPECT().AddRule(flannelIPNet, 60, 31000).Return(nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(false, nil) + networkhelper.EXPECT().AddRule(oobIPNetWith32Mask, 60, 32000).Return(nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(false, nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, defaultGateway, "br-comm-ch", nil, ptr.To(60)).Return(nil) + + ovsClient.EXPECT().SetOVNEncapIP(brOVNAddress.IP) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + By("Checking the third run") + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{brOVNAddress}, nil) + networkhelper.EXPECT().GetGateway(fakeNetwork).Return(gateway, nil) + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil).Return(true, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(true, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(true, nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(true, nil) + + ovsClient.EXPECT().SetOVNEncapIP(brOVNAddress.IP) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + By("Checking that netplan was restarted only once") + Expect(fakeExec.CommandCalls).To(Equal(1)) + }) + It("should not run netplan apply when in cooldown period and when network and ovs clients are mocked like in the real world", func(ctx context.Context) { + testCtrl := gomock.NewController(GinkgoT()) + ovsClient := ovsclientMock.NewMockOVSClient(testCtrl) + networkhelper := networkhelperMock.NewMockNetworkHelper(testCtrl) + fakeExec := &kexecTesting.FakeExec{} + _, hostCIDR, err := net.ParseCIDR("10.0.100.1/24") + Expect(err).ToNot(HaveOccurred()) + _, gatewayDiscoveryNetwork, err := net.ParseCIDR("169.254.99.100/32") + Expect(err).ToNot(HaveOccurred()) + vtepCIDR, err := netlink.ParseIPNet("192.168.1.0/23") + Expect(err).ToNot(HaveOccurred()) + oobIPNet, err := netlink.ParseIPNet("10.0.100.100/24") + Expect(err).ToNot(HaveOccurred()) + oobIPNetWith32Mask, err := netlink.ParseIPNet("10.0.100.100/32") + Expect(err).ToNot(HaveOccurred()) + flannelIP, err := netlink.ParseIPNet("10.244.6.30/24") + Expect(err).ToNot(HaveOccurred()) + _, defaultRouteNetwork, err := net.ParseCIDR("0.0.0.0/0") + Expect(err).ToNot(HaveOccurred()) + defaultGateway := net.ParseIP("10.0.100.254") + fakeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dpu1", + Labels: map[string]string{ + "provisioning.dpu.nvidia.com/host": "host1", + }, + }, + } + kubernetesClient := testclient.NewClientset(fakeNode) + fakeClock := clock.NewFakeClock(time.Now()) + provisioner := dpucniprovisioner.New(context.Background(), dpucniprovisioner.ExternalIPAM, fakeClock, ovsClient, networkhelper, fakeExec, kubernetesClient, nil, nil, vtepCIDR, hostCIDR, nil, fakeNode.Name, gatewayDiscoveryNetwork, 0) + + // Prepare Filesystem + tmpDir, err := os.MkdirTemp("", "dpucniprovisioner") + defer func() { + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + provisioner.FileSystemRoot = tmpDir + netplanDirPath := filepath.Join(tmpDir, "/etc/netplan") + Expect(os.MkdirAll(netplanDirPath, 0755)).To(Succeed()) + ovnInputDirPath := filepath.Join(tmpDir, "/etc/openvswitch") + Expect(os.MkdirAll(ovnInputDirPath, 0755)).To(Succeed()) + + fakeCommand := kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + Expect(cmd).To(Equal("netplan")) + Expect(args).To(Equal([]string{"apply"})) + return kexec.New().Command("echo") + }) + fakeExec.CommandScript = append(fakeExec.CommandScript, fakeCommand, fakeCommand) + + brOVNAddress, err := netlink.ParseIPNet("192.168.0.3/23") + Expect(err).ToNot(HaveOccurred()) + _, fakeNetwork, err := net.ParseCIDR("169.254.99.100/32") + Expect(err).ToNot(HaveOccurred()) + gateway := net.ParseIP("192.168.1.254") + + By("Checking the first run") + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{}, nil) + + err = provisioner.RunOnce() + Expect(err).To(HaveOccurred()) + + fakeClock.Step(60 * time.Second) + + By("Checking the second run") + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{}, nil) + + err = provisioner.RunOnce() + Expect(err).To(HaveOccurred()) + + fakeClock.Step(60 * time.Second) + + By("Checking the third run") + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{}, nil) + + err = provisioner.RunOnce() + Expect(err).To(HaveOccurred()) + + fakeClock.Step(60 * time.Second) + + By("Checking the fourth run") + ovsClient.EXPECT().SetKubernetesHostNodeName("host1") + networkhelper.EXPECT().GetLinkIPAddresses("br-ovn").Return([]*net.IPNet{brOVNAddress}, nil) + networkhelper.EXPECT().GetGateway(fakeNetwork).Return(gateway, nil) + networkhelper.EXPECT().RouteExists(hostCIDR, gateway, "br-ovn", nil).Return(true, nil) + + networkhelper.EXPECT().GetLinkIPAddresses("cni0").Return([]*net.IPNet{flannelIP}, nil) + _, flannelIPNet, err := net.ParseCIDR(flannelIP.String()) + Expect(err).ToNot(HaveOccurred()) + networkhelper.EXPECT().RuleExists(flannelIPNet, 60, 31000).Return(false, nil) + networkhelper.EXPECT().AddRule(flannelIPNet, 60, 31000).Return(nil) + + networkhelper.EXPECT().GetLinkIPAddresses("br-comm-ch").Return([]*net.IPNet{oobIPNet}, nil) + networkhelper.EXPECT().RuleExists(oobIPNetWith32Mask, 60, 32000).Return(false, nil) + networkhelper.EXPECT().AddRule(oobIPNetWith32Mask, 60, 32000).Return(nil) + + networkhelper.EXPECT().GetGateway(defaultRouteNetwork).Return(defaultGateway, nil) + networkhelper.EXPECT().RouteExists(vtepCIDR, defaultGateway, "br-comm-ch", ptr.To(60)).Return(false, nil) + networkhelper.EXPECT().AddRoute(vtepCIDR, defaultGateway, "br-comm-ch", nil, ptr.To(60)).Return(nil) + + ovsClient.EXPECT().SetOVNEncapIP(brOVNAddress.IP) + + err = provisioner.RunOnce() + Expect(err).ToNot(HaveOccurred()) + + By("Checking that netplan was restarted only once") + Expect(fakeExec.CommandCalls).To(Equal(2)) + }) + }) +}) + +// networkHelperMockAll mocks all networkhelper functions. Useful for tests where we don't test the network calls +func networkHelperMockAll(networkHelper *networkhelperMock.MockNetworkHelper) { + networkHelper.EXPECT().AddRoute(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + networkHelper.EXPECT().AddRule(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + networkHelper.EXPECT().GetGateway(gomock.Any()).AnyTimes() + networkHelper.EXPECT().GetLinkIPAddresses(gomock.Any()).AnyTimes() + networkHelper.EXPECT().GetPFRepMACAddress(gomock.Any()).AnyTimes() + networkHelper.EXPECT().LinkIPAddressExists(gomock.Any(), gomock.Any()).AnyTimes() + networkHelper.EXPECT().RouteExists(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + networkHelper.EXPECT().RuleExists(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + networkHelper.EXPECT().SetLinkIPAddress(gomock.Any(), gomock.Any()).AnyTimes() + networkHelper.EXPECT().SetLinkUp(gomock.Any()).AnyTimes() +} + +// ovsClientMockAll mocks all ovsclient functions. Useful for tests where we don't test the ovsclient calls +func ovsClientMockAll(ovsClient *ovsclientMock.MockOVSClient) { + ovsClient.EXPECT().SetKubernetesHostNodeName(gomock.Any()).AnyTimes() + ovsClient.EXPECT().SetOVNEncapIP(gomock.Any()).AnyTimes() +} diff --git a/dpf-utils/internal/constants/constants.go b/dpf-utils/internal/constants/constants.go new file mode 100644 index 0000000..20b886c --- /dev/null +++ b/dpf-utils/internal/constants/constants.go @@ -0,0 +1,23 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +const ( + // HostNameDPULabelKey is the label added to the DPU Kubernetes Node that indicates the hostname of the host that + // this DPU belongs to. + HostNameDPULabelKey = "provisioning.dpu.nvidia.com/host" +) diff --git a/dpf-utils/internal/ovnkubernetesresourceinjector/webhooks/webhook.go b/dpf-utils/internal/ovnkubernetesresourceinjector/webhooks/webhook.go new file mode 100644 index 0000000..a7a9187 --- /dev/null +++ b/dpf-utils/internal/ovnkubernetesresourceinjector/webhooks/webhook.go @@ -0,0 +1,243 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhooks + +import ( + "context" + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// NetworkInjector is a component that can inject Multus annotations and resources on Pods +type NetworkInjector struct { + // Client is the client to the Kubernetes API server + Client client.Reader + // Settings are the settings for this component + Settings NetworkInjectorSettings +} + +// NetworkInjectorSettings are the settings for the Network Injector +type NetworkInjectorSettings struct { + // NADName is the name of the network attachment definition that the injector should use to configure VFs for the + // default network + NADName string + // NADNamespace is the namespace of the network attachment definition that the injector should use to configure VFs + // for the default network + NADNamespace string +} + +const ( + // netAttachDefResourceNameAnnotation is the key of the network attachment definition annotation that indicates the + // resource name. + netAttachDefResourceNameAnnotation = "k8s.v1.cni.cncf.io/resourceName" + // annotationKeyToBeInjected is the multus annotation we inject to the pods so that multus can inject the VFs + annotationKeyToBeInjected = "v1.multus-cni.io/default-network" +) + +var ( + controlPlaneNodeLabels = map[string]bool{ + "node-role.kubernetes.io/master": true, + "node-role.kubernetes.io/control-plane": true, + } +) + +var _ webhook.CustomDefaulter = &NetworkInjector{} + +// +kubebuilder:webhook:path=/mutate--v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,groups="",resources=pods,verbs=create,versions=v1,name=network-injector.dpu.nvidia.com,admissionReviewVersions=v1 +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch; +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch + +func (webhook *NetworkInjector) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&corev1.Pod{}). + WithDefaulter(webhook). + Complete() +} + +// Default implements webhook.Defaulter so a webhook will be registered for the type. +func (webhook *NetworkInjector) Default(ctx context.Context, obj runtime.Object) error { + log := ctrl.LoggerFrom(ctx) + + pod, ok := obj.(*corev1.Pod) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Pod but got a %T", obj)) + } + + ctrl.LoggerInto(ctx, log.WithValues("Pod", types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name})) + // If the pod is on the host network no-op. + if pod.Spec.HostNetwork { + return nil + } + + // If the pod explicitly selects a control plane node no-op. + controlPlanePod, err := webhook.isScheduledToControlPlane(ctx, pod) + if err != nil { + return err + } + if controlPlanePod { + return nil + } + + vfResourceName, err := getVFResourceName(ctx, webhook.Client, webhook.Settings.NADName, webhook.Settings.NADNamespace) + if err != nil { + return fmt.Errorf("error while getting VF resource name: %w", err) + } + + return injectNetworkResources(ctx, pod, webhook.Settings.NADName, webhook.Settings.NADNamespace, vfResourceName) +} + +// getVFResourceName gets the resource name that relates to the VFs that should be injected. +func getVFResourceName(ctx context.Context, c client.Reader, netAttachDefName string, netAttachDefNamespace string) (corev1.ResourceName, error) { + netAttachDef := &unstructured.Unstructured{} + netAttachDef.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Kind: "NetworkAttachmentDefinition", + }) + key := client.ObjectKey{Namespace: netAttachDefNamespace, Name: netAttachDefName} + if err := c.Get(ctx, key, netAttachDef); err != nil { + return "", fmt.Errorf("error while getting %s %s: %w", netAttachDef.GetObjectKind().GroupVersionKind().String(), key.String(), err) + } + + if v, ok := netAttachDef.GetAnnotations()[netAttachDefResourceNameAnnotation]; ok { + return corev1.ResourceName(v), nil + } + + return "", fmt.Errorf("resource can't be found in network attachment definition because annotation %s doesn't exist", netAttachDefResourceNameAnnotation) +} + +// If the pod has nodeAffinity set to a specific name check if the node it's scheduled to is a control plane node. +// This is the case for pods created by DaemonSets which set node affinity matching to a node name on creation. +func (webhook *NetworkInjector) isScheduledToControlPlane(ctx context.Context, pod *corev1.Pod) (bool, error) { + // If the pod has a control plane node selector no-op. + for label := range controlPlaneNodeLabels { + if _, ok := pod.Spec.NodeSelector[label]; ok { + return true, nil + } + } + if terms := getNodeSelectorTerms(pod); terms != nil { + for _, term := range terms { + // If the pod selects for a control plane node label return true. + if term.MatchExpressions != nil { + for _, expression := range term.MatchExpressions { + if _, ok := controlPlaneNodeLabels[expression.Key]; ok { + if expression.Operator == corev1.NodeSelectorOpExists { + return true, nil + } + + if expression.Operator == corev1.NodeSelectorOpIn && + slices.Contains(expression.Values, "") { + return true, nil + } + } + } + } + + // If the pod selects a specific control plane nodename. This is the case for DaemonSet pods. + isControlPlanePod, err := webhook.hasControlPlaneNodeName(ctx, term) + if err != nil { + return false, err + } + if isControlPlanePod { + return true, nil + } + } + } + return false, nil +} + +func (webhook *NetworkInjector) hasControlPlaneNodeName(ctx context.Context, term corev1.NodeSelectorTerm) (bool, error) { + var nodeName string + if term.MatchFields != nil { + for _, field := range term.MatchFields { + if field.Key == "metadata.name" { + nodeName = field.Values[0] + } + } + } + if nodeName == "" { + return false, nil + } + node := &corev1.Node{} + if err := webhook.Client.Get(ctx, client.ObjectKey{Namespace: "", Name: nodeName}, node); err != nil { + return false, fmt.Errorf("failed to get node pod is scheduled to %q: %w", nodeName, err) + } + if node.Labels == nil { + return false, nil + } + + for label := range controlPlaneNodeLabels { + if _, ok := node.Labels[label]; ok { + return true, nil + } + } + return false, nil +} + +func getNodeSelectorTerms(pod *corev1.Pod) []corev1.NodeSelectorTerm { + if pod.Spec.Affinity != nil && + pod.Spec.Affinity.NodeAffinity != nil && + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil && + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms != nil { + return pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + } + return nil +} + +func injectNetworkResources(ctx context.Context, pod *corev1.Pod, netAttachDefName string, netAttachDefNamespace string, vfResourceName corev1.ResourceName) error { + log := ctrl.LoggerFrom(ctx) + // Inject device requests. One additional VF. + if pod.Spec.Containers[0].Resources.Requests == nil { + pod.Spec.Containers[0].Resources.Requests = corev1.ResourceList{} + } + if pod.Spec.Containers[0].Resources.Limits == nil { + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{} + + } + if _, ok := pod.Spec.Containers[0].Resources.Requests[vfResourceName]; ok { + res := pod.Spec.Containers[0].Resources.Requests[vfResourceName] + res.Add(resource.MustParse("1")) + pod.Spec.Containers[0].Resources.Requests[vfResourceName] = res + } else { + pod.Spec.Containers[0].Resources.Requests[vfResourceName] = resource.MustParse("1") + } + + if _, ok := pod.Spec.Containers[0].Resources.Limits[vfResourceName]; ok { + res := pod.Spec.Containers[0].Resources.Limits[vfResourceName] + res.Add(resource.MustParse("1")) + pod.Spec.Containers[0].Resources.Limits[vfResourceName] = res + } else { + pod.Spec.Containers[0].Resources.Limits[vfResourceName] = resource.MustParse("1") + } + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[annotationKeyToBeInjected] = fmt.Sprintf("%s/%s", netAttachDefNamespace, netAttachDefName) + log.Info(fmt.Sprintf("injected resource %v into pod", vfResourceName)) + return nil +} diff --git a/dpf-utils/internal/ovnkubernetesresourceinjector/webhooks/webhook_test.go b/dpf-utils/internal/ovnkubernetesresourceinjector/webhooks/webhook_test.go new file mode 100644 index 0000000..3026c01 --- /dev/null +++ b/dpf-utils/internal/ovnkubernetesresourceinjector/webhooks/webhook_test.go @@ -0,0 +1,328 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhooks + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestNetworkInjector_Default(t *testing.T) { + g := NewWithT(t) + controlPlaneNodeName := "control-plane-node" + workerNodeName := "worker-node" + resourceName := corev1.ResourceName("test-resource") + + objects := []client.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneNodeName, + Labels: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: workerNodeName, + Labels: map[string]string{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": "dpf-ovn-kubernetes", + "namespace": "ovn-kubernetes", + "annotations": map[string]interface{}{ + "k8s.v1.cni.cncf.io/resourceName": resourceName.String(), + }, + }, + }, + }, + } + + controlPlaneMatchExpressionsExists := []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.NodeSelectorOpExists, + }, + { + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + } + + controlPlaneMatchExpressionsIn := []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.NodeSelectorOpIn, + Values: []string{""}, + }, + { + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.NodeSelectorOpIn, + Values: []string{""}, + }, + }, + }, + } + + basePod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{}, + Limits: corev1.ResourceList{}, + }, + }, + }, + }, + } + hostNetworkPod := basePod.DeepCopy() + hostNetworkPod.Spec.HostNetwork = true + + podWithControlPlaneNodeSelector := basePod.DeepCopy() + podWithControlPlaneNodeSelector.Spec.NodeSelector = map[string]string{"node-role.kubernetes.io/master": ""} + + podWithControlPlaneNodeSelectorMatchExpressionsExists := basePod.DeepCopy() + setSelectorTerms(podWithControlPlaneNodeSelectorMatchExpressionsExists, controlPlaneMatchExpressionsExists) + + podWithControlPlaneNodeSelectorMatchExpressionsIn := basePod.DeepCopy() + setSelectorTerms(podWithControlPlaneNodeSelectorMatchExpressionsIn, controlPlaneMatchExpressionsIn) + + podWithControlPlaneNodeNameSelectorTerms := basePod.DeepCopy() + setSelectorTermsToNodeName(podWithControlPlaneNodeNameSelectorTerms, controlPlaneNodeName) + + podWithWorkerNodeSelectorTerms := basePod.DeepCopy() + setSelectorTermsToNodeName(podWithWorkerNodeSelectorTerms, workerNodeName) + + podWithExistingVFResources := basePod.DeepCopy() + + podWithExistingVFResources.Spec.Containers[0].Resources.Requests = corev1.ResourceList{ + resourceName: resource.MustParse("1"), + } + podWithExistingVFResources.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + resourceName: resource.MustParse("1"), + } + + tests := []struct { + name string + pod *corev1.Pod + expectedResourceCount string + expectAnnotation bool + }{ + { + name: "don't inject resource into pod that has hostNetwork == true", + pod: hostNetworkPod, + expectedResourceCount: "0", + }, + { + name: "don't inject resource into pod that has a node selector for a control plane machine", + pod: podWithControlPlaneNodeSelector, + expectedResourceCount: "0", + }, + { + name: "don't inject resource into pod that explicitly targets a control plane node with NodeSelectorTerms", + pod: podWithControlPlaneNodeNameSelectorTerms, + expectedResourceCount: "0", + }, + { + name: "don't inject resource into pod that explicitly targets a control plane node with NodeSelectorTerms with Exists operator", + pod: podWithControlPlaneNodeSelectorMatchExpressionsExists, + expectedResourceCount: "0", + }, + { + name: "don't inject resource into pod that explicitly targets a control plane node with NodeSelectorTerms with In operator", + pod: podWithControlPlaneNodeSelectorMatchExpressionsIn, + expectedResourceCount: "0", + }, + { + name: "inject resource into pod that explicitly targets a worker node with NodeSelectorTerms", + pod: podWithWorkerNodeSelectorTerms, + expectedResourceCount: "1", + expectAnnotation: true, + }, + { + name: "inject resource into pod that has no clear scheduling target", + pod: basePod, + expectedResourceCount: "1", + expectAnnotation: true, + }, + { + name: "inject resources into pod with existing resource claims", + pod: podWithExistingVFResources, + expectedResourceCount: "2", + expectAnnotation: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := scheme.Scheme + fakeclient := fake.NewClientBuilder().WithObjects(objects...).WithScheme(s).Build() + webhook := &NetworkInjector{ + Client: fakeclient, + Settings: NetworkInjectorSettings{ + NADName: "dpf-ovn-kubernetes", + NADNamespace: "ovn-kubernetes", + }, + } + err := webhook.Default(context.Background(), tt.pod) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(tt.pod.Spec.Containers[0].Resources.Limits[resourceName].Equal(resource.MustParse(tt.expectedResourceCount))).To(BeTrue()) + g.Expect(tt.pod.Spec.Containers[0].Resources.Requests[resourceName].Equal(resource.MustParse(tt.expectedResourceCount))).To(BeTrue()) + //nolint:ginkgolinter + g.Expect(tt.pod.Annotations[annotationKeyToBeInjected] == "ovn-kubernetes/dpf-ovn-kubernetes").To(Equal(tt.expectAnnotation)) + }) + } +} + +func TestNetworkInjector_PreReqObjects(t *testing.T) { + g := NewWithT(t) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{}, + Limits: corev1.ResourceList{}, + }, + }, + }, + }, + } + + networkAttachDefWithoutAnnotation := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": "dpf-ovn-kubernetes", + "namespace": "ovn-kubernetes", + }, + }, + } + + networkAttachDefWithAnnotation := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": "dpf-ovn-kubernetes", + "namespace": "ovn-kubernetes", + "annotations": map[string]interface{}{ + "k8s.v1.cni.cncf.io/resourceName": "some-resource", + }, + }, + }, + } + + tests := []struct { + name string + existingObjects []client.Object + expectError bool + }{ + { + name: "no NetworkAttachmentDefinition", + existingObjects: nil, + expectError: true, + }, + { + name: "no annotation on NetworkAttachmentDefinition", + existingObjects: []client.Object{networkAttachDefWithoutAnnotation}, + expectError: true, + }, + { + name: "all prereq objects exist", + existingObjects: []client.Object{networkAttachDefWithAnnotation}, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := scheme.Scheme + fakeclient := fake.NewClientBuilder().WithObjects(tt.existingObjects...).WithScheme(s).Build() + webhook := &NetworkInjector{ + Client: fakeclient, + Settings: NetworkInjectorSettings{ + NADName: "dpf-ovn-kubernetes", + NADNamespace: "ovn-kubernetes", + }, + } + err := webhook.Default(context.Background(), pod) + if tt.expectError { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func setSelectorTermsToNodeName(pod *corev1.Pod, nodeName string) { + setSelectorTerms(pod, []corev1.NodeSelectorTerm{ + { + MatchFields: []corev1.NodeSelectorRequirement{ + { + Key: "metadata.name", + Values: []string{nodeName}, + }, + }, + }, + }) +} + +func setSelectorTerms(pod *corev1.Pod, terms []corev1.NodeSelectorTerm) { + if pod.Spec.Affinity == nil { + pod.Spec.Affinity = &corev1.Affinity{} + } + if pod.Spec.Affinity.NodeAffinity == nil { + pod.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} + } + if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} + } + pod.Spec.Affinity.NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = terms +} diff --git a/dpf-utils/internal/readyz/readyz.go b/dpf-utils/internal/readyz/readyz.go new file mode 100644 index 0000000..df146bd --- /dev/null +++ b/dpf-utils/internal/readyz/readyz.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package readyz + +import ( + "fmt" + "os" +) + +const ( + // readyzFilePath is the file that is written by this package when the application is ready. + readyzFilePath = "/var/run/readyz" +) + +// ReportReady declares that the application is ready. It writes a file on the host which can be used in a readiness +// probe. +func ReportReady() error { + err := os.WriteFile(readyzFilePath, []byte("ready"), 0644) + if err != nil { + return fmt.Errorf("error while writing ready file: %w", err) + } + + return nil +} diff --git a/dpf-utils/internal/utils/ovsclient/implementation.go b/dpf-utils/internal/utils/ovsclient/implementation.go new file mode 100644 index 0000000..1ff7a0d --- /dev/null +++ b/dpf-utils/internal/utils/ovsclient/implementation.go @@ -0,0 +1,373 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ovsclient + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + + kexec "k8s.io/utils/exec" +) + +const ovsVsctl = "ovs-vsctl" +const ovsAppctl = "ovs-appctl" + +type ovsClient struct { + exec kexec.Interface + ovsVsctlPath string + ovsAppCtlPath string + fileSystemRoot string +} + +// New creates an OVSClient and returns an error if the OVS util binaries can't be found. +func newOvsClient(exec kexec.Interface) (OVSClient, error) { + var err error + c := &ovsClient{} + c.exec = exec + c.ovsVsctlPath, err = exec.LookPath(ovsVsctl) + if err != nil { + return nil, err + } + c.ovsAppCtlPath, err = exec.LookPath(ovsAppctl) + if err != nil { + return nil, err + } + return c, err +} + +func (c *ovsClient) runOVSVsctl(args ...string) (string, error) { + cmd := c.exec.Command(c.ovsVsctlPath, args...) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.SetStdout(&stdout) + cmd.SetStderr(&stderr) + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("error running ovs-vsctl command with args %v failed: err=%w stderr=%s", args, err, stderr.String()) + } + return stdout.String(), nil +} + +func (c *ovsClient) runOVSAppctl(args ...string) (string, error) { + socketPath, err := getVSwitchDSocketPath(c.fileSystemRoot) + if err != nil { + return "", fmt.Errorf("failed to construct ovs-vswitchd socket path: %w", err) + } + finalArgs := make([]string, 0, len(args)+2) + finalArgs = append(finalArgs, "-t", socketPath) + finalArgs = append(finalArgs, args...) + cmd := c.exec.Command(c.ovsAppCtlPath, finalArgs...) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.SetStdout(&stdout) + cmd.SetStderr(&stderr) + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("error running ovs-appctl command with args %v failed: err=%w stderr=%s", args, err, stderr.String()) + } + return stdout.String(), nil +} + +// getVSwitchDSocketPath returns the active socket of the ovs-vswitchd process +func getVSwitchDSocketPath(fileSystemRoot string) (string, error) { + pid, err := os.ReadFile(filepath.Join(fileSystemRoot, "/var/run/openvswitch/ovs-vswitchd.pid")) + if err != nil { + return "", fmt.Errorf("failed to get ovs-vswitch pid: %w", err) + } + + pidTrimmed := strings.TrimSpace(string(pid)) + + return filepath.Join(fileSystemRoot, fmt.Sprintf("/var/run/openvswitch/ovs-vswitchd.%s.ctl", pidTrimmed)), nil +} + +// BridgeExists checks if a bridge exists +func (c *ovsClient) BridgeExists(name string) (bool, error) { + _, err := c.runOVSVsctl("br-exists", name) + if err != nil { + var exitErr *kexec.CodeExitError + if errors.As(err, &exitErr) { + // https://github.com/openvswitch/ovs/blob/166ee41d282c506d100bc2185d60af277121b55b/utilities/ovs-vsctl.8.in#L203-L206 + if exitErr.Code == 2 { + return false, nil + } + } + return false, err + } + + return true, nil +} + +// AddBridgeIfNotExists adds a bridge if it doesn't exist +func (c *ovsClient) AddBridgeIfNotExists(name string) error { + _, err := c.runOVSVsctl("--may-exist", "add-br", name) + return err +} + +// DeleteBridgeIfExists deletes a bridge if it exists +func (c *ovsClient) DeleteBridgeIfExists(name string) error { + _, err := c.runOVSVsctl("--if-exists", "del-br", name) + return err +} + +// SetBridgeDataPathType sets the datapath type of a bridge +func (c *ovsClient) SetBridgeDataPathType(bridge string, bridgeType BridgeDataPathType) error { + _, err := c.runOVSVsctl("set", "bridge", bridge, fmt.Sprintf("datapath_type=%s", bridgeType)) + return err +} + +// SetBridgeMAC sets the MAC address for the bridge interface +func (c *ovsClient) SetBridgeMAC(bridge string, mac net.HardwareAddr) error { + _, err := c.runOVSVsctl("set", "bridge", bridge, fmt.Sprintf("other-config:hwaddr=%s", mac.String())) + return err +} + +// SetBridgeUplink sets the bridge-uplink external ID of the bridge. It overrides if already exists. +func (c *ovsClient) SetBridgeUplinkPort(bridge string, port string) error { + _, err := c.runOVSVsctl("br-set-external-id", bridge, "bridge-uplink", port) + return err +} + +// SetBridgeHostToServicePort sets the host-to-service external ID of the bridge. It overrides if already exists. +func (c *ovsClient) SetBridgeHostToServicePort(bridge string, port string) error { + _, err := c.runOVSVsctl("br-set-external-id", bridge, "host-to-service-interface", port) + return err +} + +// SetBridgeController sets the controller for a bridge +func (c *ovsClient) SetBridgeController(bridge string, controller string) error { + _, err := c.runOVSVsctl("set-controller", bridge, controller) + return err +} + +// AddPortIfNotExists adds a port to a bridge if it doesn't exist +func (c *ovsClient) AddPortIfNotExists(bridge string, port string) error { + _, err := c.runOVSVsctl("--may-exist", "add-port", bridge, port) + return err +} + +// SetPortType adds a port to a bridge +func (c *ovsClient) SetPortType(port string, portType PortType) error { + _, err := c.runOVSVsctl("set", "interface", port, fmt.Sprintf("type=%s", portType)) + return err +} + +// SetPatchPortPeer sets the peer for a patch port +func (c *ovsClient) SetPatchPortPeer(port string, peer string) error { + _, err := c.runOVSVsctl("set", "interface", port, fmt.Sprintf("options:peer=%s", peer)) + return err +} + +// SetOVNEncapIP sets the ovn-encap-ip external ID in the Open_vSwitch table in OVS +func (c *ovsClient) SetOVNEncapIP(ip net.IP) error { + _, err := c.runOVSVsctl("set", "Open_vSwitch", ".", fmt.Sprintf("external_ids:ovn-encap-ip=%s", ip.String())) + return err +} + +// SetDOCAInit sets the doca-init other_config in the Open_vSwitch table in OVS +func (c *ovsClient) SetDOCAInit(enable bool) error { + _, err := c.runOVSVsctl("set", "Open_vSwitch", ".", fmt.Sprintf("other_config:doca-init=%t", enable)) + return err +} + +// SetKubernetesHostNodeName sets the host-k8s-nodename external ID in the Open_vSwitch table in OVS +func (c *ovsClient) SetKubernetesHostNodeName(name string) error { + _, err := c.runOVSVsctl("set", "Open_vSwitch", ".", fmt.Sprintf("external_ids:host-k8s-nodename=%s", name)) + return err +} + +// InterfaceToBridge returns the bridge an interface exists in +func (c *ovsClient) InterfaceToBridge(iface string) (string, error) { + out, err := c.runOVSVsctl("iface-to-br", iface) + if err != nil { + return "", err + } + return strings.TrimSpace(out), nil +} + +// DeletePort deletes a port +func (c *ovsClient) DeletePort(port string) error { + _, err := c.runOVSVsctl("del-port", port) + return err +} + +// GetInterfaceOfPort returns the ofport number of a port +func (c *ovsClient) GetInterfaceOfPort(port string) (int, error) { + out, err := c.runOVSVsctl("--columns=ofport", "list", "int", port) + if err != nil { + return 0, err + } + + _, ofportRaw, found := strings.Cut(out, ":") + if !found { + return 0, fmt.Errorf("error finding the ofport in command output: %s", out) + } + + return strconv.Atoi(strings.TrimSpace(ofportRaw)) +} + +// GetPortExternalIDs returns the external_ids of an OVS port +func (c *ovsClient) GetPortExternalIDs(port string) (map[string]string, error) { + out, err := c.runOVSVsctl("--columns=external_ids", "list", "port", port) + if err != nil { + return nil, err + } + + _, rawIDs, found := strings.Cut(out, ":") + if !found { + return nil, fmt.Errorf("error finding the external IDs in command output: %s", out) + } + + return getExternalIDsAsMap(rawIDs) +} + +// getExternalIDsAsMap returns a map go struct for the given ids passed as string +func getExternalIDsAsMap(rawIDs string) (map[string]string, error) { + // https://github.com/openvswitch/ovs/blob/ec2a950d7d70d541323c3a48a424df565370579e/vswitchd/vswitch.ovsschema#L550 + ids := make(map[string]string) + + rawIDs = strings.TrimSpace(rawIDs) + + if rawIDs[0] == '{' { + rawIDs = rawIDs[1:] + } + + if rawIDs[len(rawIDs)-1] == '}' { + rawIDs = rawIDs[:len(rawIDs)-1] + } + + if len(rawIDs) == 0 { + return ids, nil + } + + for _, externalID := range strings.Split(rawIDs, ", ") { + kv := strings.Split(externalID, "=") + if len(kv) > 2 { + return nil, fmt.Errorf("more than 2 elements found when splitting '%s' at '='", kv) + } + key := strings.TrimSpace(kv[0]) + value := strings.TrimSpace(kv[1]) + + ids[key] = value + } + + return ids, nil +} + +// GetInterfaceExternalIDs returns the external_ids of an OVS interface +func (c *ovsClient) GetInterfaceExternalIDs(iface string) (map[string]string, error) { + out, err := c.runOVSVsctl("--columns=external_ids", "list", "interface", iface) + if err != nil { + return nil, err + } + + _, rawIDs, found := strings.Cut(out, ":") + if !found { + return nil, fmt.Errorf("error finding the external IDs in command output: %s", out) + } + + return getExternalIDsAsMap(rawIDs) +} + +// AddPortWithMetadata adds a port to the given bridge with the specified external IDs and ofport request in a single +// transaction +func (c *ovsClient) AddPortWithMetadata(bridge string, port string, portType PortType, portExternalIDs map[string]string, interfaceExternalIDs map[string]string, ofport int) error { + args := []string{} + args = append(args, "add-port", bridge, port, "--") + args = append(args, "set", "int", port, fmt.Sprintf("type=%s", portType), "--") + args = append(args, "set", "int", port, fmt.Sprintf("ofport_request=%d", ofport), "--") + for k, v := range portExternalIDs { + args = append(args, "set", "port", port, fmt.Sprintf("external_ids:%s=%s", k, v), "--") + } + for k, v := range interfaceExternalIDs { + args = append(args, "set", "int", port, fmt.Sprintf("external_ids:%s=%s", k, v), "--") + } + + args = args[:len(args)-1] + _, err := c.runOVSVsctl(args...) + return err +} + +// ListInterfaces lists all the interfaces that exist in OVS of a particular type +func (c *ovsClient) ListInterfaces(portType PortType) (map[string]interface{}, error) { + out, err := c.runOVSVsctl("--columns=name", "find", "int", fmt.Sprintf("type=%s", portType)) + if err != nil { + return nil, err + } + + ports := make(map[string]interface{}) + outReader := strings.NewReader(out) + s := bufio.NewScanner(outReader) + for s.Scan() { + line := s.Text() + _, rawPort, found := strings.Cut(line, ":") + if !found { + continue + } + + ports[strings.TrimSpace(rawPort)] = struct{}{} + } + + if err := s.Err(); err != nil { + return nil, err + } + + return ports, nil +} + +// GetInterfacesWithPMDRXQueue returns all the interfaces that have a PMD Rx queue +func (c *ovsClient) GetInterfacesWithPMDRXQueue() (map[string]interface{}, error) { + out, err := c.runOVSAppctl("dpif-netdev/pmd-rxq-show") + if err != nil { + return nil, err + } + + ports := make(map[string]interface{}) + outReader := strings.NewReader(out) + s := bufio.NewScanner(outReader) + for s.Scan() { + line := s.Text() + key, value, found := strings.Cut(line, ":") + if !found { + continue + } + + if strings.TrimSpace(key) != "port" { + continue + } + + value = strings.TrimSpace(value) + rawPort, _, found := strings.Cut(value, " ") + if !found { + return nil, fmt.Errorf("error while extracting port from string: %s", value) + } + + ports[strings.TrimSpace(rawPort)] = struct{}{} + } + + if err := s.Err(); err != nil { + return nil, err + } + + return ports, nil +} diff --git a/dpf-utils/internal/utils/ovsclient/implementation_test.go b/dpf-utils/internal/utils/ovsclient/implementation_test.go new file mode 100644 index 0000000..d9f651d --- /dev/null +++ b/dpf-utils/internal/utils/ovsclient/implementation_test.go @@ -0,0 +1,629 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ovsclient + +import ( + "os" + "path/filepath" + "testing" + + . "github.com/onsi/gomega" + kexec "k8s.io/utils/exec" + kexecTesting "k8s.io/utils/exec/testing" +) + +func TestGetExternalIDsAsMap(t *testing.T) { + g := NewWithT(t) + cases := []struct { + msg string + input string + expectedOutput map[string]string + expectedError bool + }{ + { + msg: "empty", + input: "{}", + expectedOutput: make(map[string]string), + expectedError: false, + }, + { + msg: "one external id", + input: "{owner=ovs-cni.network.kubevirt.io}", + expectedOutput: map[string]string{ + "owner": "ovs-cni.network.kubevirt.io", + }, + expectedError: false, + }, + { + msg: "one external id with extra space at the beginning and end", + input: " {owner=ovs-cni.network.kubevirt.io} ", + expectedOutput: map[string]string{ + "owner": "ovs-cni.network.kubevirt.io", + }, + expectedError: false, + }, + { + msg: "multiple external ids", + input: `{dpf-id="dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if", iface-id=""}`, + expectedOutput: map[string]string{ + "dpf-id": `"dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if"`, + "iface-id": `""`, + }, + expectedError: false, + }, + { + msg: "malformed input due to no closing bracket", + input: `{dpf-id="dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if", iface-id=""`, + expectedOutput: map[string]string{ + "dpf-id": `"dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if"`, + "iface-id": `""`, + }, + expectedError: false, + }, + { + msg: "malformed input due to multiple equals", + input: `{dpf-id=="dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if", iface-id=""}`, + expectedOutput: make(map[string]string), + expectedError: true, + }, + } + + for _, tt := range cases { + t.Run(tt.msg, func(t *testing.T) { + output, err := getExternalIDsAsMap(tt.input) + if tt.expectedError { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(BeComparableTo(tt.expectedOutput)) + }) + } +} + +func TestGetInterfaceOfPort(t *testing.T) { + g := NewWithT(t) + cases := []struct { + msg string + input string + fakeCommandOutput string + expectedOutput int + expectedError bool + }{ + { + msg: "usual command output", + input: "pf0hpf", + fakeCommandOutput: "ofport : 4", + expectedOutput: 4, + expectedError: false, + }, + { + msg: "empty command output", + input: "pf0hpf", + fakeCommandOutput: "", + expectedError: true, + }, + { + msg: "malformed command output - missing column", + input: "pf0hpf", + fakeCommandOutput: "ofport 4", + expectedError: true, + }, + } + + for _, tt := range cases { + t.Run(tt.msg, func(t *testing.T) { + fakeExec := &kexecTesting.FakeExec{LookPathFunc: func(s string) (string, error) { return s, nil }} + c, err := newOvsClient(fakeExec) + g.Expect(err).ToNot(HaveOccurred()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + g.Expect(cmd).To(Equal("ovs-vsctl")) + g.Expect(args).To(Equal([]string{ + "--columns=ofport", + "list", + "int", + tt.input, + })) + return kexec.New().Command("echo", tt.fakeCommandOutput) + })) + + output, err := c.GetInterfaceOfPort(tt.input) + if tt.expectedError { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(BeComparableTo(tt.expectedOutput)) + }) + } +} + +func TestGetPortExternalIDs(t *testing.T) { + g := NewWithT(t) + cases := []struct { + msg string + input string + fakeCommandOutput string + expectedOutput map[string]string + expectedError bool + }{ + { + msg: "usual command output", + input: "pf0hpf", + fakeCommandOutput: `external_ids : {dpf-id="dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if", iface-id=""}`, + expectedOutput: map[string]string{ + "dpf-id": `"dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if"`, + "iface-id": `""`, + }, + expectedError: false, + }, + { + msg: "empty external ids", + input: "pf0hpf", + fakeCommandOutput: "external_ids : {}", + expectedOutput: make(map[string]string), + expectedError: false, + }, + { + msg: "empty command output", + input: "pf0hpf", + fakeCommandOutput: "", + expectedError: true, + }, + { + msg: "malformed command output - missing column", + input: "pf0hpf", + fakeCommandOutput: `external_ids {dpf-id="dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if", iface-id=""}`, + expectedError: true, + }, + } + + for _, tt := range cases { + t.Run(tt.msg, func(t *testing.T) { + fakeExec := &kexecTesting.FakeExec{LookPathFunc: func(s string) (string, error) { return s, nil }} + c, err := newOvsClient(fakeExec) + g.Expect(err).ToNot(HaveOccurred()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + g.Expect(cmd).To(Equal("ovs-vsctl")) + g.Expect(args).To(Equal([]string{ + "--columns=external_ids", + "list", + "port", + tt.input, + })) + return kexec.New().Command("echo", tt.fakeCommandOutput) + })) + + output, err := c.GetPortExternalIDs(tt.input) + if tt.expectedError { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(BeComparableTo(tt.expectedOutput)) + }) + } +} + +func TestGetInterfaceExternalIDs(t *testing.T) { + g := NewWithT(t) + cases := []struct { + msg string + input string + fakeCommandOutput string + expectedOutput map[string]string + expectedError bool + }{ + { + msg: "usual command output", + input: "pf0hpf", + fakeCommandOutput: `external_ids : {dpf-id="dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if", iface-id=""}`, + expectedOutput: map[string]string{ + "dpf-id": `"dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if"`, + "iface-id": `""`, + }, + expectedError: false, + }, + { + msg: "empty external ids", + input: "pf0hpf", + fakeCommandOutput: "external_ids : {}", + expectedOutput: make(map[string]string), + expectedError: false, + }, + { + msg: "empty command output", + input: "pf0hpf", + fakeCommandOutput: "", + expectedError: true, + }, + { + msg: "malformed command output - missing column", + input: "pf0hpf", + fakeCommandOutput: `external_ids {dpf-id="dpf-operator-system/dpu-cplane-tenant1-doca-hbn-ds-98svc/p1_if", iface-id=""}`, + expectedError: true, + }, + } + + for _, tt := range cases { + t.Run(tt.msg, func(t *testing.T) { + fakeExec := &kexecTesting.FakeExec{LookPathFunc: func(s string) (string, error) { return s, nil }} + c, err := newOvsClient(fakeExec) + g.Expect(err).ToNot(HaveOccurred()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + g.Expect(cmd).To(Equal("ovs-vsctl")) + g.Expect(args).To(Equal([]string{ + "--columns=external_ids", + "list", + "interface", + tt.input, + })) + return kexec.New().Command("echo", tt.fakeCommandOutput) + })) + + output, err := c.GetInterfaceExternalIDs(tt.input) + if tt.expectedError { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(BeComparableTo(tt.expectedOutput)) + }) + } +} + +func TestAddPortWithMetadata(t *testing.T) { + g := NewWithT(t) + cases := []struct { + msg string + inputBridge string + inputPort string + inputPortType PortType + inputPortExternalIDs map[string]string + inputInterfaceExternalIDs map[string]string + inputOfport int + expectedCommandArgs []string + expectedError bool + }{ + { + msg: "full", + inputBridge: "br-ovn", + inputPort: "pf0hpf", + inputPortType: DPDK, + inputPortExternalIDs: map[string]string{ + "port-id-key-1": "port-id-value-1", + "port-id-key-2": "port-id-value-2", + }, + inputInterfaceExternalIDs: map[string]string{ + "iface-id-key-1": "iface-id-value-1", + "iface-id-key-2": "iface-id-value-2", + }, + inputOfport: 4, + expectedCommandArgs: []string{ + "add-port", "br-ovn", "pf0hpf", + "--", "set", "int", "pf0hpf", "type=dpdk", + "--", "set", "int", "pf0hpf", "ofport_request=4", + "--", "set", "port", "pf0hpf", "external_ids:port-id-key-1=port-id-value-1", + "--", "set", "port", "pf0hpf", "external_ids:port-id-key-2=port-id-value-2", + "--", "set", "int", "pf0hpf", "external_ids:iface-id-key-1=iface-id-value-1", + "--", "set", "int", "pf0hpf", "external_ids:iface-id-key-2=iface-id-value-2", + }, + expectedError: false, + }, + { + msg: "no port external ids", + inputBridge: "br-ovn", + inputPort: "pf0hpf", + inputPortType: DPDK, + inputPortExternalIDs: make(map[string]string), + inputInterfaceExternalIDs: map[string]string{ + "iface-id-key-1": "iface-id-value-1", + "iface-id-key-2": "iface-id-value-2", + }, + inputOfport: 4, + expectedCommandArgs: []string{ + "add-port", "br-ovn", "pf0hpf", + "--", "set", "int", "pf0hpf", "type=dpdk", + "--", "set", "int", "pf0hpf", "ofport_request=4", + "--", "set", "int", "pf0hpf", "external_ids:iface-id-key-1=iface-id-value-1", + "--", "set", "int", "pf0hpf", "external_ids:iface-id-key-2=iface-id-value-2", + }, + expectedError: false, + }, + { + msg: "no interface external ids", + inputBridge: "br-ovn", + inputPort: "pf0hpf", + inputPortType: DPDK, + inputPortExternalIDs: map[string]string{ + "port-id-key-1": "port-id-value-1", + "port-id-key-2": "port-id-value-2", + }, + inputInterfaceExternalIDs: make(map[string]string), + inputOfport: 4, + expectedCommandArgs: []string{ + "add-port", "br-ovn", "pf0hpf", + "--", "set", "int", "pf0hpf", "type=dpdk", + "--", "set", "int", "pf0hpf", "ofport_request=4", + "--", "set", "port", "pf0hpf", "external_ids:port-id-key-1=port-id-value-1", + "--", "set", "port", "pf0hpf", "external_ids:port-id-key-2=port-id-value-2", + }, + expectedError: false, + }, + { + msg: "no port and interface external ids", + inputBridge: "br-ovn", + inputPort: "pf0hpf", + inputPortType: DPDK, + inputPortExternalIDs: make(map[string]string), + inputInterfaceExternalIDs: make(map[string]string), + inputOfport: 4, + expectedCommandArgs: []string{ + "add-port", "br-ovn", "pf0hpf", + "--", "set", "int", "pf0hpf", "type=dpdk", + "--", "set", "int", "pf0hpf", "ofport_request=4", + }, + expectedError: false, + }, + } + + for _, tt := range cases { + t.Run(tt.msg, func(t *testing.T) { + fakeExec := &kexecTesting.FakeExec{LookPathFunc: func(s string) (string, error) { return s, nil }} + c, err := newOvsClient(fakeExec) + g.Expect(err).ToNot(HaveOccurred()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + g.Expect(cmd).To(Equal("ovs-vsctl")) + g.Expect(args).To(ContainElements(tt.expectedCommandArgs)) + return kexec.New().Command("echo") + })) + + err = c.AddPortWithMetadata( + tt.inputBridge, + tt.inputPort, + tt.inputPortType, + tt.inputPortExternalIDs, + tt.inputInterfaceExternalIDs, + tt.inputOfport) + if tt.expectedError { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestListInterfaces(t *testing.T) { + g := NewWithT(t) + cases := []struct { + msg string + fakeCommandOutput string + expectedOutput map[string]interface{} + expectedError bool + }{ + { + msg: "usual command output", + fakeCommandOutput: ` +name : pf0hpf + +name : p0 + +name : p1 + +name : ovn-k8s-mp0 + +name : en3f0pf0sf15 + +name : en3f0pf0sf12 + +name : en3f0pf0sf1`, + expectedOutput: map[string]interface{}{ + "pf0hpf": struct{}{}, + "p0": struct{}{}, + "p1": struct{}{}, + "ovn-k8s-mp0": struct{}{}, + "en3f0pf0sf15": struct{}{}, + "en3f0pf0sf12": struct{}{}, + "en3f0pf0sf1": struct{}{}, + }, + expectedError: false, + }, + { + msg: "no interfaces", + fakeCommandOutput: "", + expectedOutput: make(map[string]interface{}), + expectedError: false, + }, + { + msg: "malformed command output - missing column in one of the lines", + fakeCommandOutput: ` +name : pf0hpf + +name : p0 + +name : p1 + +name : ovn-k8s-mp0 + +name en3f0pf0sf15 + +name : en3f0pf0sf12 + +name : en3f0pf0sf1`, + expectedOutput: map[string]interface{}{ + "pf0hpf": struct{}{}, + "p0": struct{}{}, + "p1": struct{}{}, + "ovn-k8s-mp0": struct{}{}, + "en3f0pf0sf12": struct{}{}, + "en3f0pf0sf1": struct{}{}, + }, + expectedError: false, + }, + } + + for _, tt := range cases { + t.Run(tt.msg, func(t *testing.T) { + fakeExec := &kexecTesting.FakeExec{LookPathFunc: func(s string) (string, error) { return s, nil }} + c, err := newOvsClient(fakeExec) + g.Expect(err).ToNot(HaveOccurred()) + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + g.Expect(cmd).To(Equal("ovs-vsctl")) + g.Expect(args).To(Equal([]string{ + "--columns=name", + "find", + "int", + "type=dpdk", + })) + return kexec.New().Command("echo", tt.fakeCommandOutput) + })) + + output, err := c.ListInterfaces(DPDK) + if tt.expectedError { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(BeComparableTo(tt.expectedOutput)) + }) + } +} + +func TestGetInterfacesWithPMDRXQueue(t *testing.T) { + g := NewWithT(t) + cases := []struct { + msg string + fakeCommandOutput string + expectedOutput map[string]interface{} + expectedError bool + }{ + { + msg: "usual command output", + fakeCommandOutput: ` +Displaying last 60 seconds pmd usage % +pmd thread numa_id 0 core_id 11: + isolated : true + port: en3f0pf0sf12 queue-id: 0 (enabled) pmd usage: 0 % + port: en3f0pf0sf14 queue-id: 0 (enabled) pmd usage: 0 % + port: en3f0pf0sf19 queue-id: 0 (enabled) pmd usage: 0 % + port: ovn-k8s-mp0 queue-id: 0 (enabled) pmd usage: 0 % + port: p0 queue-id: 0 (enabled) pmd usage: 0 % + port: p1 queue-id: 0 (enabled) pmd usage: 0 % + port: pf0hpf queue-id: 0 (enabled) pmd usage: 0 % + port: pf0vf39 queue-id: 0 (enabled) pmd usage: 0 % + overhead: 0 %`, + expectedOutput: map[string]interface{}{ + "en3f0pf0sf12": struct{}{}, + "en3f0pf0sf14": struct{}{}, + "en3f0pf0sf19": struct{}{}, + "ovn-k8s-mp0": struct{}{}, + "p0": struct{}{}, + "p1": struct{}{}, + "pf0hpf": struct{}{}, + "pf0vf39": struct{}{}, + }, + expectedError: false, + }, + { + msg: "no interfaces", + fakeCommandOutput: ` +Displaying last 60 seconds pmd usage % +pmd thread numa_id 0 core_id 11: + isolated : true + overhead: 0 %`, + expectedOutput: make(map[string]interface{}), + expectedError: false, + }, + { + msg: "malformed input due to missing column", + fakeCommandOutput: ` +Displaying last 60 seconds pmd usage % +pmd thread numa_id 0 core_id 11: + isolated : true + port: en3f0pf0sf12 queue-id: 0 (enabled) pmd usage: 0 % + port: en3f0pf0sf14 queue-id: 0 (enabled) pmd usage: 0 % + port: en3f0pf0sf19 queue-id: 0 (enabled) pmd usage: 0 % + port ovn-k8s-mp0 queue-id: 0 (enabled) pmd usage: 0 % + port p0 queue-id: 0 (enabled) pmd usage: 0 % + port: p1 queue-id: 0 (enabled) pmd usage: 0 % + port: pf0hpf queue-id: 0 (enabled) pmd usage: 0 % + port: pf0vf39 queue-id: 0 (enabled) pmd usage: 0 % + overhead: 0 %`, + expectedOutput: map[string]interface{}{ + "en3f0pf0sf12": struct{}{}, + "en3f0pf0sf14": struct{}{}, + "en3f0pf0sf19": struct{}{}, + "p1": struct{}{}, + "pf0hpf": struct{}{}, + "pf0vf39": struct{}{}, + }, + expectedError: false, + }, + } + + for _, tt := range cases { + t.Run(tt.msg, func(t *testing.T) { + fakeExec := &kexecTesting.FakeExec{LookPathFunc: func(s string) (string, error) { return s, nil }} + c, err := newOvsClient(fakeExec) + g.Expect(err).ToNot(HaveOccurred()) + cImpl := c.(*ovsClient) + + tmpDir, err := os.MkdirTemp("", "ovsclient") + defer func() { + err := os.RemoveAll(tmpDir) + g.Expect(err).ToNot(HaveOccurred()) + }() + g.Expect(err).NotTo(HaveOccurred()) + cImpl.fileSystemRoot = tmpDir + ovsSocketDir := filepath.Join(tmpDir, "/var/run/openvswitch") + g.Expect(os.MkdirAll(ovsSocketDir, 0755)).To(Succeed()) + ovsPidFile := filepath.Join(ovsSocketDir, "ovs-vswitchd.pid") + g.Expect(os.WriteFile(ovsPidFile, []byte("12345"), 0644)).To(Succeed()) + ovsSocketPath := filepath.Join(ovsSocketDir, "ovs-vswitchd.12345.ctl") + + fakeExec.CommandScript = append(fakeExec.CommandScript, kexecTesting.FakeCommandAction(func(cmd string, args ...string) kexec.Cmd { + g.Expect(cmd).To(Equal("ovs-appctl")) + g.Expect(args).To(Equal([]string{ + "-t", + ovsSocketPath, + "dpif-netdev/pmd-rxq-show", + })) + return kexec.New().Command("echo", tt.fakeCommandOutput) + })) + + output, err := cImpl.GetInterfacesWithPMDRXQueue() + if tt.expectedError { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(output).To(BeComparableTo(tt.expectedOutput)) + }) + } +} diff --git a/dpf-utils/internal/utils/ovsclient/mock/ovsclient.go b/dpf-utils/internal/utils/ovsclient/mock/ovsclient.go new file mode 100644 index 0000000..4ae75a8 --- /dev/null +++ b/dpf-utils/internal/utils/ovsclient/mock/ovsclient.go @@ -0,0 +1,374 @@ +// /* +// Copyright 2025 NVIDIA. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: types.go +// +// Generated by this command: +// +// mockgen -copyright_file ../../../hack/boilerplate.go.txt -destination mock/ovsclient.go -source types.go +// + +// Package mock_ovsclient is a generated GoMock package. +package mock_ovsclient + +import ( + net "net" + reflect "reflect" + + ovsclient "github.com/nvidia/ovn-kubernetes-components/internal/utils/ovsclient" + gomock "go.uber.org/mock/gomock" +) + +// MockOVSClient is a mock of OVSClient interface. +type MockOVSClient struct { + ctrl *gomock.Controller + recorder *MockOVSClientMockRecorder + isgomock struct{} +} + +// MockOVSClientMockRecorder is the mock recorder for MockOVSClient. +type MockOVSClientMockRecorder struct { + mock *MockOVSClient +} + +// NewMockOVSClient creates a new mock instance. +func NewMockOVSClient(ctrl *gomock.Controller) *MockOVSClient { + mock := &MockOVSClient{ctrl: ctrl} + mock.recorder = &MockOVSClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockOVSClient) EXPECT() *MockOVSClientMockRecorder { + return m.recorder +} + +// AddBridgeIfNotExists mocks base method. +func (m *MockOVSClient) AddBridgeIfNotExists(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddBridgeIfNotExists", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddBridgeIfNotExists indicates an expected call of AddBridgeIfNotExists. +func (mr *MockOVSClientMockRecorder) AddBridgeIfNotExists(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddBridgeIfNotExists", reflect.TypeOf((*MockOVSClient)(nil).AddBridgeIfNotExists), name) +} + +// AddPortIfNotExists mocks base method. +func (m *MockOVSClient) AddPortIfNotExists(bridge, port string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddPortIfNotExists", bridge, port) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddPortIfNotExists indicates an expected call of AddPortIfNotExists. +func (mr *MockOVSClientMockRecorder) AddPortIfNotExists(bridge, port any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPortIfNotExists", reflect.TypeOf((*MockOVSClient)(nil).AddPortIfNotExists), bridge, port) +} + +// AddPortWithMetadata mocks base method. +func (m *MockOVSClient) AddPortWithMetadata(bridge, port string, portType ovsclient.PortType, portExternalIDs, interfaceExternalIDs map[string]string, ofport int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddPortWithMetadata", bridge, port, portType, portExternalIDs, interfaceExternalIDs, ofport) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddPortWithMetadata indicates an expected call of AddPortWithMetadata. +func (mr *MockOVSClientMockRecorder) AddPortWithMetadata(bridge, port, portType, portExternalIDs, interfaceExternalIDs, ofport any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPortWithMetadata", reflect.TypeOf((*MockOVSClient)(nil).AddPortWithMetadata), bridge, port, portType, portExternalIDs, interfaceExternalIDs, ofport) +} + +// BridgeExists mocks base method. +func (m *MockOVSClient) BridgeExists(name string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BridgeExists", name) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BridgeExists indicates an expected call of BridgeExists. +func (mr *MockOVSClientMockRecorder) BridgeExists(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BridgeExists", reflect.TypeOf((*MockOVSClient)(nil).BridgeExists), name) +} + +// DeleteBridgeIfExists mocks base method. +func (m *MockOVSClient) DeleteBridgeIfExists(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBridgeIfExists", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteBridgeIfExists indicates an expected call of DeleteBridgeIfExists. +func (mr *MockOVSClientMockRecorder) DeleteBridgeIfExists(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBridgeIfExists", reflect.TypeOf((*MockOVSClient)(nil).DeleteBridgeIfExists), name) +} + +// DeletePort mocks base method. +func (m *MockOVSClient) DeletePort(port string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeletePort", port) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeletePort indicates an expected call of DeletePort. +func (mr *MockOVSClientMockRecorder) DeletePort(port any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePort", reflect.TypeOf((*MockOVSClient)(nil).DeletePort), port) +} + +// GetInterfaceExternalIDs mocks base method. +func (m *MockOVSClient) GetInterfaceExternalIDs(iface string) (map[string]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInterfaceExternalIDs", iface) + ret0, _ := ret[0].(map[string]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetInterfaceExternalIDs indicates an expected call of GetInterfaceExternalIDs. +func (mr *MockOVSClientMockRecorder) GetInterfaceExternalIDs(iface any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInterfaceExternalIDs", reflect.TypeOf((*MockOVSClient)(nil).GetInterfaceExternalIDs), iface) +} + +// GetInterfaceOfPort mocks base method. +func (m *MockOVSClient) GetInterfaceOfPort(port string) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInterfaceOfPort", port) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetInterfaceOfPort indicates an expected call of GetInterfaceOfPort. +func (mr *MockOVSClientMockRecorder) GetInterfaceOfPort(port any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInterfaceOfPort", reflect.TypeOf((*MockOVSClient)(nil).GetInterfaceOfPort), port) +} + +// GetInterfacesWithPMDRXQueue mocks base method. +func (m *MockOVSClient) GetInterfacesWithPMDRXQueue() (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInterfacesWithPMDRXQueue") + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetInterfacesWithPMDRXQueue indicates an expected call of GetInterfacesWithPMDRXQueue. +func (mr *MockOVSClientMockRecorder) GetInterfacesWithPMDRXQueue() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInterfacesWithPMDRXQueue", reflect.TypeOf((*MockOVSClient)(nil).GetInterfacesWithPMDRXQueue)) +} + +// GetPortExternalIDs mocks base method. +func (m *MockOVSClient) GetPortExternalIDs(port string) (map[string]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPortExternalIDs", port) + ret0, _ := ret[0].(map[string]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPortExternalIDs indicates an expected call of GetPortExternalIDs. +func (mr *MockOVSClientMockRecorder) GetPortExternalIDs(port any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPortExternalIDs", reflect.TypeOf((*MockOVSClient)(nil).GetPortExternalIDs), port) +} + +// InterfaceToBridge mocks base method. +func (m *MockOVSClient) InterfaceToBridge(iface string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InterfaceToBridge", iface) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InterfaceToBridge indicates an expected call of InterfaceToBridge. +func (mr *MockOVSClientMockRecorder) InterfaceToBridge(iface any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InterfaceToBridge", reflect.TypeOf((*MockOVSClient)(nil).InterfaceToBridge), iface) +} + +// ListInterfaces mocks base method. +func (m *MockOVSClient) ListInterfaces(portType ovsclient.PortType) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListInterfaces", portType) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListInterfaces indicates an expected call of ListInterfaces. +func (mr *MockOVSClientMockRecorder) ListInterfaces(portType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListInterfaces", reflect.TypeOf((*MockOVSClient)(nil).ListInterfaces), portType) +} + +// SetBridgeController mocks base method. +func (m *MockOVSClient) SetBridgeController(bridge, controller string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetBridgeController", bridge, controller) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetBridgeController indicates an expected call of SetBridgeController. +func (mr *MockOVSClientMockRecorder) SetBridgeController(bridge, controller any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetBridgeController", reflect.TypeOf((*MockOVSClient)(nil).SetBridgeController), bridge, controller) +} + +// SetBridgeDataPathType mocks base method. +func (m *MockOVSClient) SetBridgeDataPathType(bridge string, bridgeType ovsclient.BridgeDataPathType) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetBridgeDataPathType", bridge, bridgeType) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetBridgeDataPathType indicates an expected call of SetBridgeDataPathType. +func (mr *MockOVSClientMockRecorder) SetBridgeDataPathType(bridge, bridgeType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetBridgeDataPathType", reflect.TypeOf((*MockOVSClient)(nil).SetBridgeDataPathType), bridge, bridgeType) +} + +// SetBridgeHostToServicePort mocks base method. +func (m *MockOVSClient) SetBridgeHostToServicePort(bridge, port string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetBridgeHostToServicePort", bridge, port) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetBridgeHostToServicePort indicates an expected call of SetBridgeHostToServicePort. +func (mr *MockOVSClientMockRecorder) SetBridgeHostToServicePort(bridge, port any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetBridgeHostToServicePort", reflect.TypeOf((*MockOVSClient)(nil).SetBridgeHostToServicePort), bridge, port) +} + +// SetBridgeMAC mocks base method. +func (m *MockOVSClient) SetBridgeMAC(bridge string, mac net.HardwareAddr) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetBridgeMAC", bridge, mac) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetBridgeMAC indicates an expected call of SetBridgeMAC. +func (mr *MockOVSClientMockRecorder) SetBridgeMAC(bridge, mac any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetBridgeMAC", reflect.TypeOf((*MockOVSClient)(nil).SetBridgeMAC), bridge, mac) +} + +// SetBridgeUplinkPort mocks base method. +func (m *MockOVSClient) SetBridgeUplinkPort(bridge, port string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetBridgeUplinkPort", bridge, port) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetBridgeUplinkPort indicates an expected call of SetBridgeUplinkPort. +func (mr *MockOVSClientMockRecorder) SetBridgeUplinkPort(bridge, port any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetBridgeUplinkPort", reflect.TypeOf((*MockOVSClient)(nil).SetBridgeUplinkPort), bridge, port) +} + +// SetDOCAInit mocks base method. +func (m *MockOVSClient) SetDOCAInit(enable bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetDOCAInit", enable) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetDOCAInit indicates an expected call of SetDOCAInit. +func (mr *MockOVSClientMockRecorder) SetDOCAInit(enable any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDOCAInit", reflect.TypeOf((*MockOVSClient)(nil).SetDOCAInit), enable) +} + +// SetKubernetesHostNodeName mocks base method. +func (m *MockOVSClient) SetKubernetesHostNodeName(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetKubernetesHostNodeName", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetKubernetesHostNodeName indicates an expected call of SetKubernetesHostNodeName. +func (mr *MockOVSClientMockRecorder) SetKubernetesHostNodeName(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetKubernetesHostNodeName", reflect.TypeOf((*MockOVSClient)(nil).SetKubernetesHostNodeName), name) +} + +// SetOVNEncapIP mocks base method. +func (m *MockOVSClient) SetOVNEncapIP(ip net.IP) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetOVNEncapIP", ip) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetOVNEncapIP indicates an expected call of SetOVNEncapIP. +func (mr *MockOVSClientMockRecorder) SetOVNEncapIP(ip any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetOVNEncapIP", reflect.TypeOf((*MockOVSClient)(nil).SetOVNEncapIP), ip) +} + +// SetPatchPortPeer mocks base method. +func (m *MockOVSClient) SetPatchPortPeer(port, peer string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetPatchPortPeer", port, peer) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetPatchPortPeer indicates an expected call of SetPatchPortPeer. +func (mr *MockOVSClientMockRecorder) SetPatchPortPeer(port, peer any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPatchPortPeer", reflect.TypeOf((*MockOVSClient)(nil).SetPatchPortPeer), port, peer) +} + +// SetPortType mocks base method. +func (m *MockOVSClient) SetPortType(port string, portType ovsclient.PortType) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetPortType", port, portType) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetPortType indicates an expected call of SetPortType. +func (mr *MockOVSClientMockRecorder) SetPortType(port, portType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPortType", reflect.TypeOf((*MockOVSClient)(nil).SetPortType), port, portType) +} diff --git a/dpf-utils/internal/utils/ovsclient/ovsclient.go b/dpf-utils/internal/utils/ovsclient/ovsclient.go new file mode 100644 index 0000000..0956b25 --- /dev/null +++ b/dpf-utils/internal/utils/ovsclient/ovsclient.go @@ -0,0 +1,24 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ovsclient + +import kexec "k8s.io/utils/exec" + +// New creates a new OVSClient +func New(exec kexec.Interface) (OVSClient, error) { + return newOvsClient(exec) +} diff --git a/dpf-utils/internal/utils/ovsclient/types.go b/dpf-utils/internal/utils/ovsclient/types.go new file mode 100644 index 0000000..c087514 --- /dev/null +++ b/dpf-utils/internal/utils/ovsclient/types.go @@ -0,0 +1,91 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ovsclient + +import ( + "net" +) + +// OVSClient is a client that can be used to do specific actions on OVS. +// +//go:generate mockgen -copyright_file ../../../hack/boilerplate.go.txt -destination mock/ovsclient.go -source types.go +type OVSClient interface { + // BridgeExists checks if a bridge exists + BridgeExists(name string) (bool, error) + // AddBridgeIfNotExists adds a bridge if it doesn't exist + AddBridgeIfNotExists(name string) error + // DeleteBridgeIfExists deletes a bridge if it exists + DeleteBridgeIfExists(name string) error + // SetBridgeDataPathType sets the datapath type of a bridge + SetBridgeDataPathType(bridge string, bridgeType BridgeDataPathType) error + // SetBridgeMAC sets the MAC address for the bridge interface + SetBridgeMAC(bridge string, mac net.HardwareAddr) error + // SetBridgeUplink sets the bridge-uplink external ID of the bridge. It overrides if already exists. + SetBridgeUplinkPort(bridge string, port string) error + // SetBridgeHostToServicePort sets the host-to-service external ID of the bridge. It overrides if already exists. + SetBridgeHostToServicePort(bridge string, port string) error + // SetBridgeController sets the controller for a bridge + SetBridgeController(bridge string, controller string) error + + // AddPortIfNotExists adds a port to a bridge if it doesn't exist + AddPortIfNotExists(bridge string, port string) error + // SetPortType sets the type of a port + SetPortType(port string, portType PortType) error + // SetPatchPortPeer sets the peer for a patch port + SetPatchPortPeer(port string, peer string) error + + // SetOVNEncapIP sets the ovn-encap-ip external ID in the Open_vSwitch table in OVS + SetOVNEncapIP(ip net.IP) error + // SetDOCAInit sets the doca-init other_config in the Open_vSwitch table in OVS. Requires OVS daemon restart. + SetDOCAInit(enable bool) error + // SetKubernetesHostNodeName sets the host-k8s-nodename external ID in the Open_vSwitch table in OVS + SetKubernetesHostNodeName(name string) error + + // InterfaceToBridge returns the bridge an interface exists in + InterfaceToBridge(iface string) (string, error) + // DeletePort deletes a port + DeletePort(port string) error + // GetInterfaceOfPort returns the ofport number of a port + GetInterfaceOfPort(port string) (int, error) + // GetPortExternalIDs returns the external_ids of an OVS port + GetPortExternalIDs(port string) (map[string]string, error) + // GetInterfaceExternalIDs returns the external_ids of an OVS interface + GetInterfaceExternalIDs(iface string) (map[string]string, error) + // AddPortWithMetadata adds a port to the given bridge with the specified external IDs and ofport request in a single + // transaction + AddPortWithMetadata(bridge string, port string, portType PortType, portExternalIDs map[string]string, interfaceExternalIDs map[string]string, ofport int) error + // ListInterfaces lists all the interfaces that exist in OVS of a particular type + ListInterfaces(portType PortType) (map[string]interface{}, error) + // GetInterfacesWithPMDRXQueue returns all the interfaces that have a PMD Rx queue + GetInterfacesWithPMDRXQueue() (map[string]interface{}, error) +} + +// BridgeDataPathType represents the various datapath types a bridge can be configured with +type BridgeDataPathType string + +const ( + NetDev BridgeDataPathType = "netdev" +) + +// PortType represents the various types a port can be configured with +type PortType string + +const ( + DPDK PortType = "dpdk" + Internal PortType = "internal" + Patch PortType = "patch" +) diff --git a/helm/ovn-kubernetes-dpf/values.yaml.tmpl b/helm/ovn-kubernetes-dpf/values.yaml.tmpl index fe5129a..0527cb7 100644 --- a/helm/ovn-kubernetes-dpf/values.yaml.tmpl +++ b/helm/ovn-kubernetes-dpf/values.yaml.tmpl @@ -28,9 +28,6 @@ nodeWithDPUManifests: # -- Variables related to manifests that are deployed for nodes without DPU nodeWithoutDPUManifests: enabled: false - imagedpf: - repository: ${OVNKUBERNETES_IMAGE} - tag: ${TAG} image: repository: ${OVNKUBERNETES_IMAGE} tag: ${TAG}