feat: remove cert manager

Author: lockwobr

.github/workflows/agent-ci.yaml

@@ -98,11 +98,14 @@ jobs:
           apt-get update && apt-get install -y make git jq
           cd agent
           # if this is a tag build, use the tag as the version, otherwise use the sha
-          TAGS="-t ${REGISTRY@L}/${{env.IMAGE_NAME}}/agent:${{ github.sha }}"
+          git fetch --all
+          export GIT_SHA=$(git rev-parse --short ${{ github.sha }})
+          TAGS="-t ${REGISTRY@L}/${{env.IMAGE_NAME}}/agent:${GIT_SHA}"
           case ${{ github.ref_type }} in
             branch)
                 # The last tag + current git sha
-                export AGENT_VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "0.0.0")+${{ github.sha }}
+                export AGENT_VERSION=$(git tag --list 'agent*' --sort=-v:refname | head -n 1 | cut -d/ -f2 || echo "0.0.0")+${GIT_SHA}
+                TAGS="$TAGS -t ${REGISTRY@L}/${{env.IMAGE_NAME}}/agent:$(echo "${AGENT_VERSION}" | tr + -)"
                 ;;
             tag)
                 # The version part of the tag

.github/workflows/operator-ci.yaml

@@ -54,19 +54,6 @@ env:
 
 # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
 jobs:
-  ## move it all down to the tests job, should do all the tests then, and not double install stuff
-  # unit-test:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Setup Go 1.23
-  #       uses: actions/setup-go@v5
-  #       with:
-  #         go-version: 1.23
-  #     - name: Unit tests
-  #       run: |
-  #         cd operator
-  #         make unit-tests
   tests:
     runs-on: ubuntu-latest
     steps:
@@ -78,6 +65,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: 1.23
+          cache-dependency-path: operator/go.sum
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
@@ -90,6 +78,29 @@ jobs:
         with:
           version: v0.26.0
           install_only: true
+      - name: Restore cached Binaries
+        id: cached-binaries
+        uses: actions/cache/restore@v4
+        with:
+          key: ${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
+          restore-keys: ${{ runner.os }}-${{ runner.arch }}-bin-
+          path: |
+            ${{ github.workspace }}/operator/bin
+            ~/.cache/go-build
+      - name: Install dependencies
+        if: steps.cached-binaries.outputs.cache-hit != 'true'
+        run: |
+          cd operator
+          make install-deps
+      - name: Save cached Binaries
+        id: save-cached-binaries
+        if: steps.cached-binaries.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          key: ${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
+          path: |
+            ${{ github.workspace }}/operator/bin
+            ~/.cache/go-build
       - name: end-to-end-tests
         run: |
           cd operator
@@ -131,11 +142,14 @@ jobs:
           apt-get update && apt-get install -y make git jq
           cd operator
           # if this is a tag build, use the tag as the version, otherwise use the sha
-          TAGS="-t ${REGISTRY@L}/${{env.IMAGE_NAME}}/operator:${{ github.sha }}"
+          git fetch --all
+          export GIT_SHA=$(git rev-parse --short ${{ github.sha }})
+          TAGS="-t ${REGISTRY@L}/${{env.IMAGE_NAME}}/operator:${GIT_SHA}"
           case ${{ github.ref_type }} in
             branch)
                 # The last tag + current git sha
-                export OPERATOR_VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "0.0.0")+${{ github.sha }}
+                export OPERATOR_VERSION=$(git tag --list 'operator*' --sort=-v:refname | head -n 1 | cut -d/ -f2 || echo "0.0.0")+${GIT_SHA}
+                TAGS="$TAGS -t ${REGISTRY@L}/${{env.IMAGE_NAME}}/operator:$(echo "${OPERATOR_VERSION}" | tr + -)"
                 ;;
             tag)
                 # The version part of the tag
@@ -149,7 +163,7 @@ jobs:
           esac
           set -x
           docker buildx build \
-            --build-arg GIT_SHA=$${{ github.sha }} \
+            --build-arg GIT_SHA=${GIT_SHA} \
             --build-arg VERSION=${OPERATOR_VERSION} \
             --build-arg GO_VERSION=${GO_VERSION} \
             --push \

.gitignore

@@ -1,3 +1,4 @@
 .cursorignore
 .pytest_cache
-.idea
+.idea
+**/err.txt

.vscode/launch.json

@@ -10,6 +10,7 @@
             "request": "launch",
             "mode": "debug",
             "program": "${workspaceRoot}/operator/cmd/main.go",
+            "cwd": "${workspaceRoot}/operator",
             "buildFlags": "--ldflags '-X github.com/NVIDIA/skyhook/internal/version.GIT_SHA=foobars -X github.com/NVIDIA/skyhook/internal/version.VERSION=v0.5.0'",
             "env": {
                 "ENABLE_WEBHOOKS": "false",

README.md

@@ -50,7 +50,6 @@ There are a few pre-built generalist packages available at [NVIDIA/skyhook-packa
 ## Quick Start
 
 ### Install the operator
-  1. Install cert-manager `kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.2/cert-manager.yaml`
   1. Create a secret for the operator to pull images `kubectl create secret generic node-init-secret --from-file=.dockerconfigjson=${HOME}/.config/containers/auth.json --type=kubernetes.io/dockerconfigjson -n  skyhook`
   1. Install the operator `helm install skyhook ./chart --namespace skyhook`
 

agent/Makefile

@@ -56,11 +56,14 @@ docker-setup:
 	$(DOCKER_CMD) buildx create --platform linux/amd64,linux/arm64 --use builder
 	$(DOCKER_CMD) run --privileged --rm tonistiigi/binfmt --install amd64,arm64
 
-ACTUAL_TAGS=$(shell echo "-t $(REGISTRY)/$(AGENT_IMAGE):$(shell date +%y.%m.%d-%H%M%S)-$(COMMIT_SHORT_SHA) $(TAGS)" | tr A-Z a-z)
+GIT_SHA=$(shell git rev-parse --short HEAD)
+ACTUAL_TAGS=$(shell echo "-t $(REGISTRY)/$(AGENT_IMAGE):$(shell date +%y.%m.%d-%H%M%S)-$(GIT_SHA) $(TAGS)" | tr A-Z a-z)
 .PHONY: docker-build-only
 docker-build-only:
 	@echo "Building skyhook-agent $(DOCKER_CMD) image with tags: $(ACTUAL_TAGS)"
-	$(DOCKER_CMD) buildx build $(BUILD_ARGS) --build-arg AGENT_VERSION=$(AGENT_VERSION) --platform linux/amd64,linux/arm64 $(ACTUAL_TAGS) --metadata-file=metadata.json -f ../containers/agent.Dockerfile .
+	$(DOCKER_CMD) buildx build $(BUILD_ARGS) --build-arg AGENT_VERSION=$(AGENT_VERSION) \
+		--build-arg GIT_SHA=$(GIT_SHA) \
+		--platform linux/amd64,linux/arm64 $(ACTUAL_TAGS) --metadata-file=metadata.json -f ../containers/agent.Dockerfile .
 
 ##@ Vendor
 .PHONY: vendor

chart/README.md

@@ -23,7 +23,7 @@ Settings | Description | Default |
 | controllerManager.tolerations | add tolerations to the controller manager pod | [] |
 | controllerManager.selectors | add node selectors to the controller manager pod | {} |
 | controllerManager.manager.env.copyDirRoot | Directory for which the operator will work from on the host. Some environments may require this to be set to a specific directory. | /tmp |
-| controllerManager.manager.env.enableWebhooks | Enable the webhook setup in the operator controller. Default is "true" and is required for production. | "true" |
+| webhooks.enable | Enable the webhook setup in the operator controller. Default is "true" and is required for production. | "true" |
 | controllerManager.manager.env.leaderElection | Enable leader election for the operator controller. Default is "true" and is required for production. | "true" |
 | controllerManager.manager.env.logLevel | Log level for the operator controller. If you want more or less logs, change this value to "debug" or "error". | "info" |
 | controllerManager.manager.env.reapplyOnReboot | Reapply the packages on reboot. This is useful for systems that are read-only. | "false" |

chart/templates/cleanup-webhook-job.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.webhook.enable }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ include "chart.fullname" . }}-webhook-cleanup"
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ include "chart.fullname" . }}-controller-manager
+      containers:
+      - name: cleanup
+        image: {{ .Values.webhook.removalImage | default "bitnami/kubectl" }}:{{ .Values.webhook.removalTag | default "latest" }}
+        command:
+        - /bin/sh
+        - -c
+        - |
+          NAMESPACE="{{ .Release.Namespace }}"
+          WEBHOOK_SECRET_NAME="{{ .Values.webhook.secretName | default "webhook-cert" }}"
+          VALIDATING_WEBHOOK_CONFIGURATION_NAME="skyhook-operator-validating-webhook"
+          MUTATING_WEBHOOK_CONFIGURATION_NAME="skyhook-operator-mutating-webhook"
+          kubectl delete secret -n $NAMESPACE $WEBHOOK_SECRET_NAME || true
+          kubectl delete validatingwebhookconfiguration $VALIDATING_WEBHOOK_CONFIGURATION_NAME || true
+          kubectl delete mutatingwebhookconfiguration $MUTATING_WEBHOOK_CONFIGURATION_NAME || true
+{{- end }} 

chart/templates/deployment.yaml

@@ -62,7 +62,7 @@ spec:
         - name: LOG_LEVEL
           value: {{ quote .Values.controllerManager.manager.env.logLevel }}
         - name: ENABLE_WEBHOOKS
-          value: {{ quote .Values.controllerManager.manager.env.enableWebhooks }}
+          value: {{ quote .Values.webhook.enable }}
         - name: NAMESPACE
           value: {{ .Release.Namespace }}
         - name: IMAGE_PULL_SECRET
@@ -97,7 +97,10 @@ spec:
             path: /readyz
             port: 8081
           initialDelaySeconds: 5
-          periodSeconds: 10
+          periodSeconds: 20
+          successThreshold: 1
+          failureThreshold: 2
+          timeoutSeconds: 3
         resources: {{ if .Values.controllerManager.manager.resources }}{{- toYaml .Values.controllerManager.manager.resources | nindent 10 }}{{ else }}
           limits:
             cpu: {{ maxf 1000 (mulf 1.6 .Values.estimatedNodeCount (maxf 1 (mulf 0.4 .Values.estimatedPackageCount))) | int }}m
@@ -108,10 +111,6 @@ spec:
           {{- end }}
         securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext
           | nindent 10 }}
-        volumeMounts:
-        - mountPath: /tmp/k8s-webhook-server/serving-certs
-          name: cert
-          readOnly: true
       - args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }}
         env:
         - name: KUBERNETES_CLUSTER_DOMAIN
@@ -132,11 +131,6 @@ spec:
         runAsNonRoot: true
       serviceAccountName: {{ include "chart.fullname" . }}-controller-manager
       terminationGracePeriodSeconds: 10
-      volumes:
-      - name: cert
-        secret:
-          defaultMode: 420
-          secretName: webhook-server-cert
 {{ if ((.Values.controllerManager.podDisruptionBudget).minAvailable) }}
 {{ if ge .Values.controllerManager.podDisruptionBudget.minAvailable .Values.controllerManager.replicas }}
 {{- $_ := required "minAvailable to be less than replicas" .nil }}

chart/templates/manager-rbac.yaml

@@ -5,6 +5,19 @@ metadata:
   labels:
   {{- include "chart.labels" . | nindent 4 }}
 rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  - validatingwebhookconfigurations
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -66,6 +79,18 @@ rules:
   - pods/status
   verbs:
   - get
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - skyhook.nvidia.com
   resources: