fix: remove interrupt timeout which was flawed by design

additionally changing the default copy dir to
work with versions of linux that do not
support exec in the /tmp dir

Author: lockwobr

.vscode/launch.json

@@ -9,7 +9,7 @@
             "type": "go",
             "request": "launch",
             "mode": "debug",
-            "program": "${workspaceRoot}/cmd/main.go",
+            "program": "${workspaceRoot}/operator/cmd/main.go",
             "buildFlags": "--ldflags '-X github.com/NVIDIA/skyhook/internal/version.GIT_SHA=foobars -X github.com/NVIDIA/skyhook/internal/version.VERSION=v0.5.0'",
             "env": {
                 "ENABLE_WEBHOOKS": "false",

chart/templates/skyhook-crd.yaml

@@ -126,12 +126,6 @@ spec:
                     minimum: 0
                     nullable: true
                     type: integer
-                  timeout:
-                    default: 15m
-                    description: Timeout is how long we wait before we assume something
-                      bad happened to a give node, and are going to move forward.
-                    format: duration
-                    type: string
                 type: object
               nodeSelectors:
                 description: NodeSelector are a set of labels we want to monitor nodes

chart/values.yaml

@@ -43,7 +43,7 @@ controllerManager:
     env:
       ## copyDirRoot is the directory for which the operator will work from on the host. 
       ## Some environments may require this to be set to a specific directory.
-      copyDirRoot: /tmp
+      copyDirRoot: /var/lib/skyhook
       ## enableWebhooks: "true" will enable the webhook setup in the operator controller. 
       ## Default is "true" and is required for production.
       enableWebhooks: "true"

operator/Makefile

@@ -190,7 +190,7 @@ create-kind-cluster: ## deletes and creates a new kind cluster. versions is set
 
 podman-create-machine: ## creates a podman machine
 	podman machine stop podman-machine-default || true
-	podman machine rm podman-machine-default || true
+	podman machine rm -f podman-machine-default || true
 	podman machine init --cpus=6 -m=12288 --disk-size=300 podman-machine-default
 	podman machine start podman-machine-default
 
@@ -271,10 +271,29 @@ ifndef ignore-not-found
   ignore-not-found = false
 endif
 
+create-namespace: ## Create the namespace in the K8s cluster specified in ~/.kube/config.
+	$(KUBECTL) create namespace $(SKYHOOK_NAMESPACE) --dry-run=client -o yaml | kubectl apply -f -
+
 .PHONY: install
-install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+install: manifests kustomize create-namespace ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
-	$(KUBECTL) create namespace $(SKYHOOK_NAMESPACE) --dry-run=client -o yaml | kubectl apply -f -
+
+cert-manager-yaml-url=https://github.com/cert-manager/cert-manager/releases/download/v1.17.0/cert-manager.yaml
+.PHONY: install-cert-manager
+install-cert-manager: ## Install cert-manager into the K8s cluster specified in ~/.kube/config.
+	kubectl apply -f $(cert-manager-yaml-url)
+	sleep 30 ## wait for cert-manager to be ready
+
+.PHONY: install-helm-chart
+install-helm-chart: helm create-namespace install-cert-manager ## Install helm chart into the K8s cluster specified in ~/.kube/config.
+	$(HELM) install skyhook-operator ../chart -n $(SKYHOOK_NAMESPACE) 
+
+.PHONY: uninstall-helm-chart
+uninstall-helm-chart: helm ## Uninstall helm chart from the K8s cluster specified in ~/.kube/config.
+	$(HELM) uninstall skyhook-operator -n $(SKYHOOK_NAMESPACE)
+
+uninstall-cert-manager: ## Uninstall cert-manager from the K8s cluster specified in ~/.kube/config.
+	kubectl delete -f $(cert-manager-yaml-url)
 
 .PHONY: uninstall
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.

operator/api/v1alpha1/skyhook_types.go

@@ -146,11 +146,6 @@ type InterruptionBudget struct {
 	//+kubebuilder:validation:Minimum=0
 	//+nullable
 	Count *int `json:"count,omitempty"`
-
-	// Timeout is how long we wait before we assume something bad happened to a give node, and are going to move forward.
-	//+kubebuilder:validation:Format=duration
-	//+kubebuilder:default="15m"
-	Timeout *metav1.Duration `json:"timeout,omitempty"`
 }
 
 func (i *InterruptionBudget) Validate() error {

operator/api/v1alpha1/zz_generated.deepcopy.go

@@ -113,12 +113,6 @@ spec:
                     minimum: 0
                     nullable: true
                     type: integer
-                  timeout:
-                    default: 15m
-                    description: Timeout is how long we wait before we assume something
-                      bad happened to a give node, and are going to move forward.
-                    format: duration
-                    type: string
                 type: object
               nodeSelectors:
                 description: NodeSelector are a set of labels we want to monitor nodes

operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml

@@ -84,7 +84,7 @@ spec:
           - name: IMAGE_PULL_SECRET
             value: node-init-secret
           - name: COPY_DIR_ROOT
-            value: /tmp
+            value: /var/lib/skyhook
           - name: REAPPLY_ON_REBOOT
             value: "false"
           - name: RUNTIME_REQUIRED_TAINT_KEY

operator/config/manager/manager.yaml

@@ -73,6 +73,7 @@ GINKGO ?= $(LOCALBIN)/ginkgo
 MOCKERY ?= $(LOCALBIN)/mockery
 CHAINSAW ?= $(LOCALBIN)/chainsaw
 HELMIFY ?= $(LOCALBIN)/helmify
+HELM ?= $(LOCALBIN)/helm
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading.

operator/deps.mk

@@ -265,23 +265,18 @@ func NewNodePicker(runtimeRequiredToleration corev1.Toleration) *NodePicker {
 }
 
 // primeAndPruneNodes add current priority from skyhook status, and check time removing old ones
-func (s *NodePicker) primeAndPruneNodes(skyhook *wrapper.Skyhook) {
-
-	now := time.Now()
-	for n, t := range skyhook.Status.NodePriority {
-		if now.Sub(t.Time) > skyhook.Spec.InterruptionBudget.Timeout.Duration {
-			// prune
-			// was not cleaned up before now, so it has timed out...
-			// this could cause issues if the node is still in the cluster api,
-			// but is bad, but seems like if that is true, there might be bigger problems
-			// so removing from status
-			delete(skyhook.Status.NodePriority, n)
-			skyhook.Updated = true
+func (s *NodePicker) primeAndPruneNodes(skyhook *skyhookNodes) {
+
+	for n, t := range skyhook.skyhook.Status.NodePriority {
+		// prune
+		// if the node is complete, remove it from the priority list
+		if nodeStatus, _ := skyhook.GetNode(n); nodeStatus == v1alpha1.StatusComplete {
+			delete(skyhook.skyhook.Status.NodePriority, n)
+			skyhook.skyhook.Updated = true
 		} else {
 			s.priorityNodes[n] = t.Time
 		}
 	}
-
 }
 
 // upsertPick updates or inserts the node priority for a given name in the Skyhook object.
@@ -328,7 +323,7 @@ func CheckTaintToleration(tolerations []corev1.Toleration, taints []corev1.Taint
 
 func (np *NodePicker) SelectNodes(s *skyhookNodes) []wrapper.SkyhookNode {
 
-	np.primeAndPruneNodes(s.skyhook)
+	np.primeAndPruneNodes(s)
 
 	nodes := make([]wrapper.SkyhookNode, 0)