setting node annotation with 'sriovnetwork.openshift.io/use-external-drainer=true' in case exteranl drainer is enabled

The motivation is for external drainer verification, that SRIOV operator is set with external drainer
Signed-off-by: Ido Heyvi <[email protected]>

Author: heyvister1

pkg/consts/constants.go

@@ -76,6 +76,7 @@ const (
 	SriovDevicePluginLabelEnabled  = "Enabled"
 	SriovDevicePluginLabelDisabled = "Disabled"
 
+	NodeExternalDrainerAnnotation   = "sriovnetwork.openshift.io/use-external-drainer"
 	NodeDrainAnnotation             = "sriovnetwork.openshift.io/state"
 	NodeStateDrainAnnotation        = "sriovnetwork.openshift.io/desired-state"
 	NodeStateDrainAnnotationCurrent = "sriovnetwork.openshift.io/current-state"

pkg/daemon/daemon.go

@@ -408,6 +408,17 @@ func (dn *NodeReconciler) apply(ctx context.Context, desiredNodeState *sriovnetw
 		return ctrl.Result{}, err
 	}
 
+	// remove external drainer annotation
+	if err := utils.AnnotateNode(ctx,
+		desiredNodeState.Name,
+		consts.NodeExternalDrainerAnnotation,
+		"", dn.client); err != nil {
+		if !errors.IsNotFound(err) {
+			reqLogger.Error(err, "failed to remove node external drainer annotation")
+			return ctrl.Result{}, err
+		}
+	}
+
 	reqLogger.Info("sync succeeded")
 	syncStatus := consts.SyncStatusSucceeded
 	lastSyncError := ""
@@ -532,6 +543,17 @@ func (dn *NodeReconciler) handleDrain(ctx context.Context, desiredNodeState *sri
 		return false, nil
 	}
 
+	// add external drainer annotation if enabled
+	if vars.UseExternalDrainer {
+		if err := utils.AnnotateNode(ctx,
+			desiredNodeState.Name,
+			consts.NodeExternalDrainerAnnotation,
+			"true", dn.client); err != nil {
+			funcLog.Error(err, "failed to add node external drainer annotation")
+			return false, err
+		}
+	}
+
 	// annotate both node and node state with drain or reboot
 	annotation := consts.DrainRequired
 	if reqReboot {

pkg/daemon/daemon_test.go

@@ -51,8 +51,9 @@ var (
 )
 
 const (
-	waitTime  = 30 * time.Minute
+	waitTime  = 10 * time.Second
 	retryTime = 5 * time.Second
+	nodeName  = "node1"
 )
 
 var _ = Describe("Daemon Controller", Ordered, func() {
@@ -151,7 +152,7 @@ var _ = Describe("Daemon Controller", Ordered, func() {
 		daemonReconciler = createDaemon(hostHelper, platformHelper, featureGates, []string{})
 		startDaemon(daemonReconciler)
 
-		_, nodeState = createNode("node1")
+		_, nodeState = createNode(nodeName)
 	})
 
 	AfterAll(func() {
@@ -253,6 +254,11 @@ var _ = Describe("Daemon Controller", Ordered, func() {
 					ToNot(HaveOccurred())
 
 				g.Expect(nodeState.Annotations[constants.NodeStateDrainAnnotation]).To(Equal(constants.DrainRequired))
+				// verify that external drainer annotation doesn't exist
+				node := &corev1.Node{}
+				g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: nodeName}, node)).
+					ToNot(HaveOccurred())
+				g.Expect(node.Annotations).NotTo(ContainElement(constants.NodeExternalDrainerAnnotation))
 			}, waitTime, retryTime).Should(Succeed())
 
 			patchAnnotation(nodeState, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete)
@@ -334,6 +340,73 @@ var _ = Describe("Daemon Controller", Ordered, func() {
 			eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded)
 			assertLastStatusTransitionsContains(nodeState, 2, constants.SyncStatusInProgress)
 		})
+
+		It("Should apply external drainer annotation when useExternalDrainer is true", func(ctx context.Context) {
+			DeferCleanup(func(x bool) { vars.UseExternalDrainer = x }, vars.UseExternalDrainer)
+			vars.UseExternalDrainer = true
+
+			discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{
+				{
+					Name:           "eno1",
+					Driver:         "ice",
+					PciAddress:     "0000:16:00.0",
+					DeviceID:       "1593",
+					Vendor:         "8086",
+					EswitchMode:    "legacy",
+					LinkAdminState: "up",
+					LinkSpeed:      "10000 Mb/s",
+					LinkType:       "ETH",
+					Mac:            "aa:bb:cc:dd:ee:ff",
+					Mtu:            1500,
+					TotalVfs:       2,
+					NumVfs:         0,
+				},
+			})
+
+			By("waiting for state to be in progress")
+			eventuallySyncStatusEqual(nodeState, constants.SyncStatusInProgress)
+
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("waiting to require drain")
+			err = k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)
+			Expect(err).ToNot(HaveOccurred())
+			nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{}
+			err = k8sClient.Update(ctx, nodeState)
+			Expect(err).ToNot(HaveOccurred())
+
+			EventuallyWithOffset(1, func(g Gomega) {
+				g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
+					ToNot(HaveOccurred())
+
+				g.Expect(nodeState.Annotations[constants.NodeStateDrainAnnotation]).To(Equal(constants.DrainRequired))
+			}, waitTime, retryTime).Should(Succeed())
+
+			// Validate status
+			EventuallyWithOffset(1, func(g Gomega) {
+				// get node by name
+				node := &corev1.Node{}
+				g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: nodeName}, node)).
+					ToNot(HaveOccurred())
+				g.Expect(node.Annotations[constants.NodeExternalDrainerAnnotation]).To(Equal("true"))
+			}, waitTime, retryTime).Should(Succeed())
+
+			patchAnnotation(nodeState, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete)
+			// Validate status
+			EventuallyWithOffset(1, func(g Gomega) {
+				g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
+					ToNot(HaveOccurred())
+
+				g.Expect(nodeState.Status.SyncStatus).To(Equal(constants.SyncStatusInProgress))
+				node := &corev1.Node{}
+				g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: nodeName}, node)).
+					ToNot(HaveOccurred())
+				// verify that external drainer annotation is removed
+				g.Expect(node.Annotations).NotTo(ContainElement(constants.NodeExternalDrainerAnnotation))
+			}, waitTime, retryTime).Should(Succeed())
+
+		})
 	})
 })