drain: adding 'USE_EXTERNAL_DRAINER' provoding an option to enable/disable SRIOV OP drain controller, in favor of using maintenance OP to drive node drain aspects

Signed-off-by: Ido Heyvi <[email protected]>

Author: heyvister1

README.md

@@ -464,6 +464,7 @@ nodes in parallel from the pool the operator can drain in parallel. maxUnavailab
 
 > **NOTE**: If a node is not part of any pool it will have a default configuration of maxUnavailable 1
 
+> **NOTE**: Internal drain controller can be disabled by exposing the following `USE_EXTERNAL_DRAINER` env variable. This means that drain operations will be done externally, utilizing [NVIDIA maintenance OP](https://github.com/Mellanox/maintenance-operator). In addition, `SriovNetworkPoolConfig` will not take any effect during drain procedure, since the maintenance operator will be in charge of [parallel node operations](https://github.com/Mellanox/maintenance-operator/blob/main/api/v1alpha1/maintenanceoperatorconfig_types.go#L38-L46).
 
 #### RDMA Mode Configuration
 

deployment/sriov-network-operator-chart/templates/operator.yaml

@@ -78,6 +78,8 @@ spec:
               value: {{ .Values.operator.metricsExporter.certificates.secretName }}
             - name: METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE
               value: {{ .Values.images.metricsExporterKubeRbacProxy }}
+            - name: USE_EXTERNAL_DRAINER
+              value: {{ .Values.operator.externalDrainer.enabled | quote }}
             {{- if .Values.operator.metricsExporter.prometheusOperator.enabled }}
             - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED
               value: {{ .Values.operator.metricsExporter.prometheusOperator.enabled | quote}}

deployment/sriov-network-operator-chart/values.yaml

@@ -40,6 +40,9 @@ operator:
       serviceAccount: "prometheus-k8s"
       namespace: "monitoring"
       deployRules: false
+  # use external drain controller, utilizing NVIDIA maintenance operator
+  externalDrainer:
+    enabled: false
   admissionControllers:
     enabled: false
     networkPolicy:

doc/design/parallel-node-config.md

@@ -60,6 +60,8 @@ Node annotation, `sriovnetwork.openshift.io/state` and SriovNetworkNodeState ann
 
 *NOTE:* In the future we are going to drop the node annotation and only use the SriovNetworkNodeState
 
+*NOTE:* Internal drain controller can be disabled by exposing the following `USE_EXTERNAL_DRAINER` env variable. This means that drain operations will be done externally, utilizing [NVIDIA maintenance OP](https://github.com/Mellanox/maintenance-operator). In addition, `SriovNetworkPoolConfig` will not take any effect during drain procedure, since the maintenance operator will be in charge of parallel node operations.
+
 Draining procedure:
 
 1. config daemon mark the node as `Drain_Required` or `Reboot_Required` by adding that to both the Node annotation, `sriovnetwork.openshift.io/state`

main.go

@@ -31,6 +31,7 @@ import (
 	// to ensure that exec-entrypoint and run can make use of them.
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -207,33 +208,8 @@ func main() {
 		os.Exit(1)
 	}
 
-	// we need a client that doesn't use the local cache for the objects
-	drainKClient, err := client.New(restConfig, client.Options{
-		Scheme: scheme,
-		Cache: &client.CacheOptions{
-			DisableFor: []client.Object{
-				&sriovnetworkv1.SriovNetworkNodeState{},
-				&corev1.Node{},
-				&mcfgv1.MachineConfigPool{},
-			},
-		},
-	})
-	if err != nil {
-		setupLog.Error(err, "unable to create drain kubernetes client")
-		os.Exit(1)
-	}
-
-	drainController, err := controllers.NewDrainReconcileController(drainKClient,
-		mgr.GetScheme(),
-		mgr.GetEventRecorderFor("SR-IOV operator"),
-		platformsHelper)
-	if err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "DrainReconcile")
-		os.Exit(1)
-	}
-
-	if err = drainController.SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to setup controller with manager", "controller", "DrainReconcile")
+	if err := setupDrainController(mgr, restConfig, platformsHelper, scheme); err != nil {
+		setupLog.Error(err, "unable to setup drain controller")
 		os.Exit(1)
 	}
 	// +kubebuilder:scaffold:builder
@@ -295,6 +271,46 @@ func main() {
 	}
 }
 
+func setupDrainController(mgr ctrl.Manager, restConfig *rest.Config,
+	platformsHelper platforms.Interface, scheme *runtime.Scheme) error {
+	if vars.UseExternalDrainer {
+		setupLog.Info("internal drain controller is disabled, draining will be done externally by the maintenance operator")
+		return nil
+	}
+
+	// we need a client that doesn't use the local cache for the objects
+	drainKClient, err := client.New(restConfig, client.Options{
+		Scheme: scheme,
+		Cache: &client.CacheOptions{
+			DisableFor: []client.Object{
+				&sriovnetworkv1.SriovNetworkNodeState{},
+				&corev1.Node{},
+				&mcfgv1.MachineConfigPool{},
+			},
+		},
+	})
+	if err != nil {
+		setupLog.Error(err, "unable to create drain kubernetes client")
+		return err
+	}
+
+	drainController, err := controllers.NewDrainReconcileController(drainKClient,
+		mgr.GetScheme(),
+		mgr.GetEventRecorderFor("SR-IOV operator"),
+		platformsHelper)
+	if err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "DrainReconcile")
+		return err
+	}
+
+	if err = drainController.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to setup controller with manager", "controller", "DrainReconcile")
+		return err
+	}
+
+	return nil
+}
+
 func initNicIDMap() error {
 	kubeclient := kubernetes.NewForConfigOrDie(ctrl.GetConfigOrDie())
 	if err := sriovnetworkv1.InitNicIDMapFromConfigMap(kubeclient, vars.Namespace); err != nil {

pkg/vars/vars.go

@@ -79,6 +79,10 @@ var (
 
 	// FeatureGates interface to interact with feature gates
 	FeatureGate featuregate.FeatureGate
+
+	// UseExternalDrainer controls if SRIOV operator will use an external drainer
+	// for draining nodes or its internal drain controller (default)
+	UseExternalDrainer bool
 )
 
 func init() {
@@ -101,4 +105,6 @@ func init() {
 	ResourcePrefix = os.Getenv("RESOURCE_PREFIX")
 
 	FeatureGate = featuregate.New()
+
+	UseExternalDrainer = os.Getenv("USE_EXTERNAL_DRAINER") == "true"
 }