Merge pull request #4293 from Nordix/kcp-rollout-strategy-backport

✨ Backport KCP Rollout Strategy to v1alpha3 api

Author: k8s-ci-robot

controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go

@@ -19,12 +19,21 @@ package v1alpha3
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 
 	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/errors"
 )
 
+type RolloutStrategyType string
+
+const (
+	// Replace the old control planes by new one using rolling update
+	// i.e. gradually scale up or down the old control planes and scale up or down the new one.
+	RollingUpdateStrategyType RolloutStrategyType = "RollingUpdate"
+)
+
 const (
 	KubeadmControlPlaneFinalizer = "kubeadm.controlplane.cluster.x-k8s.io"
 
@@ -72,6 +81,38 @@ type KubeadmControlPlaneSpec struct {
 	// NOTE: NodeDrainTimeout is different from `kubectl drain --timeout`
 	// +optional
 	NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"`
+
+	// The RolloutStrategy to use to replace control plane machines with
+	// new ones.
+	// +optional
+	RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"`
+}
+
+// RolloutStrategy describes how to replace existing machines
+// with new ones.
+type RolloutStrategy struct {
+	// Type of rollout. Currently the only supported strategy is
+	// "RollingUpdate".
+	// Default is RollingUpdate.
+	// +optional
+	Type RolloutStrategyType `json:"type,omitempty"`
+
+	// Rolling update config params. Present only if
+	// RolloutStrategyType = RollingUpdate.
+	// +optional
+	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+}
+
+// RollingUpdate is used to control the desired behavior of rolling update.
+type RollingUpdate struct {
+	// The maximum number of control planes that can be scheduled above or under the
+	// desired number of control planes.
+	// Value can be an absolute number 1 or 0.
+	// Defaults to 1.
+	// Example: when this is set to 1, the control plane can be scaled
+	// up immediately when the rolling update starts.
+	// +optional
+	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`
 }
 
 // KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane.

controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pkg/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	kubeadmv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1"
 	"sigs.k8s.io/cluster-api/util"
@@ -64,6 +65,25 @@ func (in *KubeadmControlPlane) Default() {
 	if !strings.HasPrefix(in.Spec.Version, "v") {
 		in.Spec.Version = "v" + in.Spec.Version
 	}
+
+	ios1 := intstr.FromInt(1)
+
+	if in.Spec.RolloutStrategy == nil {
+		in.Spec.RolloutStrategy = &RolloutStrategy{}
+	}
+
+	// Enforce RollingUpdate strategy and default MaxSurge if not set.
+	if in.Spec.RolloutStrategy != nil {
+		if len(in.Spec.RolloutStrategy.Type) == 0 {
+			in.Spec.RolloutStrategy.Type = RollingUpdateStrategyType
+		}
+		if in.Spec.RolloutStrategy.Type == RollingUpdateStrategyType {
+			if in.Spec.RolloutStrategy.RollingUpdate == nil {
+				in.Spec.RolloutStrategy.RollingUpdate = &RollingUpdate{}
+			}
+			in.Spec.RolloutStrategy.RollingUpdate.MaxSurge = intstr.ValueOrDefault(in.Spec.RolloutStrategy.RollingUpdate.MaxSurge, ios1)
+		}
+	}
 }
 
 // ValidateCreate implements webhook.Validator so a webhook will be registered for the type
@@ -119,6 +139,7 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error {
 		{spec, "version"},
 		{spec, "upgradeAfter"},
 		{spec, "nodeDrainTimeout"},
+		{spec, "rolloutStrategy"},
 	}
 
 	allErrs := in.validateCommon()
@@ -272,6 +293,41 @@ func (in *KubeadmControlPlane) validateCommon() (allErrs field.ErrorList) {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), in.Spec.Version, "must be a valid semantic version"))
 	}
 
+	if in.Spec.RolloutStrategy != nil {
+		if in.Spec.RolloutStrategy.Type != RollingUpdateStrategyType {
+			allErrs = append(
+				allErrs,
+				field.Required(
+					field.NewPath("spec", "rolloutStrategy", "type"),
+					"only RollingUpdateStrategyType is supported",
+				),
+			)
+		}
+
+		ios1 := intstr.FromInt(1)
+		ios0 := intstr.FromInt(0)
+
+		if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge == ios0 && *in.Spec.Replicas < int32(3) {
+			allErrs = append(
+				allErrs,
+				field.Required(
+					field.NewPath("spec", "rolloutStrategy", "rollingUpdate"),
+					"when KubeadmControlPlane is configured to scale-in, replica count needs to be at least 3",
+				),
+			)
+		}
+
+		if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios1 && *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios0 {
+			allErrs = append(
+				allErrs,
+				field.Required(
+					field.NewPath("spec", "rolloutStrategy", "rollingUpdate", "maxSurge"),
+					"value must be 1 or 0",
+				),
+			)
+		}
+	}
+
 	allErrs = append(allErrs, in.validateCoreDNSImage()...)
 
 	return allErrs

controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook_test.go

@@ -24,6 +24,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
 	kubeadmv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1"
@@ -39,12 +40,15 @@ func TestKubeadmControlPlaneDefault(t *testing.T) {
 		Spec: KubeadmControlPlaneSpec{
 			InfrastructureTemplate: corev1.ObjectReference{},
 			Version:                "1.18.3",
+			RolloutStrategy:        &RolloutStrategy{},
 		},
 	}
 	kcp.Default()
 
 	g.Expect(kcp.Spec.InfrastructureTemplate.Namespace).To(Equal(kcp.Namespace))
 	g.Expect(kcp.Spec.Version).To(Equal("v1.18.3"))
+	g.Expect(kcp.Spec.RolloutStrategy.Type).To(Equal(RollingUpdateStrategyType))
+	g.Expect(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal).To(Equal(int32(1)))
 }
 
 func TestKubeadmControlPlaneValidateCreate(t *testing.T) {
@@ -60,11 +64,22 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) {
 			},
 			Replicas: pointer.Int32Ptr(1),
 			Version:  "v1.19.0",
+			RolloutStrategy: &RolloutStrategy{
+				Type: RollingUpdateStrategyType,
+				RollingUpdate: &RollingUpdate{
+					MaxSurge: &intstr.IntOrString{
+						IntVal: 1,
+					},
+				},
+			},
 		},
 	}
 	invalidNamespace := valid.DeepCopy()
 	invalidNamespace.Spec.InfrastructureTemplate.Namespace = "bar"
 
+	invalidMaxSurge := valid.DeepCopy()
+	invalidMaxSurge.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal = int32(3)
+
 	missingReplicas := valid.DeepCopy()
 	missingReplicas.Spec.Replicas = nil
 
@@ -142,6 +157,11 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) {
 			expectErr: true,
 			kcp:       invalidVersion1,
 		},
+		{
+			name:      "should return error when maxSurge is not 1",
+			expectErr: true,
+			kcp:       invalidMaxSurge,
+		},
 	}
 
 	for _, tt := range tests {

controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go

@@ -1045,6 +1045,30 @@ spec:
                   This is a pointer to distinguish between explicit zero and not specified.
                 format: int32
                 type: integer
+              rolloutStrategy:
+                description: The RolloutStrategy to use to replace control plane machines
+                  with new ones.
+                properties:
+                  rollingUpdate:
+                    description: Rolling update config params. Present only if RolloutStrategyType
+                      = RollingUpdate.
+                    properties:
+                      maxSurge:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        description: 'The maximum number of control planes that can
+                          be scheduled above or under the desired number of control
+                          planes. Value can be an absolute number 1 or 0. Defaults
+                          to 1. Example: when this is set to 1, the control plane
+                          can be scaled up immediately when the rolling update starts.'
+                        x-kubernetes-int-or-string: true
+                    type: object
+                  type:
+                    description: Type of rollout. Currently the only supported strategy
+                      is "RollingUpdate". Default is RollingUpdate.
+                    type: string
+                type: object
               upgradeAfter:
                 description: UpgradeAfter is a field to indicate an upgrade should
                   be performed after the specified time even if no changes have been

controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/klogr"
@@ -1345,6 +1346,14 @@ func createClusterWithControlPlane() (*clusterv1.Cluster, *controlplanev1.Kubead
 			},
 			Replicas: pointer.Int32Ptr(int32(3)),
 			Version:  "v1.16.6",
+			RolloutStrategy: &controlplanev1.RolloutStrategy{
+				Type: "RollingUpdate",
+				RollingUpdate: &controlplanev1.RollingUpdate{
+					MaxSurge: &intstr.IntOrString{
+						IntVal: 1,
+					},
+				},
+			},
 		},
 	}
 

controlplane/kubeadm/controllers/controller_test.go

@@ -100,14 +100,17 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
 		return ctrl.Result{}, errors.Wrap(err, "failed to upgrade kubelet config map")
 	}
 
-	status, err := workloadCluster.ClusterStatus(ctx)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-
-	if status.Nodes <= *kcp.Spec.Replicas {
-		// scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs
-		return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane)
+	switch kcp.Spec.RolloutStrategy.Type {
+	case controlplanev1.RollingUpdateStrategyType:
+		// RolloutStrategy is currently defaulted and validated to be RollingUpdate
+		// We can ignore MaxUnavailable because we are enforcing health checks before we get here.
+		maxNodes := *kcp.Spec.Replicas + int32(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntValue())
+		if int32(controlPlane.Machines.Len()) < maxNodes {
+			// scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs
+			return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane)
+		}
+		return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade)
+	default:
+		return ctrl.Result{}, errors.New("rolloutStrategy type is not set to rollingupdatestrategytype, unable to determine the strategy for rolling out machines")
 	}
-	return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade)
 }