Add in-place updates support for machine controller

Signed-off-by: Alexandr Demicev <[email protected]>

Author: alexander-demicev

api/core/v1beta2/common_types.go

@@ -181,6 +181,11 @@ const (
 	// This annotation can be used to inform MachinePool status during in-progress scaling scenarios.
 	ReplicasManagedByAnnotation = "cluster.x-k8s.io/replicas-managed-by"
 
+	// InPlaceUpdateInProgressAnnotation is set on Machine, InfraMachine, and BootstrapConfig when an in-place update is in progress.
+	// The Machine controller waits for all three objects to have this annotation before starting the update.
+	// The Machine controller removes this annotation when the update is complete.
+	InPlaceUpdateInProgressAnnotation = "cluster.x-k8s.io/in-place-update-in-progress"
+
 	// AutoscalerMinSizeAnnotation defines the minimum node group size.
 	// The annotation is used by autoscaler.
 	// The annotation is copied from kubernetes/autoscaler.

api/core/v1beta2/machine_types.go

@@ -153,6 +153,31 @@ const (
 	MachineNotUpToDateReason = "NotUpToDate"
 )
 
+// Machine's Updating condition and corresponding reasons.
+// Note: Updating condition is set by the Machine controller during in-place updates.
+const (
+	// MachineUpdatingCondition is true while an in-place update is in progress on the Machine.
+	// The condition is owned by the Machine controller and is used to track the progress of in-place updates.
+	// This condition is considered when computing the UpToDate condition.
+	MachineUpdatingCondition = "Updating"
+
+	// MachineNotUpdatingReason surfaces when the Machine is not performing an in-place update.
+	MachineNotUpdatingReason = "NotUpdating"
+
+	// MachineWaitingForInPlaceUpdateAnnotationsReason surfaces when the Machine is waiting for
+	// InfraMachine and BootstrapConfig to be annotated for in-place update.
+	MachineWaitingForInPlaceUpdateAnnotationsReason = "WaitingForInPlaceUpdateAnnotations"
+
+	// MachineWaitingForUpdateMachineHookReason surfaces when the Machine is waiting for the UpdateMachine hook to complete.
+	MachineWaitingForUpdateMachineHookReason = "WaitingForUpdateMachineHook"
+
+	// MachineUpdateFailedReason surfaces when the in-place update has failed.
+	MachineUpdateFailedReason = "UpdateFailed"
+
+	// MachineUpdatingInternalErrorReason surfaces unexpected failures during in-place update.
+	MachineUpdatingInternalErrorReason = InternalErrorReason
+)
+
 // Machine's BootstrapConfigReady condition and corresponding reasons.
 // Note: when possible, BootstrapConfigReady condition will use reasons surfaced from the underlying bootstrap config object.
 const (

controllers/alias.go

@@ -72,9 +72,10 @@ func (r *ClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manag
 
 // MachineReconciler reconciles a Machine object.
 type MachineReconciler struct {
-	Client       client.Client
-	APIReader    client.Reader
-	ClusterCache clustercache.ClusterCache
+	Client        client.Client
+	APIReader     client.Reader
+	ClusterCache  clustercache.ClusterCache
+	RuntimeClient runtimeclient.Client
 
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
@@ -90,6 +91,7 @@ func (r *MachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manag
 		Client:                           r.Client,
 		APIReader:                        r.APIReader,
 		ClusterCache:                     r.ClusterCache,
+		RuntimeClient:                    r.RuntimeClient,
 		WatchFilterValue:                 r.WatchFilterValue,
 		RemoteConditionsGracePeriod:      r.RemoteConditionsGracePeriod,
 		AdditionalSyncMachineLabels:      r.AdditionalSyncMachineLabels,

internal/controllers/machine/machine_controller.go

@@ -52,6 +52,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/clustercache"
 	"sigs.k8s.io/cluster-api/controllers/external"
 	"sigs.k8s.io/cluster-api/controllers/noderefutil"
+	runtimeclient "sigs.k8s.io/cluster-api/exp/runtime/client"
 	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/internal/contract"
 	"sigs.k8s.io/cluster-api/internal/controllers/machine/drain"
@@ -93,9 +94,10 @@ var (
 
 // Reconciler reconciles a Machine object.
 type Reconciler struct {
-	Client       client.Client
-	APIReader    client.Reader
-	ClusterCache clustercache.ClusterCache
+	Client        client.Client
+	APIReader     client.Reader
+	ClusterCache  clustercache.ClusterCache
+	RuntimeClient runtimeclient.Client
 
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
@@ -129,6 +131,9 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 		// to have some buffer.
 		return errors.New("Client, APIReader and ClusterCache must not be nil and RemoteConditionsGracePeriod must not be < 2m")
 	}
+	if feature.Gates.Enabled(feature.InPlaceUpdates) && r.RuntimeClient == nil {
+		return errors.New("RuntimeClient must not be nil when InPlaceUpdates feature gate is enabled")
+	}
 
 	r.predicateLog = ptr.To(ctrl.LoggerFrom(ctx).WithValues("controller", "machine"))
 	clusterToMachines, err := util.ClusterToTypedObjectsMapper(mgr.GetClient(), &clusterv1.MachineList{}, mgr.GetScheme())
@@ -282,7 +287,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
 	}
 
 	// Handle normal reconciliation loop.
-	return doReconcile(ctx, alwaysReconcile, s)
+	reconcileNormal := append(
+		alwaysReconcile,
+		r.reconcileInPlaceUpdate,
+	)
+
+	return doReconcile(ctx, reconcileNormal, s)
 }
 
 func patchMachine(ctx context.Context, patchHelper *patch.Helper, machine *clusterv1.Machine, options ...patch.Option) error {
@@ -326,6 +336,7 @@ func patchMachine(ctx context.Context, patchHelper *patch.Helper, machine *clust
 			clusterv1.MachineNodeReadyCondition,
 			clusterv1.MachineNodeHealthyCondition,
 			clusterv1.MachineDeletingCondition,
+			clusterv1.MachineUpdatingCondition,
 		}},
 	)
 

internal/controllers/machine/machine_controller_inplace_update.go

@@ -0,0 +1,279 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package machine
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1"
+	"sigs.k8s.io/cluster-api/feature"
+	"sigs.k8s.io/cluster-api/internal/hooks"
+	"sigs.k8s.io/cluster-api/util/conditions"
+	"sigs.k8s.io/cluster-api/util/patch"
+)
+
+// reconcileInPlaceUpdate handles the in-place update workflow for a Machine.
+func (r *Reconciler) reconcileInPlaceUpdate(ctx context.Context, s *scope) (ctrl.Result, error) {
+	if !feature.Gates.Enabled(feature.InPlaceUpdates) {
+		return ctrl.Result{}, nil
+	}
+
+	log := ctrl.LoggerFrom(ctx)
+
+	machineAnnotations := s.machine.GetAnnotations()
+	_, inPlaceUpdateInProgress := machineAnnotations[clusterv1.InPlaceUpdateInProgressAnnotation]
+	hasUpdateMachinePending := hooks.IsPending(runtimehooksv1.UpdateMachine, s.machine)
+
+	if !inPlaceUpdateInProgress {
+		// Clean up any orphaned pending hooks before exiting.
+		if hasUpdateMachinePending {
+			log.Info("In-place update annotation removed but UpdateMachine hook still pending, cleaning up orphaned hook")
+			if err := hooks.MarkAsDone(ctx, r.Client, s.machine, runtimehooksv1.UpdateMachine); err != nil { // this patches the machine, should we defer that patch until the end of reconciliation?
+				return ctrl.Result{}, errors.Wrap(err, "failed to clean up orphaned UpdateMachine hook")
+			}
+		}
+
+		conditions.Set(s.machine, metav1.Condition{
+			Type:   clusterv1.MachineUpdatingCondition,
+			Status: metav1.ConditionFalse,
+			Reason: clusterv1.MachineNotUpdatingReason,
+		})
+		return ctrl.Result{}, nil
+	}
+
+	infraReady, infraErr := r.isInfraMachineReadyForUpdate(s)
+	if infraErr != nil {
+		return ctrl.Result{}, errors.Wrap(infraErr, "failed to check if InfraMachine is ready for in-place update")
+	}
+
+	bootstrapReady, bootstrapErr := r.isBootstrapConfigReadyForUpdate(s)
+	if bootstrapErr != nil {
+		return ctrl.Result{}, errors.Wrap(bootstrapErr, "failed to check if BootstrapConfig is ready for in-place update")
+	}
+
+	if !infraReady || !bootstrapReady {
+		log.Info("Waiting for InfraMachine and BootstrapConfig to be marked for in-place update")
+		conditions.Set(s.machine, metav1.Condition{
+			Type:    clusterv1.MachineUpdatingCondition,
+			Status:  metav1.ConditionFalse,
+			Reason:  clusterv1.MachineWaitingForInPlaceUpdateAnnotationsReason,
+			Message: "Waiting for InfraMachine and BootstrapConfig to be marked for update",
+		})
+		return ctrl.Result{}, nil
+	}
+
+	if hasUpdateMachinePending {
+		log.Info("UpdateMachine hook is pending, calling runtime hook")
+		result, err := r.callUpdateMachineHook(ctx, s)
+		if err != nil {
+			conditions.Set(s.machine, metav1.Condition{
+				Type:    clusterv1.MachineUpdatingCondition,
+				Status:  metav1.ConditionFalse,
+				Reason:  clusterv1.MachineUpdateFailedReason,
+				Message: fmt.Sprintf("UpdateMachine hook failed: %v", err),
+			})
+			return ctrl.Result{}, err
+		}
+
+		if result.RequeueAfter > 0 {
+			conditions.Set(s.machine, metav1.Condition{
+				Type:    clusterv1.MachineUpdatingCondition,
+				Status:  metav1.ConditionTrue,
+				Reason:  clusterv1.MachineWaitingForUpdateMachineHookReason,
+				Message: "UpdateMachine hook in progress",
+			})
+			return result, nil
+		}
+
+		if err := hooks.MarkAsDone(ctx, r.Client, s.machine, runtimehooksv1.UpdateMachine); err != nil { // this patches the machine, should we defer that patch until the end of reconciliation?
+			return ctrl.Result{}, errors.Wrap(err, "failed to mark UpdateMachine hook as done")
+		}
+
+		log.Info("In-place update completed successfully")
+		if err := r.completeInPlaceUpdate(ctx, s); err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to complete in-place update")
+		}
+
+		conditions.Set(s.machine, metav1.Condition{
+			Type:   clusterv1.MachineUpdatingCondition,
+			Status: metav1.ConditionFalse,
+			Reason: clusterv1.MachineNotUpdatingReason,
+		})
+
+		return ctrl.Result{}, nil
+	}
+
+	// If we reach here, annotations are set but hook is not pending.
+	// This means we're waiting for the owner controller to mark the hook as pending.
+	log.Info("In-place update annotations are set, waiting for UpdateMachine hook to be marked as pending")
+	conditions.Set(s.machine, metav1.Condition{
+		Type:    clusterv1.MachineUpdatingCondition,
+		Status:  metav1.ConditionFalse,
+		Reason:  clusterv1.MachineWaitingForInPlaceUpdateAnnotationsReason,
+		Message: "Waiting for UpdateMachine hook to be marked as pending",
+	})
+
+	return ctrl.Result{}, nil
+}
+
+// isInfraMachineReadyForUpdate checks if the InfraMachine has the in-place update annotation.
+func (r *Reconciler) isInfraMachineReadyForUpdate(s *scope) (bool, error) {
+	if s.infraMachine == nil {
+		return false, nil
+	}
+	infraMachineAnnotations := s.infraMachine.GetAnnotations()
+	if infraMachineAnnotations == nil {
+		return false, nil
+	}
+	_, hasAnnotation := infraMachineAnnotations[clusterv1.InPlaceUpdateInProgressAnnotation]
+	return hasAnnotation, nil
+}
+
+// isBootstrapConfigReadyForUpdate checks if the BootstrapConfig has the in-place update annotation.
+func (r *Reconciler) isBootstrapConfigReadyForUpdate(s *scope) (bool, error) {
+	if s.bootstrapConfig == nil {
+		return true, nil
+	}
+	bootstrapConfigAnnotations := s.bootstrapConfig.GetAnnotations()
+	if bootstrapConfigAnnotations == nil {
+		return false, nil
+	}
+	_, hasAnnotation := bootstrapConfigAnnotations[clusterv1.InPlaceUpdateInProgressAnnotation]
+	return hasAnnotation, nil
+}
+
+// callUpdateMachineHook calls the UpdateMachine runtime hook for the machine.
+func (r *Reconciler) callUpdateMachineHook(ctx context.Context, s *scope) (ctrl.Result, error) {
+	log := ctrl.LoggerFrom(ctx)
+
+	request := &runtimehooksv1.UpdateMachineRequest{}
+	request.Desired.Machine = *s.machine.DeepCopy()
+
+	if s.infraMachine != nil { // should it return an error if infraMachine is nil?
+		infraMachineRaw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(s.infraMachine)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to convert InfraMachine to unstructured")
+		}
+		request.Desired.InfrastructureMachine.Raw, err = json.Marshal(infraMachineRaw)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to marshal InfraMachine")
+		}
+	}
+
+	if s.bootstrapConfig != nil { // should it return an error if bootstrapConfig is nil?
+		bootstrapConfigRaw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(s.bootstrapConfig)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to convert BootstrapConfig to unstructured")
+		}
+		request.Desired.BootstrapConfig.Raw, err = json.Marshal(bootstrapConfigRaw)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to marshal BootstrapConfig")
+		}
+	}
+
+	response := &runtimehooksv1.UpdateMachineResponse{}
+
+	if err := r.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.UpdateMachine, s.machine, request, response); err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to call UpdateMachine hook")
+	}
+
+	if response.GetRetryAfterSeconds() != 0 {
+		log.Info(fmt.Sprintf("UpdateMachine hook requested retry after %d seconds", response.GetRetryAfterSeconds()))
+		return ctrl.Result{RequeueAfter: time.Duration(response.GetRetryAfterSeconds()) * time.Second}, nil
+	}
+
+	log.Info("UpdateMachine hook completed successfully")
+	return ctrl.Result{}, nil
+}
+
+// completeInPlaceUpdate removes in-place update annotations from InfraMachine, BootstrapConfig and Machine.
+func (r *Reconciler) completeInPlaceUpdate(ctx context.Context, s *scope) error {
+	log := ctrl.LoggerFrom(ctx)
+
+	if s.infraMachine != nil {
+		if err := r.removeInPlaceUpdateAnnotation(ctx, s.infraMachine); err != nil {
+			return errors.Wrap(err, "failed to remove in-place update annotation from InfraMachine")
+		}
+	}
+
+	if s.bootstrapConfig != nil {
+		if err := r.removeInPlaceUpdateAnnotation(ctx, s.bootstrapConfig); err != nil {
+			return errors.Wrap(err, "failed to remove in-place update annotation from BootstrapConfig")
+		}
+	}
+
+	// Only remove from Machine if all child object patches succeeded.
+	if err := r.removeInPlaceUpdateAnnotationFromMachine(s.machine); err != nil {
+		return errors.Wrap(err, "failed to remove in-place update annotation from Machine")
+	}
+
+	log.Info("Removed in-place update annotations from all objects")
+	return nil
+}
+
+// removeInPlaceUpdateAnnotationFromMachine removes the in-place update annotation from the Machine.
+func (r *Reconciler) removeInPlaceUpdateAnnotationFromMachine(machine *clusterv1.Machine) error {
+	annotations := machine.GetAnnotations()
+	if annotations == nil {
+		return nil
+	}
+
+	if _, exists := annotations[clusterv1.InPlaceUpdateInProgressAnnotation]; !exists {
+		return nil
+	}
+
+	delete(annotations, clusterv1.InPlaceUpdateInProgressAnnotation)
+	machine.SetAnnotations(annotations)
+
+	return nil
+}
+
+// removeInPlaceUpdateAnnotation removes the in-place update annotation from an object and patches it.
+func (r *Reconciler) removeInPlaceUpdateAnnotation(ctx context.Context, obj client.Object) error {
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		return nil
+	}
+
+	if _, exists := annotations[clusterv1.InPlaceUpdateInProgressAnnotation]; !exists {
+		return nil
+	}
+
+	patchHelper, err := patch.NewHelper(obj, r.Client)
+	if err != nil {
+		return errors.Wrap(err, "failed to create patch helper")
+	}
+
+	delete(annotations, clusterv1.InPlaceUpdateInProgressAnnotation)
+	obj.SetAnnotations(annotations)
+
+	if err := patchHelper.Patch(ctx, obj); err != nil {
+		return errors.Wrap(err, "failed to patch object")
+	}
+
+	return nil
+}