Skip to content

✨ Add in-place to rollout planner #12865

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 21, 2025
Merged

✨ Add in-place to rollout planner #12865

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f0c0f2f
Add in-place to rollout planner
fabriziopandini Oct 15, 2025
17a97eb
Address comments
fabriziopandini Oct 20, 2025
e1cdf66
More comments
fabriziopandini Oct 20, 2025
0a0e997
More comments
fabriziopandini Oct 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions api/core/v1beta2/machine_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,16 @@ const (

// ManagedNodeAnnotationDomain is one of the CAPI managed Node annotation domains.
ManagedNodeAnnotationDomain = "node.cluster.x-k8s.io"

// MachinePendingAcknowledgeMoveAnnotationName is an internal annotation added by the MS controller to a machine when being
// moved from the oldMS to the newMS. The annotation is removed as soon as the MS controller get the acknowledgment about the
// replica being accounted from the corresponding MD.
// Note: the annotation is added when reconciling the oldMS, and it is removed when reconciling the newMS.
MachinePendingAcknowledgeMoveAnnotationName = "in-place-updates.internal.cluster.x-k8s.io/pending-acknowledge-move"

// MachineUpdatingInPlaceAnnotationName is an internal annotation added to machines by the controller owning the Machine when in-place update
// is started, e.g. by the MachineSet controller; the annotation will be removed by the Machine controller when in-place update is completed.
MachineUpdatingInPlaceAnnotationName = "in-place-updates.internal.cluster.x-k8s.io/update-in-progress"
)

// Machine's Available condition and corresponding reasons.
Expand Down
21 changes: 21 additions & 0 deletions api/core/v1beta2/machineset_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,27 @@ const (
// MachineSetFinalizer is the finalizer used by the MachineSet controller to
// ensure ordered cleanup of corresponding Machines when a Machineset is being deleted.
MachineSetFinalizer = "cluster.x-k8s.io/machineset"

// MachineSetMoveMachinesToAnnotationName is an internal annotation added by the MD controller to the oldMS
// when it should scale down by moving machines that can be updated in-place to the newMS instead of deleting them.
// Note: This annotation is used to perform a two-ways check before moving a machine from oldMS to newMS:
//
// "oldMS must have: move to newMS" and "newMS must have: receive replicas from oldMS"
MachineSetMoveMachinesToAnnotationName = "in-place-updates.internal.cluster.x-k8s.io/move-machines-to-machineset"

// MachineSetReceiveMachinesFromAnnotationName is an internal annotation added by the MD controller to the newMS
// when it should receive replicas from an oldMS as a first step of an in-place upgrade operation.
// Note: This annotation is used to perform a two-ways check before moving a machine from oldMS to newMS:
//
// "oldMS must have: move to newMS" and "newMS must have: receive replicas from oldMS"
MachineSetReceiveMachinesFromAnnotationName = "in-place-updates.internal.cluster.x-k8s.io/receive-machines-from-machinesets"

// MachineSetAcknowledgeMoveAnnotationName is an internal annotation with a list of machines added by the MD controller
// to a MachineSet when it acknowledges a machine pending acknowled after being moved from an oldMS.
// A machine is dropped from this annotation as soon as pending-acknowledge-move is removed from the machine;
// the annotation is dropped when empty.
// Note: this annotation is used in pair with replicasMachineDeploymentAcknowledgeAnnotation and replicaPendingMachineDeploymentAcknowledgeAnnotation.
MachineSetAcknowledgeMoveAnnotationName = "in-place-updates.internal.cluster.x-k8s.io/acknowledge-move"
)

// MachineSetSpec defines the desired state of MachineSet.
Expand Down
17 changes: 13 additions & 4 deletions internal/controllers/machinedeployment/machinedeployment_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (
"sigs.k8s.io/cluster-api/controllers/external"
"sigs.k8s.io/cluster-api/internal/util/ssa"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/collections"
v1beta1conditions "sigs.k8s.io/cluster-api/util/conditions/deprecated/v1beta1"
"sigs.k8s.io/cluster-api/util/finalizers"
clog "sigs.k8s.io/cluster-api/util/log"
Expand Down Expand Up @@ -163,6 +164,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (retres ct
cluster: cluster,
}

if selectorMap, err := metav1.LabelSelectorAsMap(&s.machineDeployment.Spec.Selector); err == nil {
machineList := &clusterv1.MachineList{}
if err := r.Client.List(ctx, machineList, client.InNamespace(s.machineDeployment.Namespace), client.MatchingLabels(selectorMap)); err == nil {
s.machines = collections.FromMachineList(machineList)
}
}

defer func() {
if err := r.updateStatus(ctx, s); err != nil {
reterr = kerrors.NewAggregate([]error{reterr, err})
Expand Down Expand Up @@ -195,6 +203,7 @@ type scope struct {
machineDeployment *clusterv1.MachineDeployment
cluster *clusterv1.Cluster
machineSets []*clusterv1.MachineSet
machines collections.Machines
bootstrapTemplateNotFound bool
bootstrapTemplateExists bool
infrastructureTemplateNotFound bool
Expand Down Expand Up @@ -291,15 +300,15 @@ func (r *Reconciler) reconcile(ctx context.Context, s *scope) error {
templateExists := s.infrastructureTemplateExists && (!md.Spec.Template.Spec.Bootstrap.ConfigRef.IsDefined() || s.bootstrapTemplateExists)

if ptr.Deref(md.Spec.Paused, false) {
return r.sync(ctx, md, s.machineSets, templateExists)
return r.sync(ctx, md, s.machineSets, s.machines, templateExists)
}

if md.Spec.Rollout.Strategy.Type == clusterv1.RollingUpdateMachineDeploymentStrategyType {
return r.rolloutRollingUpdate(ctx, md, s.machineSets, templateExists)
return r.rolloutRollingUpdate(ctx, md, s.machineSets, s.machines, templateExists)
}

if md.Spec.Rollout.Strategy.Type == clusterv1.OnDeleteMachineDeploymentStrategyType {
return r.rolloutOnDelete(ctx, md, s.machineSets, templateExists)
return r.rolloutOnDelete(ctx, md, s.machineSets, s.machines, templateExists)
}

return errors.Errorf("unexpected deployment strategy type: %s", md.Spec.Rollout.Strategy.Type)
Expand All @@ -320,7 +329,6 @@ func (r *Reconciler) createOrUpdateMachineSetsAndSyncMachineDeploymentRevision(c
log = log.WithValues("MachineSet", klog.KObj(ms))
ctx = ctrl.LoggerInto(ctx, log)

originalReplicas := ptr.Deref(ms.Spec.Replicas, 0)
if scaleIntent, ok := p.scaleIntents[ms.Name]; ok {
ms.Spec.Replicas = &scaleIntent
}
Expand Down Expand Up @@ -363,6 +371,7 @@ func (r *Reconciler) createOrUpdateMachineSetsAndSyncMachineDeploymentRevision(c
if !ok {
return errors.Errorf("failed to update MachineSet %s, original MS is missing", klog.KObj(ms))
}
originalReplicas := ptr.Deref(originalMS.Spec.Replicas, 0)

err := ssa.Patch(ctx, r.Client, machineDeploymentManagerName, ms, ssa.WithCachingProxy{Cache: r.ssaCache, Original: originalMS})
if err != nil {
Expand Down
5 changes: 3 additions & 2 deletions internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,13 @@ import (

clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
"sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil"
"sigs.k8s.io/cluster-api/util/collections"
)

// rolloutOnDelete reconcile machine sets controlled by a MachineDeployment that is using the OnDelete strategy.
func (r *Reconciler) rolloutOnDelete(ctx context.Context, md *clusterv1.MachineDeployment, msList []*clusterv1.MachineSet, templateExists bool) error {
func (r *Reconciler) rolloutOnDelete(ctx context.Context, md *clusterv1.MachineDeployment, msList []*clusterv1.MachineSet, machines collections.Machines, templateExists bool) error {
planner := newRolloutPlanner()
if err := planner.init(ctx, md, msList, nil, true, templateExists); err != nil {
if err := planner.init(ctx, md, msList, machines.UnsortedList(), true, templateExists); err != nil {
return err
}

Expand Down
5 changes: 4 additions & 1 deletion internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/google/go-cmp/cmp"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog/v2"
"k8s.io/klog/v2/textlogger"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
Expand Down Expand Up @@ -580,13 +581,15 @@ func runOnDeleteTestCase(ctx context.Context, t *testing.T, tt onDeleteSequenceT

// Running a small subset of MD reconcile (the rollout logic and a bit of setReplicas)
p := newRolloutPlanner()
p.computeDesiredMS = func(_ context.Context, deployment *clusterv1.MachineDeployment, currentNewMS *clusterv1.MachineSet) (*clusterv1.MachineSet, error) {
p.overrideComputeDesiredMS = func(ctx context.Context, deployment *clusterv1.MachineDeployment, currentNewMS *clusterv1.MachineSet) (*clusterv1.MachineSet, error) {
log := ctrl.LoggerFrom(ctx)
desiredNewMS := currentNewMS
if currentNewMS == nil {
// uses a predictable MS name when creating newMS, also add the newMS to current.machineSets
totMS := len(current.machineSets)
desiredNewMS = createMS(fmt.Sprintf("ms%d", totMS+1), deployment.Spec.Template.Spec.FailureDomain, 0)
current.machineSets = append(current.machineSets, desiredNewMS)
log.V(5).Info(fmt.Sprintf("Computing new MachineSet %s with %d replicas", desiredNewMS.Name, ptr.Deref(desiredNewMS.Spec.Replicas, 0)), "MachineSet", klog.KObj(desiredNewMS))
}
return desiredNewMS, nil
}
Expand Down
24 changes: 18 additions & 6 deletions internal/controllers/machinedeployment/machinedeployment_rollout_planner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
Expand All @@ -45,14 +47,14 @@ type rolloutPlanner struct {
oldMSs []*clusterv1.MachineSet
upToDateResults map[string]mdutil.UpToDateResult

scaleIntents map[string]int32
computeDesiredMS func(ctx context.Context, deployment *clusterv1.MachineDeployment, currentMS *clusterv1.MachineSet) (*clusterv1.MachineSet, error)
scaleIntents map[string]int32
overrideComputeDesiredMS func(ctx context.Context, deployment *clusterv1.MachineDeployment, currentMS *clusterv1.MachineSet) (*clusterv1.MachineSet, error)
overrideCanUpdateMachineSetInPlace func(oldMS *clusterv1.MachineSet) bool
}

func newRolloutPlanner() *rolloutPlanner {
return &rolloutPlanner{
scaleIntents: make(map[string]int32),
computeDesiredMS: computeDesiredMS,
scaleIntents: make(map[string]int32),
}
}

Expand Down Expand Up @@ -137,7 +139,11 @@ func (p *rolloutPlanner) init(ctx context.Context, md *clusterv1.MachineDeployme
// Additionally, this procedure ensure the annotations tracking revisions numbers on the newMS is upToDate.
// Note: because we are using Server-Side-Apply we always have to calculate the full object.
func (p *rolloutPlanner) computeDesiredNewMS(ctx context.Context, currentNewMS *clusterv1.MachineSet) (*clusterv1.MachineSet, error) {
desiredNewMS, err := p.computeDesiredMS(ctx, p.md, currentNewMS)
computeFunc := computeDesiredMS
if p.overrideComputeDesiredMS != nil {
computeFunc = p.overrideComputeDesiredMS
}
desiredNewMS, err := computeFunc(ctx, p.md, currentNewMS)
if err != nil {
return nil, err
}
Expand All @@ -159,7 +165,11 @@ func (p *rolloutPlanner) computeDesiredNewMS(ctx context.Context, currentNewMS *
// Additionally, this procedure ensure the annotations tracking revisions numbers are carried over.
// Note: because we are using Server-Side-Apply we always have to calculate the full object.
func (p *rolloutPlanner) computeDesiredOldMS(ctx context.Context, currentOldMS *clusterv1.MachineSet) (*clusterv1.MachineSet, error) {
desiredOldMS, err := p.computeDesiredMS(ctx, p.md, currentOldMS)
computeFunc := computeDesiredMS
if p.overrideComputeDesiredMS != nil {
computeFunc = p.overrideComputeDesiredMS
}
desiredOldMS, err := computeFunc(ctx, p.md, currentOldMS)
if err != nil {
return nil, err
}
Expand All @@ -183,6 +193,7 @@ func (p *rolloutPlanner) computeDesiredOldMS(ctx context.Context, currentOldMS *
// computeDesiredMS computes the desired MachineSet, which could be either a newly created newMS, or the new desired version of an existing newMS/OldMS.
// Note: because we are using Server-Side-Apply we always have to calculate the full object.
func computeDesiredMS(ctx context.Context, deployment *clusterv1.MachineDeployment, currentMS *clusterv1.MachineSet) (*clusterv1.MachineSet, error) {
log := ctrl.LoggerFrom(ctx)
var name string
var uid types.UID
var finalizers []string
Expand Down Expand Up @@ -212,6 +223,7 @@ func computeDesiredMS(ctx context.Context, deployment *clusterv1.MachineDeployme
replicas = 0
machineTemplateSpec = *deployment.Spec.Template.Spec.DeepCopy()
creationTimestamp = metav1.NewTime(time.Now())
log.V(5).Info(fmt.Sprintf("Computing new MachineSet %s with %d replicas", name, replicas), "MachineSet", klog.KRef(deployment.Namespace, name))
} else {
// For updating an existing MachineSet use name, uid, finalizers, replicas, uniqueIdentifier and machine template spec from existingMS.
// Note: We use the uid, to ensure that the Server-Side-Apply only updates the existingMS.
Expand Down
Loading