diff --git a/apis/v1beta1/vspheremachine_types.go b/apis/v1beta1/vspheremachine_types.go index cc6d31d1aa..cbdd5ff7ac 100644 --- a/apis/v1beta1/vspheremachine_types.go +++ b/apis/v1beta1/vspheremachine_types.go @@ -81,6 +81,10 @@ const ( // Note: This reason is used only in supervisor mode. VSphereMachineVirtualMachinePoweringOnV1Beta2Reason = "PoweringOn" + // VSphereMachineVirtualMachineWaitingForVirtualMachineGroupV1Beta2Reason surfaces that the VirtualMachine + // is waiting for its corresponding VirtualMachineGroup to be created and to include this VM as a member. + VSphereMachineVirtualMachineWaitingForVirtualMachineGroupV1Beta2Reason = "WaitingForVirtualMachineGroup" + // VSphereMachineVirtualMachineWaitingForNetworkAddressV1Beta2Reason surfaces when the VirtualMachine that is controlled // by the VSphereMachine waiting for the machine network settings to be reported after machine being powered on. VSphereMachineVirtualMachineWaitingForNetworkAddressV1Beta2Reason = "WaitingForNetworkAddress" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 401dd765e5..102217c078 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -21,7 +21,7 @@ spec: - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" - --v=4 - - "--feature-gates=MultiNetworks=${EXP_MULTI_NETWORKS:=false},NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false}" + - "--feature-gates=MultiNetworks=${EXP_MULTI_NETWORKS:=false},NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},NodeAutoPlacement=${EXP_NODE_AUTO_PLACEMENT:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false}" image: controller:latest imagePullPolicy: IfNotPresent name: manager diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ff4613da71..d3963fb5bf 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -249,6 +249,8 @@ rules: - apiGroups: - vmoperator.vmware.com resources: + - virtualmachinegroups + - virtualmachinegroups/status - virtualmachineimages - virtualmachineimages/status - virtualmachines diff --git a/controllers/vmware/controllers_suite_test.go b/controllers/vmware/controllers_suite_test.go index 87d99112e0..128ee2086d 100644 --- a/controllers/vmware/controllers_suite_test.go +++ b/controllers/vmware/controllers_suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -71,6 +72,7 @@ func setup(ctx context.Context) (*helpers.TestEnvironment, clustercache.ClusterC utilruntime.Must(infrav1.AddToScheme(scheme.Scheme)) utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) utilruntime.Must(vmwarev1.AddToScheme(scheme.Scheme)) + utilruntime.Must(vmoprv1.AddToScheme(scheme.Scheme)) testEnv := helpers.NewTestEnvironment(ctx) diff --git a/controllers/vmware/virtualmachinegroup_controller.go b/controllers/vmware/virtualmachinegroup_controller.go new file mode 100644 index 0000000000..22767f12f3 --- /dev/null +++ b/controllers/vmware/virtualmachinegroup_controller.go @@ -0,0 +1,101 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "context" + + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + apitypes "k8s.io/apimachinery/pkg/types" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/util/predicates" + ctrl "sigs.k8s.io/controller-runtime" + ctrlbldr "sigs.k8s.io/controller-runtime/pkg/builder" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + capvcontext "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context" +) + +// AddVirtualMachineGroupControllerToManager adds the VirtualMachineGroup controller to the provided manager. +func AddVirtualMachineGroupControllerToManager(ctx context.Context, controllerManagerCtx *capvcontext.ControllerManagerContext, mgr manager.Manager, options controller.Options) error { + predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "virtualmachinegroup") + + reconciler := &VirtualMachineGroupReconciler{ + Client: controllerManagerCtx.Client, + Recorder: mgr.GetEventRecorderFor("virtualmachinegroup-controller"), + } + + builder := ctrl.NewControllerManagedBy(mgr). + For(&clusterv1.Cluster{}). + WithOptions(options). + // Set the controller's name explicitly to virtualmachinegroup. + Named("virtualmachinegroup"). + Watches( + &vmoprv1.VirtualMachineGroup{}, + handler.EnqueueRequestForOwner(mgr.GetScheme(), reconciler.Client.RESTMapper(), &clusterv1.Cluster{}), + ctrlbldr.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog)), + ). + Watches( + &vmwarev1.VSphereMachine{}, + handler.EnqueueRequestsFromMapFunc(reconciler.VSphereMachineToCluster), + ctrlbldr.WithPredicates( + predicate.Funcs{ + UpdateFunc: func(event.UpdateEvent) bool { return false }, + CreateFunc: func(e event.CreateEvent) bool { + // Only handle VSphereMachine which belongs to a MachineDeployment + _, found := e.Object.GetLabels()[clusterv1.MachineDeploymentNameLabel] + return found + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // Only handle VSphereMachine which belongs to a MachineDeployment + _, found := e.Object.GetLabels()[clusterv1.MachineDeploymentNameLabel] + return found + }, + GenericFunc: func(event.GenericEvent) bool { return false }, + }), + ). + WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, controllerManagerCtx.WatchFilterValue)) + + return builder.Complete(reconciler) +} + +// VSphereMachineToCluster maps VSphereMachine events to Cluster reconcile requests. +func (r *VirtualMachineGroupReconciler) VSphereMachineToCluster(_ context.Context, a ctrlclient.Object) []reconcile.Request { + vSphereMachine, ok := a.(*vmwarev1.VSphereMachine) + if !ok { + return nil + } + + clusterName, ok := vSphereMachine.Labels[clusterv1.ClusterNameLabel] + if !ok || clusterName == "" { + return nil + } + + return []reconcile.Request{{ + NamespacedName: apitypes.NamespacedName{ + Namespace: vSphereMachine.Namespace, + Name: clusterName, + }, + }} +} diff --git a/controllers/vmware/virtualmachinegroup_reconciler.go b/controllers/vmware/virtualmachinegroup_reconciler.go new file mode 100644 index 0000000000..619ddbc4ad --- /dev/null +++ b/controllers/vmware/virtualmachinegroup_reconciler.go @@ -0,0 +1,488 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package vmware contains the VirtualMachineGroup Reconciler. +package vmware + +import ( + "context" + "fmt" + "maps" + "slices" + "sort" + "strings" + + "github.com/pkg/errors" + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/conditions" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/vmoperator" +) + +// VirtualMachineGroupReconciler reconciles VirtualMachineGroup. +type VirtualMachineGroupReconciler struct { + Client client.Client + Recorder record.EventRecorder +} + +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters/status,verbs=get +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinegroups,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinegroups/status,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vspheremachines,verbs=get;list;watch +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinedeployments,verbs=get;list;watch +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;list;watch + +// This controller is introduced to coordinate the creation and maintenance of +// the VirtualMachineGroup (VMG) object with respect to the worker VSphereMachines in the Cluster. +// +// - Batch Coordination: Gating the initial creation of the VMG until for the first time all the +// MachineDeployment replicas will have a corresponding VSphereMachine. +// Once this condition is met, the VirtualMachineGroup is created considering +// the initial set of machines for the initial placement decision. +// When the VirtualMachineGroup reports the placement decision, then finally +// creation of VirtualMachines is unblocked. +// +// - Placement Persistence: Persisting the MachineDeployment-to-Zone mapping (placement decision) as a +// metadata annotation on the VMG object. The same decision must be respected also for placement +// of machines created after initial placement. +// +// - Membership Maintenance: Dynamically updating the VMG's member list to reflect the current +// state of VMs belonging to MachineDeployments (handling scale-up/down events). + +func (r *VirtualMachineGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + // Fetch the Cluster instance. + cluster := &clusterv1.Cluster{} + if err := r.Client.Get(ctx, req.NamespacedName, cluster); err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + // Note: VirtualMachineGroup is going to have same name and namespace of the cluster. + // Using cluster here, because VirtualMachineGroup is created only once we are ready. + log = log.WithValues("VirtualMachineGroup", klog.KObj(cluster)) + ctx = ctrl.LoggerInto(ctx, log) + + // If Cluster is deleted, just return as VirtualMachineGroup will be GCed and no extra processing needed. + if !cluster.DeletionTimestamp.IsZero() { + return reconcile.Result{}, nil + } + + // If ControlPlane haven't initialized, return since CAPV will only start to reconcile VSphereMachines of + // MachineDeployment after ControlPlane is initialized. + if !conditions.IsTrue(cluster, clusterv1.ClusterControlPlaneInitializedCondition) { + return reconcile.Result{}, nil + } + + return r.reconcileNormal(ctx, cluster) +} + +func (r *VirtualMachineGroupReconciler) reconcileNormal(ctx context.Context, cluster *clusterv1.Cluster) (reconcile.Result, error) { + log := ctrl.LoggerFrom(ctx) + + // Get all the data required for computing the desired VMG. + currentVMG, err := r.getVirtualMachineGroup(ctx, cluster) + if err != nil { + return reconcile.Result{}, err + } + vSphereMachines, err := r.getVSphereMachines(ctx, cluster) + if err != nil { + return reconcile.Result{}, err + } + machineDeployments, err := r.getMachineDeployments(ctx, cluster) + if err != nil { + return reconcile.Result{}, err + } + + // Before initial placement VirtualMachineGroup does not exist yet. + if currentVMG == nil { + // VirtualMachineGroup creation starts the initial placement process that should take care + // of spreading VSphereMachines across failure domains in an ideal way / according to user intent. + // The initial placement can be performed only when all the VSphereMachines to be considered for the + // placement decision exist. If this condition is not met, return (watches will trigger new + // reconciles whenever new VSphereMachines are created). + // Note: In case there are no MachineDeployments, or all the MachineDeployments have zero replicas, + // no placement decision is required, and thus no VirtualMachineGroup will be created. + if !shouldCreateVirtualMachineGroup(ctx, machineDeployments, vSphereMachines) { + return reconcile.Result{}, nil + } + + // Computes the new VirtualMachineGroup including all the VSphereMachines to be considered + // for the initial placement decision. + newVMG, err := computeVirtualMachineGroup(ctx, cluster, machineDeployments, vSphereMachines, nil) + if err != nil { + return reconcile.Result{}, err + } + + log.Info("Creating VirtualMachineGroup", "members", nameList(memberNames(newVMG))) + if err := r.Client.Create(ctx, newVMG); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to create new VMG") + } + return reconcile.Result{}, nil + } + + // If the VirtualMachineGroup exists, either the initial placement is in progress or + // the initial placement has been already completed. In both cases, the VirtualMachineGroup + // must be kept up to date with the changes that happen to MachineDeployments and vSphereMachines. + // + // However, while the initial placement is in progress, the addition of new + // VSphereMachines to the VirtualMachineGroup must be deferred to prevent race conditions. + // + // After initial placement, new vSphereMachines will be added to the VirtualMachineGroup for + // sake of consistency, but those machines will be placed in the same failureDomain + // already used for the other vSphereMachines in the same MachineDeployment (new vSphereMachines + // will align to the initial placement decision). + + // Computes the updated VirtualMachineGroup including reflecting changes in the cluster. + updatedVMG, err := computeVirtualMachineGroup(ctx, cluster, machineDeployments, vSphereMachines, currentVMG) + if err != nil { + return reconcile.Result{}, err + } + + existingVirtualMachineNames := sets.New[string](memberNames(currentVMG)...) + updatedVirtualMachineNames := sets.New[string](memberNames(updatedVMG)...) + + addedVirtualMachineNames := updatedVirtualMachineNames.Difference(existingVirtualMachineNames) + deletedVirtualMachineNames := existingVirtualMachineNames.Difference(updatedVirtualMachineNames) + if addedVirtualMachineNames.Len() > 0 || deletedVirtualMachineNames.Len() > 0 { + log.Info("Updating VirtualMachineGroup", "addedMembers", nameList(addedVirtualMachineNames.UnsortedList()), "deletedMembers", nameList(deletedVirtualMachineNames.UnsortedList())) + } + if err := r.Client.Patch(ctx, updatedVMG, client.MergeFromWithOptions(currentVMG, client.MergeFromWithOptimisticLock{})); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to patch VMG") + } + return reconcile.Result{}, nil +} + +// computeVirtualMachineGroup gets the desired VirtualMachineGroup. +func computeVirtualMachineGroup(ctx context.Context, cluster *clusterv1.Cluster, mds []clusterv1.MachineDeployment, vSphereMachines []vmwarev1.VSphereMachine, existingVMG *vmoprv1.VirtualMachineGroup) (*vmoprv1.VirtualMachineGroup, error) { + // Create an empty VirtualMachineGroup + vmg := &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: cluster.Name, + Namespace: cluster.Namespace, + Annotations: map[string]string{}, + }, + } + + // If there is an VirtualMachineGroup, clone it into the desired VirtualMachineGroup + // and clean up all the info that must be re-computed during this reconcile. + if existingVMG != nil { + vmg = existingVMG.DeepCopy() + vmg.Annotations = make(map[string]string) + for key, value := range existingVMG.Annotations { + if !strings.HasPrefix(key, vmoperator.ZoneAnnotationPrefix+"/") { + vmg.Annotations[key] = value + } + } + } + vmg.Spec.BootOrder = []vmoprv1.VirtualMachineGroupBootOrderGroup{{}} + + // Add cluster label and ownerReference to the cluster. + if vmg.Labels == nil { + vmg.Labels = map[string]string{} + } + vmg.Labels[clusterv1.ClusterNameLabel] = cluster.Name + vmg.OwnerReferences = util.EnsureOwnerRef(vmg.OwnerReferences, metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: ptr.To(true), + }) + + // Compute the info required to compute the VirtualMachineGroup. + + // Get the mapping between the VirtualMachine name that will be generated from a VSphereMachine + // and the MachineDeployment that controls the vSphereMachine. + virtualMachineNameToMachineDeployment, err := getVirtualMachineNameToMachineDeploymentMapping(ctx, vSphereMachines) + if err != nil { + return nil, err + } + + // Sort VirtualMachine names to ensure VirtualMachineGroup is generated in a consistent way across reconciles. + sortedVirtualMachineNames := slices.Sorted(maps.Keys(virtualMachineNameToMachineDeployment)) + + // Get the mapping between the MachineDeployment and failure domain, which is one of: + // - the failureDomain explicitly assigned by the user to a MachineDeployment (by setting spec.template.spec.failureDomain). + // - the failureDomain selected by the placement decision for a MachineDeployment + // Note: if a MachineDeployment is not included in this mapping, the MachineDeployment is still pending a placement decision. + machineDeploymentToFailureDomain := getMachineDeploymentToFailureDomainMapping(ctx, mds, existingVMG, virtualMachineNameToMachineDeployment) + + // Set the annotations on the VirtualMachineGroup surfacing the failure domain selected during the + // placement decision for each MachineDeployment. + // Note: when a MachineDeployment will be deleted, the corresponding annotation will be removed (not added anymore by this func). + for md, failureDomain := range machineDeploymentToFailureDomain { + vmg.Annotations[fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, md)] = failureDomain + } + + // Compute the list of Members for the VirtualMachineGroup. + + // If the VirtualMachineGroup is being created, ensure that all the existing VirtualMachines are + // included in the VirtualMachineGroup for the initial placement decision. + if existingVMG == nil { + for _, virtualMachineName := range sortedVirtualMachineNames { + vmg.Spec.BootOrder[0].Members = append(vmg.Spec.BootOrder[0].Members, vmoprv1.GroupMember{ + Name: virtualMachineName, + Kind: "VirtualMachine", + }) + } + return vmg, nil + } + + // If the VirtualMachineGroup exists, keep the list of VirtualMachines up to date. + // Note: while the initial placement is in progress, the addition of new + // VirtualMachines to the VirtualMachineGroup must be deferred to prevent race conditions. + // + // After initial placement, new VirtualMachine will be added to the VirtualMachineGroup for + // sake of consistency, but those Machines will be placed in the same failureDomain + // already used for the other VirtualMachines in the same MachineDeployment (new VirtualMachine + // will align to the initial placement decision). + existingVirtualMachineNames := sets.New[string](memberNames(existingVMG)...) + + for _, virtualMachineName := range sortedVirtualMachineNames { + // If a VirtualMachine is already part of the VirtualMachineGroup, keep it in the VirtualMachineGroup + // Note: when a VirtualMachine will be deleted, the corresponding member will be removed (not added anymore by this func) + if existingVirtualMachineNames.Has(virtualMachineName) { + vmg.Spec.BootOrder[0].Members = append(vmg.Spec.BootOrder[0].Members, vmoprv1.GroupMember{ + Name: virtualMachineName, + Kind: "VirtualMachine", + }) + continue + } + + // If a VirtualMachine is not yet in the VirtualMachineGroup, it should be added only if + // the VirtualMachine is controlled by a MachineDeployment for which the placement decision is already + // completed. + // Note: If the placement decision for the MachineDeployment controlling a VirtualMachine is still pending, + // this logic defers adding the VirtualMachine in the VirtualMachineGroup to prevent race conditions. + md := virtualMachineNameToMachineDeployment[virtualMachineName] + if _, isPlaced := machineDeploymentToFailureDomain[md]; isPlaced { + vmg.Spec.BootOrder[0].Members = append(vmg.Spec.BootOrder[0].Members, vmoprv1.GroupMember{ + Name: virtualMachineName, + Kind: "VirtualMachine", + }) + } + } + + return vmg, nil +} + +// getMachineDeploymentToFailureDomainMapping returns the mapping between MachineDeployment and failure domain. +// The mapping is computed according to following rules: +// - If the MachineDeployment is explicitly assigned to a failure domain by setting spec.template.spec.failureDomain, +// use this value for the mapping. +// - If the annotations on the VirtualMachineGroup already has the failure domain selected during the +// initial placement decision for a MachineDeployment, use it. +// - If annotations on the VirtualMachineGroup are not yet set, try to get the failure domain selected +// during the initial placement decision from VirtualMachineGroup status (placement decision just completed). +// - If none of the above rules are satisfied, the MachineDeployment is still pending a placement decision. +// +// Note: In case the failure domain is explicitly assigned by setting spec.template.spec.failureDomain, the mapping always +// report the latest value for this field (even if there might still be Machines yet to be rolled out to the new failure domain). +func getMachineDeploymentToFailureDomainMapping(ctx context.Context, mds []clusterv1.MachineDeployment, existingVMG *vmoprv1.VirtualMachineGroup, virtualMachineNameToMachineDeployment map[string]string) map[string]string { + log := ctrl.LoggerFrom(ctx) + + machineDeploymentToFailureDomainMapping := map[string]string{} + for _, md := range mds { + if !md.DeletionTimestamp.IsZero() { + continue + } + + // If the MachineDeployment is explicitly assigned to a failure domain by setting spec.template.spec.failureDomain, use this value for the mapping. + if md.Spec.Template.Spec.FailureDomain != "" { + machineDeploymentToFailureDomainMapping[md.Name] = md.Spec.Template.Spec.FailureDomain + continue + } + + // If the MachineDeployment is not explicitly assigned to a failure domain (spec.template.spec.failureDomain is empty), + // and VirtualMachineGroup does not exist yet, the MachineDeployment is still pending a placement decision. + if existingVMG == nil { + continue + } + + // If the VirtualMachineGroup exist, check if the placement decision for the MachineDeployment + // has been already surfaced into the VirtualMachineGroup annotations. + if failureDomain := existingVMG.Annotations[fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, md.Name)]; failureDomain != "" { + machineDeploymentToFailureDomainMapping[md.Name] = failureDomain + continue + } + + // If the placement decision for the MachineDeployment is not yet surfaced in the annotation, try to get + // the failure domain selected during the initial placement decision from VirtualMachineGroup status + // (placement decision just completed). + // Note: this info will surface in VirtualMachineGroup annotations at the end of the current reconcile. + for _, member := range existingVMG.Status.Members { + // Ignore members controlled by other MachineDeployments + if memberMD := virtualMachineNameToMachineDeployment[member.Name]; memberMD != md.Name { + continue + } + + // Consider only VirtualMachineGroup members for which the placement decision has been completed. + // Note: given that all the VirtualMachines in a MachineDeployment must be placed in the + // same failure domain / zone, the mapping can be inferred as soon as one member is placed. + if !conditions.IsTrue(&member, vmoprv1.VirtualMachineGroupMemberConditionPlacementReady) { + continue + } + if member.Placement != nil && member.Placement.Zone != "" { + log.Info(fmt.Sprintf("MachineDeployment %s has been placed to failure domanin %s", md.Name, member.Placement.Zone), "MachineDeployment", klog.KObj(&md)) + machineDeploymentToFailureDomainMapping[md.Name] = member.Placement.Zone + break + } + } + } + return machineDeploymentToFailureDomainMapping +} + +// getVirtualMachineNameToMachineDeploymentMapping returns the mapping between VirtualMachine name and corresponding MachineDeployment. +// The mapping is inferred from vSphereMachines. Please note: +// - The name of the VirtualMachine generated by a VSphereMachine can be computed in a deterministic way (it is not required to wait for the VirtualMachine to exist) +// - The name of the MachineDeployment corresponding to a vSphereMachine can be derived from the annotation that is propagated by CAPI. +func getVirtualMachineNameToMachineDeploymentMapping(_ context.Context, vSphereMachines []vmwarev1.VSphereMachine) (map[string]string, error) { + virtualMachineNameToMachineDeployment := map[string]string{} + for _, vsphereMachine := range vSphereMachines { + if !vsphereMachine.DeletionTimestamp.IsZero() { + continue + } + + virtualMachineName, err := vmoperator.GenerateVirtualMachineName(vsphereMachine.Name, vsphereMachine.Spec.NamingStrategy) + if err != nil { + return nil, err + } + if md := vsphereMachine.Labels[clusterv1.MachineDeploymentNameLabel]; md != "" { + virtualMachineNameToMachineDeployment[virtualMachineName] = md + } + } + return virtualMachineNameToMachineDeployment, nil +} + +// shouldCreateVirtualMachineGroup should return true when the conditions to create a VirtualMachineGroup are met. +func shouldCreateVirtualMachineGroup(ctx context.Context, mds []clusterv1.MachineDeployment, vSphereMachines []vmwarev1.VSphereMachine) bool { + log := ctrl.LoggerFrom(ctx) + + // Gets the total number or worker machines that should exist in the cluster at a given time. + // Note. Deleting MachineDeployment are ignored. + var expectedVSphereMachineCount int32 + mdNames := sets.Set[string]{} + for _, md := range mds { + if !md.DeletionTimestamp.IsZero() { + continue + } + expectedVSphereMachineCount += ptr.Deref(md.Spec.Replicas, 0) + mdNames.Insert(md.Name) + } + + // In case there are no MachineDeployments or all the MachineDeployments have zero replicas, there is + // no need to create a VirtualMachineGroup. + if expectedVSphereMachineCount == 0 { + return false + } + + // Filter down VSphereMachines to the ones belonging to the MachineDeployment considered above. + // Note: if at least one of those VSphereMachines is deleting, wait for the deletion to complete. + currentVSphereMachineCount := int32(0) + for _, vSphereMachine := range vSphereMachines { + md := vSphereMachine.Labels[clusterv1.MachineDeploymentNameLabel] + if !mdNames.Has(md) { + continue + } + + if !vSphereMachine.DeletionTimestamp.IsZero() { + log.Info("Waiting for VSphereMachines required for the initial placement to be deleted") + return false + } + + currentVSphereMachineCount++ + } + + // If the number of workers VSphereMachines matches the number of expected replicas in the MachineDeployments, + // then all the VSphereMachines required for the initial placement decision do exist, then it is possible to create + // the VirtualMachineGroup. + if currentVSphereMachineCount != expectedVSphereMachineCount { + log.Info(fmt.Sprintf("Waiting for VSphereMachines required for the initial placement (expected %d, current %d)", expectedVSphereMachineCount, currentVSphereMachineCount)) + return false + } + return true +} + +func (r *VirtualMachineGroupReconciler) getVirtualMachineGroup(ctx context.Context, cluster *clusterv1.Cluster) (*vmoprv1.VirtualMachineGroup, error) { + vmg := &vmoprv1.VirtualMachineGroup{} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(cluster), vmg); err != nil { + if !apierrors.IsNotFound(err) { + return nil, errors.Wrapf(err, "failed to get VirtualMachineGroup %s", klog.KObj(vmg)) + } + return nil, nil + } + return vmg, nil +} + +func (r *VirtualMachineGroupReconciler) getVSphereMachines(ctx context.Context, cluster *clusterv1.Cluster) ([]vmwarev1.VSphereMachine, error) { + var vsMachineList vmwarev1.VSphereMachineList + if err := r.Client.List(ctx, &vsMachineList, + client.InNamespace(cluster.Namespace), + client.MatchingLabels{clusterv1.ClusterNameLabel: cluster.Name}, + client.HasLabels{clusterv1.MachineDeploymentNameLabel}, + ); err != nil { + return nil, errors.Wrap(err, "failed to get VSphereMachines") + } + return vsMachineList.Items, nil +} + +func (r *VirtualMachineGroupReconciler) getMachineDeployments(ctx context.Context, cluster *clusterv1.Cluster) ([]clusterv1.MachineDeployment, error) { + machineDeployments := &clusterv1.MachineDeploymentList{} + if err := r.Client.List(ctx, machineDeployments, + client.InNamespace(cluster.Namespace), + client.MatchingLabels{clusterv1.ClusterNameLabel: cluster.Name}, + ); err != nil { + return nil, errors.Wrap(err, "failed to list MachineDeployments") + } + return machineDeployments.Items, nil +} + +func memberNames(vmg *vmoprv1.VirtualMachineGroup) []string { + names := []string{} + if len(vmg.Spec.BootOrder) > 0 { + for _, member := range vmg.Spec.BootOrder[0].Members { + names = append(names, member.Name) + } + } + return names +} + +func nameList(names []string) string { + sort.Strings(names) + switch { + case len(names) <= 20: + return strings.Join(names, ", ") + default: + return fmt.Sprintf("%s, ... (%d more)", strings.Join(names[:20], ", "), len(names)-20) + } +} diff --git a/controllers/vmware/virtualmachinegroup_reconciler_test.go b/controllers/vmware/virtualmachinegroup_reconciler_test.go new file mode 100644 index 0000000000..bc3be37f6c --- /dev/null +++ b/controllers/vmware/virtualmachinegroup_reconciler_test.go @@ -0,0 +1,1227 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "fmt" + "testing" + + . "github.com/onsi/gomega" + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/vmoperator" +) + +func Test_shouldCreateVirtualMachineGroup(t *testing.T) { + tests := []struct { + name string + mds []clusterv1.MachineDeployment + vSphereMachines []vmwarev1.VSphereMachine + want bool + }{ + { + name: "Should create a VMG if all the expected VSphereMachines exists", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 2), + *createMD("md2", "test-cluster", "", 1), + *createMD("md3", "test-cluster", "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", "test-cluster", "md1", ""), + *createVSphereMachine("m2", "test-cluster", "md1", ""), + *createVSphereMachine("m3", "test-cluster", "md2", ""), + *createVSphereMachine("m4", "test-cluster", "md3", "zone1"), + }, + want: true, // tot replicas = 4, 4 VSphereMachine exist + }, + { + name: "Should create a VMG if all the expected VSphereMachines exists, deleting MD should be ignored", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 2), + *createMD("md2", "test-cluster", "", 1, func(md *clusterv1.MachineDeployment) { + md.DeletionTimestamp = ptr.To(metav1.Now()) + }), // Should not be included in the count + *createMD("md3", "test-cluster", "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", "test-cluster", "md1", ""), + *createVSphereMachine("m2", "test-cluster", "md1", ""), + *createVSphereMachine("m4", "test-cluster", "md3", "zone1"), + }, + want: true, // tot replicas = 3 (one md is deleting, so not included in the total), 3 VSphereMachine exist + }, + { + name: "Should not create a VMG if some of the expected VSphereMachines does not exist", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 2), + *createMD("md2", "test-cluster", "", 1), + *createMD("md3", "test-cluster", "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", "test-cluster", "md1", ""), + *createVSphereMachine("m3", "test-cluster", "md2", ""), + *createVSphereMachine("m4", "test-cluster", "md3", "zone1"), + }, + want: false, // tot replicas = 4, 3 VSphereMachine exist + }, + { + name: "Should not create a VMG there are no expected VSphereMachines", + mds: []clusterv1.MachineDeployment{}, // No Machine deployments + vSphereMachines: []vmwarev1.VSphereMachine{}, // No VSphereMachine + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got := shouldCreateVirtualMachineGroup(ctx, tt.mds, tt.vSphereMachines) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_getVirtualMachineNameToMachineDeploymentMapping(t *testing.T) { + tests := []struct { + name string + vSphereMachines []vmwarev1.VSphereMachine + want map[string]string + }{ + { + name: "mapping from VirtualMachineName to MachineDeployment is inferred from vSphereMachines", + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", "test-cluster", "md1", ""), + *createVSphereMachine("m2", "test-cluster", "md1", ""), + *createVSphereMachine("m3", "test-cluster", "md2", ""), + *createVSphereMachine("m4", "test-cluster", "md3", "zone1"), + }, + want: map[string]string{ + // Note VirtualMachineName is equal to the VSphereMachine name because when using the default naming strategy + "m1": "md1", + "m2": "md1", + "m3": "md2", + "m4": "md3", + }, + }, + { + name: "mapping from VirtualMachineName to MachineDeployment is inferred from vSphereMachines (custom naming strategy)", + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", "test-cluster", "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", "test-cluster", "md1", "", withCustomNamingStrategy(), func(m *vmwarev1.VSphereMachine) { + m.DeletionTimestamp = ptr.To(metav1.Now()) + }), // Should not be included in the mapping + *createVSphereMachine("m3", "test-cluster", "md2", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", "test-cluster", "md3", "zone1"), + }, + want: map[string]string{ + "m1-vm": "md1", + // "m2-vm" not be included in the count + "m3-vm": "md2", + "m4": "md3", + }, + }, + { + name: "deleting vSphereMachines are not included in the mapping", + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", "test-cluster", "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", "test-cluster", "md1", "", withCustomNamingStrategy(), func(m *vmwarev1.VSphereMachine) { + m.DeletionTimestamp = ptr.To(metav1.Now()) + }), // Should not be included in the mapping + *createVSphereMachine("m3", "test-cluster", "md2", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", "test-cluster", "md3", "zone1"), + }, + want: map[string]string{ + "m1-vm": "md1", + // "m2-vm" not be included in the count + "m3-vm": "md2", + "m4": "md3", + }, + }, + { + name: "vSphereMachines without the MachineDeploymentNameLabel are not included in the mapping", + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", "test-cluster", "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", "test-cluster", "md1", "", withCustomNamingStrategy(), func(m *vmwarev1.VSphereMachine) { + delete(m.Labels, clusterv1.MachineDeploymentNameLabel) + }), // Should not be included in the mapping + *createVSphereMachine("m3", "test-cluster", "md2", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", "test-cluster", "md3", "zone1"), + }, + want: map[string]string{ + "m1-vm": "md1", + // "m2-vm" not be included in the count + "m3-vm": "md2", + "m4": "md3", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got, err := getVirtualMachineNameToMachineDeploymentMapping(ctx, tt.vSphereMachines) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_getMachineDeploymentToFailureDomainMapping(t *testing.T) { + tests := []struct { + name string + mds []clusterv1.MachineDeployment + existingVMG *vmoprv1.VirtualMachineGroup + virtualMachineNameToMachineDeployment map[string]string + want map[string]string + }{ + { + name: "MachineDeployment mapping should use spec.FailureDomain", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "zone1", 1), // failure domain explicitly set + }, + existingVMG: nil, + virtualMachineNameToMachineDeployment: nil, + want: map[string]string{ + "md1": "zone1", + }, + }, + { + name: "MachineDeployment mapping should use spec.FailureDomain (latest value must be used)", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "zone2", 1), // failure domain explicitly set + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone1", // Previously md1 was assigned to zone1 + }, + }, + }, + virtualMachineNameToMachineDeployment: nil, + want: map[string]string{ + "md1": "zone2", // latest spec.failure must be used + }, + }, + { + name: "MachineDeployment mapping should use placement decision from VirtualMachineGroup annotations", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 1), // failure domain not explicitly set + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone1", // Placement decision for md1 already reported into annotation + }, + }, + Status: vmoprv1.VirtualMachineGroupStatus{ + Members: []vmoprv1.VirtualMachineGroupMemberStatus{ + { + Name: "m1-vm", + Placement: &vmoprv1.VirtualMachinePlacementStatus{ + Zone: "zone2", // Note: this should never happen (different placement decision than what is in the annotation), but using this value to validate that the mapping used is the one from the annotation. + }, + }, + }, + }, + }, + virtualMachineNameToMachineDeployment: map[string]string{ + "m1-vm": "md1", + }, + want: map[string]string{ + "md1": "zone1", + }, + }, + { + name: "MachineDeployment mapping should use placement decision from VirtualMachineGroup status", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 1), // failure domain not explicitly set + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + // Placement decision for md1 not yet reported into annotation + }, + }, + Status: vmoprv1.VirtualMachineGroupStatus{ + Members: []vmoprv1.VirtualMachineGroupMemberStatus{ + { + Name: "m1-vm", + Placement: &vmoprv1.VirtualMachinePlacementStatus{ + Zone: "zone1", + }, + Conditions: []metav1.Condition{ + { + Type: vmoprv1.VirtualMachineGroupMemberConditionPlacementReady, + Status: metav1.ConditionTrue, + }, + }, + }, + }, + }, + }, + virtualMachineNameToMachineDeployment: map[string]string{ + "m1-vm": "md1", + }, + want: map[string]string{ + "md1": "zone1", + }, + }, + { + name: "MachineDeployment not yet placed (VirtualMachineGroup not yet created)", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 1), // failure domain not explicitly set + }, + existingVMG: nil, + virtualMachineNameToMachineDeployment: map[string]string{ + "m1-vm": "md1", + }, + want: map[string]string{ + // "md1" not yet placed + }, + }, + { + name: "MachineDeployment not yet placed (VirtualMachineGroup status not yet reporting placement for MachineDeployment's VirtualMachines)", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 1), // failure domain not explicitly set + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + // Placement decision for md1 not yet reported into annotation + }, + }, + // Status empty + }, + virtualMachineNameToMachineDeployment: nil, + want: map[string]string{}, // "md1" not yet placed + }, + { + name: "MachineDeployment not yet placed (VirtualMachineGroup status not yet reporting placement completed for MachineDeployment's VirtualMachines)", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", "test-cluster", "", 1), // failure domain not explicitly set + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + // Placement decision for md1 not yet reported into annotation + }, + }, + Status: vmoprv1.VirtualMachineGroupStatus{ + Members: []vmoprv1.VirtualMachineGroupMemberStatus{ + { + Name: "m1-vm", + Conditions: []metav1.Condition{ + { + Type: vmoprv1.VirtualMachineGroupMemberConditionPlacementReady, + Status: metav1.ConditionFalse, // placement not completed yet + }, + }, + }, + }, + }, + }, + virtualMachineNameToMachineDeployment: map[string]string{ + "m1-vm": "md1", + }, + want: map[string]string{ + // "md1" not yet placed + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got := getMachineDeploymentToFailureDomainMapping(ctx, tt.mds, tt.existingVMG, tt.virtualMachineNameToMachineDeployment) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func TestVirtualMachineGroupReconciler_computeVirtualMachineGroup(t *testing.T) { + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "test-cluster", + }, + } + tests := []struct { + name string + mds []clusterv1.MachineDeployment + vSphereMachines []vmwarev1.VSphereMachine + existingVMG *vmoprv1.VirtualMachineGroup + want *vmoprv1.VirtualMachineGroup + }{ + // Compute new VirtualMachineGroup (start initial placement) + { + name: "compute new VirtualMachineGroup", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", cluster.Name, "", 2), + *createMD("md2", cluster.Name, "", 1), + *createMD("md3", cluster.Name, "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", cluster.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", cluster.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m3", cluster.Name, "md2", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", cluster.Name, "md3", "zone1"), + }, + existingVMG: nil, + want: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: cluster.Name, + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster.Name, + }, + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md3"): "zone1", // failureDomain for md3 is explicitly set by the user + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3-vm", Kind: "VirtualMachine"}, + {Name: "m4", Kind: "VirtualMachine"}, + }, + }, + }, + }, + }, + }, + + // Compute updated VirtualMachineGroup (during initial placement) + { + name: "compute updated VirtualMachineGroup during initial placement", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", cluster.Name, "", 2), + *createMD("md3", cluster.Name, "zone1", 2), + *createMD("md4", cluster.Name, "zone2", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", cluster.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m5", cluster.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", cluster.Name, "md3", "zone1"), + *createVSphereMachine("m6", cluster.Name, "md3", "zone1"), + *createVSphereMachine("m7", cluster.Name, "md4", "zone2"), + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: cluster.Name, + UID: types.UID("uid"), + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster.Name, + }, + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md3"): "zone1", // failureDomain for md3 is explicitly set by the user + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, // Deleted after VMG creation + {Name: "m3-vm", Kind: "VirtualMachine"}, // Deleted after VMG creation (the entire md2 was deleted). + {Name: "m4", Kind: "VirtualMachine"}, + // m5-vm (md1), m6 (md3), m7 (md4) created after VMG creation. + }, + }, + }, + }, + // Not setting status for sake of simplicity (also we are simulating when placing decision is not yet completed) + }, + want: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: cluster.Name, + UID: types.UID("uid"), + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster.Name, + }, + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md3"): "zone1", // failureDomain for md3 is explicitly set by the user + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md4"): "zone2", // failureDomain for md4 is explicitly set by the user + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, // existing before, still existing + // "m2-vm" was deleted + // "m3-vm" was deleted + {Name: "m4", Kind: "VirtualMachine"}, // existing before, still existing + // "m5-vm" was added, but it should not be added yet because md1 is not yet placed + {Name: "m6", Kind: "VirtualMachine"}, // added, failureDomain for md3 is explicitly set by the user + {Name: "m7", Kind: "VirtualMachine"}, // added, failureDomain for md4 is explicitly set by the user + }, + }, + }, + }, + }, + }, + + // Compute updated VirtualMachineGroup (after initial placement) + { + name: "compute updated VirtualMachineGroup after initial placement", + mds: []clusterv1.MachineDeployment{ + *createMD("md1", cluster.Name, "", 2), + *createMD("md3", cluster.Name, "zone1", 2), + *createMD("md4", cluster.Name, "zone2", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", cluster.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m5", cluster.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", cluster.Name, "md3", "zone1"), + *createVSphereMachine("m6", cluster.Name, "md3", "zone1"), + *createVSphereMachine("m7", cluster.Name, "md4", "zone2"), + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: cluster.Name, + UID: types.UID("uid"), + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster.Name, + }, + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone5", // failureDomain for md2 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md3"): "zone1", // failureDomain for md3 is explicitly set by the user + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, // Deleted after VMG creation + {Name: "m3-vm", Kind: "VirtualMachine"}, // Deleted after VMG creation (the entire md2 was deleted). + {Name: "m4", Kind: "VirtualMachine"}, + // m5-vm (md1), m6 (md3), m7 (md4) created after VMG creation. + }, + }, + }, + }, + // Not setting status for sake of simplicity (in a real VMG, after the placement decision status should have members) + }, + want: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: cluster.Name, + UID: types.UID("uid"), + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster.Name, + }, + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + // annotation for md2 deleted, md2 does not exist anymore + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md3"): "zone1", // failureDomain for md3 is explicitly set by the user + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md4"): "zone2", // failureDomain for md4 is explicitly set by the user + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, // existing before, still existing + // "m2-vm" was deleted + // "m3-vm" was deleted + {Name: "m4", Kind: "VirtualMachine"}, // existing before, still existing + {Name: "m5-vm", Kind: "VirtualMachine"}, // added, failureDomain for md1 set by initial placement + {Name: "m6", Kind: "VirtualMachine"}, // added, failureDomain for md3 is explicitly set by the user + {Name: "m7", Kind: "VirtualMachine"}, // added, failureDomain for md4 is explicitly set by the user + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got, err := computeVirtualMachineGroup(ctx, cluster, tt.mds, tt.vSphereMachines, tt.existingVMG) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(BeComparableTo(tt.want)) + }) + } +} + +func TestVirtualMachineGroupReconciler_ReconcileSequence(t *testing.T) { + clusterNotYetInitialized := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "test-cluster", + }, + } + clusterInitialized := clusterNotYetInitialized.DeepCopy() + clusterInitialized.Status.Conditions = []metav1.Condition{ + { + Type: clusterv1.ClusterControlPlaneInitializedCondition, + Status: metav1.ConditionTrue, + }, + } + + tests := []struct { + name string + cluster *clusterv1.Cluster + mds []clusterv1.MachineDeployment + vSphereMachines []vmwarev1.VSphereMachine + existingVMG *vmoprv1.VirtualMachineGroup + wantResult ctrl.Result + wantVMG *vmoprv1.VirtualMachineGroup + }{ + // Before initial placement + { + name: "VirtualMachineGroup should not be created when the cluster is not yet initialized", + cluster: clusterNotYetInitialized, + mds: nil, + vSphereMachines: nil, + existingVMG: nil, + wantResult: ctrl.Result{}, + wantVMG: nil, + }, + { + name: "VirtualMachineGroup should not be created when waiting for vSphereMachines to exist", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 1), + *createMD("md2", clusterInitialized.Name, "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + }, + existingVMG: nil, + wantResult: ctrl.Result{}, + wantVMG: nil, + }, + { + name: "VirtualMachineGroup should not be created when waiting for vSphereMachines to exist (adapt to changes)", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 2), // Scaled up one additional machine is still missing + *createMD("md2", clusterInitialized.Name, "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + }, + existingVMG: nil, + wantResult: ctrl.Result{}, + wantVMG: nil, + }, + { + name: "VirtualMachineGroup should be created when all the vSphereMachines exist", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 2), + *createMD("md2", clusterInitialized.Name, "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + }, + existingVMG: nil, + wantResult: ctrl.Result{}, + wantVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + }, + }, + }, + }, + }, + }, + + // During initial placement + { + name: "No op if nothing changes during initial placement", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 2), + *createMD("md2", clusterInitialized.Name, "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + }, + }, + }, + }, + }, + wantResult: ctrl.Result{}, + wantVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + }, + }, + }, + }, + }, + }, + { + name: "Only new VSphereMachines with an explicit placement are added during initial placement", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 3), // scaled up + *createMD("md2", clusterInitialized.Name, "zone1", 2), // scaled up + *createMD("md3", clusterInitialized.Name, "", 1), // new + *createMD("md4", clusterInitialized.Name, "zone2", 1), // new + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), // new + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + *createVSphereMachine("m5", clusterInitialized.Name, "md2", "zone1"), // new + *createVSphereMachine("m6", clusterInitialized.Name, "md3", "", withCustomNamingStrategy()), // new + *createVSphereMachine("m7", clusterInitialized.Name, "md4", "zone3"), // new + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + }, + }, + }, + }, + }, + wantResult: ctrl.Result{}, + wantVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md4"): "zone2", // failureDomain for md4 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + // "m4-vm" not added, placement decision for md1 not yet completed + {Name: "m3", Kind: "VirtualMachine"}, + {Name: "m5", Kind: "VirtualMachine"}, // added, failureDomain for md2 is explicitly set by the user + // "m6-vm" not added, placement decision for md3 not yet completed + {Name: "m7", Kind: "VirtualMachine"}, // added, failureDomain for md4 is explicitly set by the user + }, + }, + }, + }, + }, + }, + { + name: "VSphereMachines are removed during initial placement", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 3), // scaled down + *createMD("md2", clusterInitialized.Name, "zone1", 2), // scaled down + // md3 deleted + // md4 deleted + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + // m4 deleted + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + // m5 deleted + // m6 deleted + // m7 deleted + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md4"): "zone2", // failureDomain for md4 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + // "m4-vm" not added, placement decision for md1 not yet completed + {Name: "m3", Kind: "VirtualMachine"}, + {Name: "m5", Kind: "VirtualMachine"}, // added, failureDomain for md2 is explicitly set by the user + // "m6-vm" not added, placement decision for md3 not yet completed + {Name: "m7", Kind: "VirtualMachine"}, // added, failureDomain for md4 is explicitly set by the user + }, + }, + }, + }, + }, + wantResult: ctrl.Result{}, + wantVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + // md4 deleted + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + // "m4-vm" deleted (it was never added) + {Name: "m3", Kind: "VirtualMachine"}, + // "m5" deleted + // "m6" deleted + // "m7" deleted + }, + }, + }, + }, + }, + }, + + // After initial placement + { + name: "No op if nothing changes after initial placement", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 2), + *createMD("md2", clusterInitialized.Name, "zone1", 1), + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + }, + }, + }, + }, + // Not setting status for sake of simplicity (in a real VMG, after the placement decision status should have members) + }, + wantResult: ctrl.Result{}, + wantVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + }, + }, + }, + }, + }, + }, + { + name: "New VSphereMachines are added after initial placement", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 3), // scaled up + *createMD("md2", clusterInitialized.Name, "zone1", 2), // scaled up + *createMD("md3", clusterInitialized.Name, "zone2", 1), // new + // Adding a new MD without explicit placement is not supported at this stage + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m4", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), // new + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + *createVSphereMachine("m5", clusterInitialized.Name, "md2", "zone1"), // new + *createVSphereMachine("m6", clusterInitialized.Name, "md3", "zone2"), // new + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + }, + }, + }, + }, + // Not setting status for sake of simplicity (in a real VMG, after the placement decision status should have members) + }, + wantResult: ctrl.Result{}, + wantVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md3"): "zone2", // failureDomain for md3 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + {Name: "m4-vm", Kind: "VirtualMachine"}, // added, failureDomain for md1 set by initial placement + {Name: "m5", Kind: "VirtualMachine"}, // added, failureDomain for md2 is explicitly set by the user + {Name: "m6", Kind: "VirtualMachine"}, // added, failureDomain for md3 is explicitly set by the user + }, + }, + }, + }, + }, + }, + { + name: "VSphereMachines are removed after initial placement", + cluster: clusterInitialized, + mds: []clusterv1.MachineDeployment{ + *createMD("md1", clusterInitialized.Name, "", 3), // scaled down + *createMD("md2", clusterInitialized.Name, "zone1", 2), // scaled down + // md3 deleted + }, + vSphereMachines: []vmwarev1.VSphereMachine{ + *createVSphereMachine("m1", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + *createVSphereMachine("m2", clusterInitialized.Name, "md1", "", withCustomNamingStrategy()), + // m4 deleted + *createVSphereMachine("m3", clusterInitialized.Name, "md2", "zone1"), + // m5 deleted + // m5 deleted + }, + existingVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md3"): "zone2", // failureDomain for md3 is explicitly set by the user + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + {Name: "m4-vm", Kind: "VirtualMachine"}, + {Name: "m5", Kind: "VirtualMachine"}, + {Name: "m6", Kind: "VirtualMachine"}, + }, + }, + }, + }, + // Not setting status for sake of simplicity (in a real VMG, after the placement decision status should have members) + }, + wantResult: ctrl.Result{}, + wantVMG: &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterInitialized.Namespace, + Name: clusterInitialized.Name, + UID: types.UID("uid"), + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md1"): "zone4", // failureDomain for md1 set by initial placement + fmt.Sprintf("%s/%s", vmoperator.ZoneAnnotationPrefix, "md2"): "zone1", // failureDomain for md2 is explicitly set by the user + // md3 deleted + }, + // Not setting labels and ownerReferences for sake of simplicity + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + {Name: "m1-vm", Kind: "VirtualMachine"}, + {Name: "m2-vm", Kind: "VirtualMachine"}, + {Name: "m3", Kind: "VirtualMachine"}, + // m4-vm deleted + // m5 deleted + // m6 deleted + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + objects := []client.Object{tt.cluster} + if tt.existingVMG != nil { + objects = append(objects, tt.existingVMG) + } + for _, md := range tt.mds { + objects = append(objects, &md) + } + for _, vSphereMachine := range tt.vSphereMachines { + objects = append(objects, &vSphereMachine) + } + + c := fake.NewClientBuilder().WithObjects(objects...).Build() + r := &VirtualMachineGroupReconciler{ + Client: c, + } + got, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: tt.cluster.Namespace, Name: tt.cluster.Name}}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.wantResult)) + + vmg := &vmoprv1.VirtualMachineGroup{} + err = r.Client.Get(ctx, client.ObjectKeyFromObject(tt.cluster), vmg) + + if tt.wantVMG == nil { + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(vmg.Labels).To(HaveKeyWithValue(clusterv1.ClusterNameLabel, tt.cluster.Name)) + g.Expect(vmg.OwnerReferences).To(ContainElement(metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: tt.cluster.Name, + UID: tt.cluster.UID, + Controller: ptr.To(true), + })) + g.Expect(vmg.Annotations).To(Equal(tt.wantVMG.Annotations)) + g.Expect(vmg.Spec.BootOrder).To(Equal(tt.wantVMG.Spec.BootOrder)) + }) + } +} + +type machineDeploymentOption func(md *clusterv1.MachineDeployment) + +func createMD(name, cluster, failureDomain string, replicas int32, options ...machineDeploymentOption) *clusterv1.MachineDeployment { + md := &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: name, + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster, + }, + }, + Spec: clusterv1.MachineDeploymentSpec{ + Template: clusterv1.MachineTemplateSpec{Spec: clusterv1.MachineSpec{FailureDomain: failureDomain}}, + Replicas: &replicas, + }, + } + for _, opt := range options { + opt(md) + } + return md +} + +type vSphereMachineOption func(m *vmwarev1.VSphereMachine) + +func withCustomNamingStrategy() func(m *vmwarev1.VSphereMachine) { + return func(m *vmwarev1.VSphereMachine) { + m.Spec.NamingStrategy = &vmwarev1.VirtualMachineNamingStrategy{ + Template: ptr.To[string]("{{ .machine.name }}-vm"), + } + } +} + +func createVSphereMachine(name, cluster, md, failureDomain string, options ...vSphereMachineOption) *vmwarev1.VSphereMachine { + m := &vmwarev1.VSphereMachine{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: name, + Labels: map[string]string{ + clusterv1.ClusterNameLabel: cluster, + clusterv1.MachineDeploymentNameLabel: md, + }, + }, + Spec: vmwarev1.VSphereMachineSpec{ + FailureDomain: &failureDomain, + }, + } + for _, opt := range options { + opt(m) + } + return m +} diff --git a/feature/feature.go b/feature/feature.go index a233d351c7..1799aaeb68 100644 --- a/feature/feature.go +++ b/feature/feature.go @@ -44,6 +44,11 @@ const ( // alpha: v1.11 NamespaceScopedZones featuregate.Feature = "NamespaceScopedZones" + // NodeAutoPlacement is a feature gate for the NodeAutoPlacement functionality for supervisor. + // + // alpha: v1.15 + NodeAutoPlacement featuregate.Feature = "NodeAutoPlacement" + // PriorityQueue is a feature gate that controls if the controller uses the controller-runtime PriorityQueue // instead of the default queue implementation. // @@ -61,6 +66,7 @@ var defaultCAPVFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ // Every feature should be initiated here: NodeAntiAffinity: {Default: false, PreRelease: featuregate.Alpha}, NamespaceScopedZones: {Default: false, PreRelease: featuregate.Alpha}, + NodeAutoPlacement: {Default: false, PreRelease: featuregate.Alpha}, PriorityQueue: {Default: false, PreRelease: featuregate.Alpha}, MultiNetworks: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/go.mod b/go.mod index 44f52fa9ea..1e30e9c7ca 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,16 @@ go 1.24.0 replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.11.0-rc.0.0.20250905091528-eb4e38c46ff6 -replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels v0.0.0-20240404200847-de75746a9505 +replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels v1.9.1-0.20251029150609-93918c59a719 // The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-tests -replace github.com/vmware-tanzu/vm-operator/api => github.com/vmware-tanzu/vm-operator/api v1.8.6 +replace github.com/vmware-tanzu/vm-operator/api => github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719 require ( github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20241112044858-9da8637c1b0d // The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-tests - github.com/vmware-tanzu/vm-operator/api v1.8.6 + github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 github.com/vmware/govmomi v0.52.0 ) diff --git a/go.sum b/go.sum index 47a16466b0..38eb0f2114 100644 --- a/go.sum +++ b/go.sum @@ -243,8 +243,8 @@ github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:c github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20241112044858-9da8637c1b0d h1:z9lrzKVtNlujduv9BilzPxuge/LE2F0N1ms3TP4JZvw= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20241112044858-9da8637c1b0d/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= -github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU= -github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A= +github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719 h1:nb/5ytRj7E/5eo9UzLfaR29JytMtbGpqMVs3hjaRwZ0= +github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719/go.mod h1:nWTPpxfe4gHuuYuFcrs86+NMxfkqPk3a3IlvI8TCWak= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505/go.mod h1:5rqRJ9zGR+KnKbkGx373WgN8xJpvAj99kHnfoDYRO5I= github.com/vmware/govmomi v0.52.0 h1:JyxQ1IQdllrY7PJbv2am9mRsv3p9xWlIQ66bv+XnyLw= diff --git a/internal/test/helpers/envtest.go b/internal/test/helpers/envtest.go index 41341b70cb..0acbcd68eb 100644 --- a/internal/test/helpers/envtest.go +++ b/internal/test/helpers/envtest.go @@ -29,6 +29,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/pkg/errors" + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware/govmomi/simulator" "golang.org/x/tools/go/packages" admissionv1 "k8s.io/api/admissionregistration/v1" @@ -89,6 +90,7 @@ func init() { utilruntime.Must(admissionv1.AddToScheme(scheme)) utilruntime.Must(clusterv1.AddToScheme(scheme)) utilruntime.Must(infrav1.AddToScheme(scheme)) + utilruntime.Must(vmoprv1.AddToScheme(scheme)) // Get the root of the current file to use in CRD paths. _, filename, _, ok := goruntime.Caller(0) diff --git a/main.go b/main.go index b92f48d25a..6d6ea9e011 100644 --- a/main.go +++ b/main.go @@ -94,6 +94,7 @@ var ( vSphereVMConcurrency int vSphereClusterIdentityConcurrency int vSphereDeploymentZoneConcurrency int + virtualMachineGroupConcurrency int skipCRDMigrationPhases []string managerOptions = capiflags.ManagerOptions{} @@ -141,6 +142,9 @@ func InitFlags(fs *pflag.FlagSet) { fs.IntVar(&vSphereDeploymentZoneConcurrency, "vspheredeploymentzone-concurrency", 10, "Number of vSphere deployment zones to process simultaneously") + fs.IntVar(&virtualMachineGroupConcurrency, "virtualmachinegroup-concurrency", 10, + "Number of virtual machine group to process simultaneously") + fs.StringVar( &managerOpts.PodName, "pod-name", @@ -482,6 +486,12 @@ func setupSupervisorControllers(ctx context.Context, controllerCtx *capvcontext. return err } + if feature.Gates.Enabled(feature.NamespaceScopedZones) && feature.Gates.Enabled(feature.NodeAutoPlacement) { + if err := vmware.AddVirtualMachineGroupControllerToManager(ctx, controllerCtx, mgr, concurrency(virtualMachineGroupConcurrency)); err != nil { + return err + } + } + return vmware.AddServiceDiscoveryControllerToManager(ctx, controllerCtx, mgr, clusterCache, concurrency(serviceDiscoveryConcurrency)) } diff --git a/packaging/go.sum b/packaging/go.sum index 14a389257b..449ae296a6 100644 --- a/packaging/go.sum +++ b/packaging/go.sum @@ -135,8 +135,8 @@ github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:c github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20241112044858-9da8637c1b0d h1:z9lrzKVtNlujduv9BilzPxuge/LE2F0N1ms3TP4JZvw= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20241112044858-9da8637c1b0d/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= -github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU= -github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A= +github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719 h1:nb/5ytRj7E/5eo9UzLfaR29JytMtbGpqMVs3hjaRwZ0= +github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719/go.mod h1:nWTPpxfe4gHuuYuFcrs86+NMxfkqPk3a3IlvI8TCWak= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505/go.mod h1:5rqRJ9zGR+KnKbkGx373WgN8xJpvAj99kHnfoDYRO5I= github.com/vmware/govmomi v0.52.0 h1:JyxQ1IQdllrY7PJbv2am9mRsv3p9xWlIQ66bv+XnyLw= diff --git a/pkg/services/network/netop_provider.go b/pkg/services/network/netop_provider.go index fa1c1860fa..e13de3bd4d 100644 --- a/pkg/services/network/netop_provider.go +++ b/pkg/services/network/netop_provider.go @@ -136,7 +136,7 @@ func (np *netopNetworkProvider) ConfigureVirtualMachine(ctx context.Context, clu // Set the VM primary interface vm.Spec.Network.Interfaces = append(vm.Spec.Network.Interfaces, vmoprv1.VirtualMachineNetworkInterfaceSpec{ Name: PrimaryInterfaceName, - Network: vmoprv1common.PartialObjectRef{ + Network: &vmoprv1common.PartialObjectRef{ TypeMeta: metav1.TypeMeta{ Kind: NetworkGVKNetOperator.Kind, APIVersion: NetworkGVKNetOperator.GroupVersion().String(), diff --git a/pkg/services/network/nsxt_provider.go b/pkg/services/network/nsxt_provider.go index 96a0450bb7..90885cb568 100644 --- a/pkg/services/network/nsxt_provider.go +++ b/pkg/services/network/nsxt_provider.go @@ -223,7 +223,7 @@ func (np *nsxtNetworkProvider) ConfigureVirtualMachine(_ context.Context, cluste } vm.Spec.Network.Interfaces = append(vm.Spec.Network.Interfaces, vmoprv1.VirtualMachineNetworkInterfaceSpec{ Name: fmt.Sprintf("eth%d", len(vm.Spec.Network.Interfaces)), - Network: vmoprv1common.PartialObjectRef{ + Network: &vmoprv1common.PartialObjectRef{ TypeMeta: metav1.TypeMeta{ Kind: NetworkGVKNSXT.Kind, APIVersion: NetworkGVKNSXT.GroupVersion().String(), diff --git a/pkg/services/network/nsxt_vpc_provider.go b/pkg/services/network/nsxt_vpc_provider.go index 0c3533a37c..9b2c8defa0 100644 --- a/pkg/services/network/nsxt_vpc_provider.go +++ b/pkg/services/network/nsxt_vpc_provider.go @@ -224,7 +224,7 @@ func (vp *nsxtVPCNetworkProvider) ConfigureVirtualMachine(_ context.Context, clu networkName := clusterCtx.VSphereCluster.Name vm.Spec.Network.Interfaces = append(vm.Spec.Network.Interfaces, vmoprv1.VirtualMachineNetworkInterfaceSpec{ Name: PrimaryInterfaceName, - Network: vmoprv1common.PartialObjectRef{ + Network: &vmoprv1common.PartialObjectRef{ TypeMeta: metav1.TypeMeta{ Kind: NetworkGVKNSXTVPCSubnetSet.Kind, APIVersion: NetworkGVKNSXTVPCSubnetSet.GroupVersion().String(), @@ -243,7 +243,7 @@ func (vp *nsxtVPCNetworkProvider) ConfigureVirtualMachine(_ context.Context, clu } vmInterface := vmoprv1.VirtualMachineNetworkInterfaceSpec{ Name: PrimaryInterfaceName, - Network: vmoprv1common.PartialObjectRef{ + Network: &vmoprv1common.PartialObjectRef{ TypeMeta: metav1.TypeMeta{ Kind: primary.Network.Kind, APIVersion: primary.Network.APIVersion, @@ -281,7 +281,7 @@ func setVMSecondaryInterfaces(machine *vmwarev1.VSphereMachine, vm *vmoprv1.Virt } vmInterface := vmoprv1.VirtualMachineNetworkInterfaceSpec{ Name: secondaryInterface.Name, - Network: vmoprv1common.PartialObjectRef{ + Network: &vmoprv1common.PartialObjectRef{ TypeMeta: metav1.TypeMeta{ Kind: secondaryInterface.Network.Kind, APIVersion: secondaryInterface.Network.APIVersion, diff --git a/pkg/services/vmoperator/constants.go b/pkg/services/vmoperator/constants.go index 011082a06c..37ca556fc6 100644 --- a/pkg/services/vmoperator/constants.go +++ b/pkg/services/vmoperator/constants.go @@ -18,8 +18,6 @@ limitations under the License. package vmoperator const ( - kubeTopologyZoneLabelKey = "topology.kubernetes.io/zone" - // ControlPlaneVMClusterModuleGroupName is the name used for the control plane Cluster Module. ControlPlaneVMClusterModuleGroupName = "control-plane-group" // ClusterModuleNameAnnotationKey is key for the Cluster Module annotation. diff --git a/pkg/services/vmoperator/control_plane_endpoint.go b/pkg/services/vmoperator/control_plane_endpoint.go index e0070188e3..3b500711d7 100644 --- a/pkg/services/vmoperator/control_plane_endpoint.go +++ b/pkg/services/vmoperator/control_plane_endpoint.go @@ -189,7 +189,7 @@ func newVirtualMachineService(ctx *vmware.ClusterContext) *vmoprv1.VirtualMachin Namespace: ctx.Cluster.Namespace, }, TypeMeta: metav1.TypeMeta{ - APIVersion: vmoprv1.SchemeGroupVersion.String(), + APIVersion: vmoprv1.GroupVersion.String(), Kind: "VirtualMachineService", }, } diff --git a/pkg/services/vmoperator/vmopmachine.go b/pkg/services/vmoperator/vmopmachine.go index 840b166406..4a7ba23188 100644 --- a/pkg/services/vmoperator/vmopmachine.go +++ b/pkg/services/vmoperator/vmopmachine.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "sort" "github.com/pkg/errors" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" @@ -30,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apitypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/klog/v2" "k8s.io/utils/ptr" clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -41,11 +43,17 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + "sigs.k8s.io/cluster-api-provider-vsphere/feature" capvcontext "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/vmware" infrautilv1 "sigs.k8s.io/cluster-api-provider-vsphere/pkg/util" ) +const ( + // ZoneAnnotationPrefix is the prefix used for placement decision annotations which will be set on VirtualMachineGroup. + ZoneAnnotationPrefix = "zone.vmware.infrastructure.cluster.x-k8s.io" +) + // VmopMachineService reconciles VM Operator VM. type VmopMachineService struct { Client client.Client @@ -163,6 +171,13 @@ func (v *VmopMachineService) SyncFailureReason(_ context.Context, machineCtx cap return supervisorMachineCtx.VSphereMachine.Status.FailureReason != nil || supervisorMachineCtx.VSphereMachine.Status.FailureMessage != nil, nil } +// affinityInfo is an internal struct used to store information about VM affinity. +type affinityInfo struct { + affinitySpec vmoprv1.AffinitySpec + vmGroupName string + failureDomain string +} + // ReconcileNormal reconciles create and update events for VM Operator VMs. func (v *VmopMachineService) ReconcileNormal(ctx context.Context, machineCtx capvcontext.MachineContext) (bool, error) { log := ctrl.LoggerFrom(ctx) @@ -171,10 +186,6 @@ func (v *VmopMachineService) ReconcileNormal(ctx context.Context, machineCtx cap return false, errors.New("received unexpected SupervisorMachineContext type") } - if supervisorMachineCtx.Machine.Spec.FailureDomain != "" { - supervisorMachineCtx.VSphereMachine.Spec.FailureDomain = ptr.To(supervisorMachineCtx.Machine.Spec.FailureDomain) - } - // If debug logging is enabled, report the number of vms in the cluster before and after the reconcile if log.V(5).Enabled() { vms, err := v.getVirtualMachinesInCluster(ctx, supervisorMachineCtx) @@ -188,27 +199,161 @@ func (v *VmopMachineService) ReconcileNormal(ctx context.Context, machineCtx cap // Set the VM state. Will get reset throughout the reconcile supervisorMachineCtx.VSphereMachine.Status.VMStatus = vmwarev1.VirtualMachineStatePending - // Check for the presence of an existing object + // Get the VirtualMachine object Key vmOperatorVM := &vmoprv1.VirtualMachine{} - key, err := virtualMachineObjectKey(supervisorMachineCtx.Machine.Name, supervisorMachineCtx.Machine.Namespace, supervisorMachineCtx.VSphereMachine.Spec.NamingStrategy) + vmKey, err := virtualMachineObjectKey(supervisorMachineCtx.Machine.Name, supervisorMachineCtx.Machine.Namespace, supervisorMachineCtx.VSphereMachine.Spec.NamingStrategy) if err != nil { return false, err } - if err := v.Client.Get(ctx, *key, vmOperatorVM); err != nil { + + // When creating a new cluster and the user doesn't provide info about placement of VMs in a specific failure domain, + // CAPV will define affinity rules to ensure proper placement of the machine. + // + // - All the machines belonging to the same MachineDeployment should be placed in the same failure domain - required. + // - All the machines belonging to the same MachineDeployment should be spread across esxi hosts in the same failure domain - best-efforts. + // - Different MachineDeployments and corresponding VMs should be spread across failure domains - best-efforts. + // + // Note: Control plane VM placement doesn't follow the above rules, and the assumption + // is that failureDomain is always set for control plane VMs. + var affInfo *affinityInfo + if feature.Gates.Enabled(feature.NodeAutoPlacement) && + !infrautilv1.IsControlPlaneMachine(machineCtx.GetVSphereMachine()) { + vmGroup := &vmoprv1.VirtualMachineGroup{} + key := client.ObjectKey{ + Namespace: supervisorMachineCtx.Cluster.Namespace, + Name: supervisorMachineCtx.Cluster.Name, + } + err := v.Client.Get(ctx, key, vmGroup) + + // The VirtualMachineGroup controller is going to create the vmg only when all the VSphereMachines required for the placement + // decision exist. If the vmg does not exist yet, requeue. + if err != nil { + if !apierrors.IsNotFound(err) { + return false, err + } + + v1beta2conditions.Set(supervisorMachineCtx.VSphereMachine, metav1.Condition{ + Type: infrav1.VSphereMachineVirtualMachineProvisionedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: infrav1.VSphereMachineVirtualMachineWaitingForVirtualMachineGroupV1Beta2Reason, + Message: fmt.Sprintf("Waiting for VSphereMachine's VirtualMachineGroup %s to exist", key), + }) + log.V(4).Info(fmt.Sprintf("Waiting for VirtualMachineGroup %s, requeueing", key.Name), "VirtualMachineGroup", klog.KRef(key.Namespace, key.Name)) + return true, nil + } + + // The VirtualMachineGroup controller is going to add a VM in the vmg only when the creation of this + // VM does not impact the placement decision. If the VM is not yet included in the member list, requeue. + isMember := v.checkVirtualMachineGroupMembership(vmGroup, vmKey.Name) + if !isMember { + v1beta2conditions.Set(supervisorMachineCtx.VSphereMachine, metav1.Condition{ + Type: infrav1.VSphereMachineVirtualMachineProvisionedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: infrav1.VSphereMachineVirtualMachineWaitingForVirtualMachineGroupV1Beta2Reason, + Message: fmt.Sprintf("Waiting for VirtualMachineGroup %s membership", klog.KRef(key.Namespace, key.Name)), + }) + log.V(4).Info(fmt.Sprintf("Waiting for VirtualMachineGroup %s membership, requeueing", key.Name), "VirtualMachineGroup", klog.KRef(key.Namespace, key.Name)) + return true, nil + } + + affInfo = &affinityInfo{ + vmGroupName: vmGroup.Name, + } + + // Set the zone label using the annotation of the per-md zone mapping from VirtualMachineGroup. + // This is for new VMs created during day-2 operations when Node Auto Placement is enabled. + mdName := supervisorMachineCtx.Machine.Labels[clusterv1.MachineDeploymentNameLabel] + if fd, ok := vmGroup.Annotations[fmt.Sprintf("%s/%s", ZoneAnnotationPrefix, mdName)]; ok && fd != "" { + affInfo.failureDomain = fd + } + + // VM in a MachineDeployment ideally should be placed in a different failure domain than VMs + // in other MachineDeployments. + // In order to do so, collect names of all the MachineDeployments except the one the VM belongs to. + machineDeployments := &clusterv1.MachineDeploymentList{} + if err := v.Client.List(ctx, machineDeployments, + client.InNamespace(supervisorMachineCtx.Cluster.Namespace), + client.MatchingLabels{clusterv1.ClusterNameLabel: supervisorMachineCtx.Cluster.Name}); err != nil { + return false, err + } + othermMDNames := []string{} + for _, machineDeployment := range machineDeployments.Items { + if machineDeployment.Spec.Template.Spec.FailureDomain == "" && machineDeployment.Name != mdName { + othermMDNames = append(othermMDNames, machineDeployment.Name) + } + } + sort.Strings(othermMDNames) + + affInfo.affinitySpec = vmoprv1.AffinitySpec{ + VMAffinity: &vmoprv1.VMAffinitySpec{ + // All the machines belonging to the same MachineDeployment should be placed in the same failure domain - required. + RequiredDuringSchedulingPreferredDuringExecution: []vmoprv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + clusterv1.MachineDeploymentNameLabel: mdName, + }, + }, + TopologyKey: corev1.LabelTopologyZone, + }, + }, + }, + VMAntiAffinity: &vmoprv1.VMAntiAffinitySpec{ + // All the machines belonging to the same MachineDeployment should be spread across esxi hosts in the same failure domain - best-efforts. + PreferredDuringSchedulingPreferredDuringExecution: []vmoprv1.VMAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + clusterv1.MachineDeploymentNameLabel: mdName, + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + }, + } + if len(othermMDNames) > 0 { + // Different MachineDeployments and corresponding VMs should be spread across failure domains - best-efforts. + affInfo.affinitySpec.VMAntiAffinity.PreferredDuringSchedulingPreferredDuringExecution = append( + affInfo.affinitySpec.VMAntiAffinity.PreferredDuringSchedulingPreferredDuringExecution, + vmoprv1.VMAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: clusterv1.MachineDeploymentNameLabel, + Operator: metav1.LabelSelectorOpIn, + Values: othermMDNames, + }, + }, + }, + TopologyKey: corev1.LabelTopologyZone, + }, + ) + } + } + + // If the failureDomain is explicitly define for a machine, forward this info to the VM. + // Note: for consistency, affinity rules will be set on all the VMs, no matter if they are explicitly assigned to a failureDomain or not. + if supervisorMachineCtx.Machine.Spec.FailureDomain != "" { + supervisorMachineCtx.VSphereMachine.Spec.FailureDomain = ptr.To(supervisorMachineCtx.Machine.Spec.FailureDomain) + } + + // Check for the presence of an existing object + if err := v.Client.Get(ctx, *vmKey, vmOperatorVM); err != nil { if !apierrors.IsNotFound(err) { return false, err } // Define the VM Operator VirtualMachine resource to reconcile. vmOperatorVM = &vmoprv1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, + Name: vmKey.Name, + Namespace: vmKey.Namespace, }, } } // Reconcile the VM Operator VirtualMachine. - if err := v.reconcileVMOperatorVM(ctx, supervisorMachineCtx, vmOperatorVM); err != nil { + if err := v.reconcileVMOperatorVM(ctx, supervisorMachineCtx, vmOperatorVM, affInfo); err != nil { v1beta1conditions.MarkFalse(supervisorMachineCtx.VSphereMachine, infrav1.VMProvisionedCondition, vmwarev1.VMCreationFailedReason, clusterv1beta1.ConditionSeverityWarning, "failed to create or update VirtualMachine: %v", err) v1beta2conditions.Set(supervisorMachineCtx.VSphereMachine, metav1.Condition{ @@ -378,7 +523,7 @@ func (v *VmopMachineService) GetHostInfo(ctx context.Context, machineCtx capvcon return vmOperatorVM.Status.Host, nil } -func (v *VmopMachineService) reconcileVMOperatorVM(ctx context.Context, supervisorMachineCtx *vmware.SupervisorMachineContext, vmOperatorVM *vmoprv1.VirtualMachine) error { +func (v *VmopMachineService) reconcileVMOperatorVM(ctx context.Context, supervisorMachineCtx *vmware.SupervisorMachineContext, vmOperatorVM *vmoprv1.VirtualMachine, affinityInfo *affinityInfo) error { // All Machine resources should define the version of Kubernetes to use. if supervisorMachineCtx.Machine.Spec.Version == "" { return errors.Errorf( @@ -472,7 +617,7 @@ func (v *VmopMachineService) reconcileVMOperatorVM(ctx context.Context, supervis } // Assign the VM's labels. - vmOperatorVM.Labels = getVMLabels(supervisorMachineCtx, vmOperatorVM.Labels) + vmOperatorVM.Labels = getVMLabels(supervisorMachineCtx, vmOperatorVM.Labels, affinityInfo) addResourcePolicyAnnotations(supervisorMachineCtx, vmOperatorVM) @@ -494,6 +639,23 @@ func (v *VmopMachineService) reconcileVMOperatorVM(ctx context.Context, supervis vmOperatorVM = typedModified } + // Set VM Affinity rules and GroupName. + // The Affinity rules set in Spec.Affinity primarily take effect only during the + // initial placement. + // These rules DO NOT impact new VMs created after initial placement, such as scaling up, + // because placement relies on information derived from + // VirtualMachineGroup annotations. This ensures all the VMs + // for a MachineDeployment are placed in the same failureDomain. + // Note: no matter of the different placement behaviour, we are setting affinity rules on all machines for consistency. + if affinityInfo != nil { + if vmOperatorVM.Spec.Affinity == nil { + vmOperatorVM.Spec.Affinity = &affinityInfo.affinitySpec + } + if vmOperatorVM.Spec.GroupName == "" { + vmOperatorVM.Spec.GroupName = affinityInfo.vmGroupName + } + } + // Make sure the VSphereMachine owns the VM Operator VirtualMachine. if err := ctrlutil.SetControllerReference(supervisorMachineCtx.VSphereMachine, vmOperatorVM, v.Client.Scheme()); err != nil { return errors.Wrapf(err, "failed to mark %s %s/%s as owner of %s %s/%s", @@ -731,11 +893,15 @@ func (v *VmopMachineService) addVolumes(ctx context.Context, supervisorMachineCt // which is required when the cluster has multiple (3) zones. // Single zone clusters (legacy/default) do not support zonal storage and must not // have the zone annotation set. + // + // However, with Node Auto Placement enabled, failureDomain is optional and CAPV no longer + // sets PVC annotations when creating worker VMs. PVC placement now follows the StorageClass behavior (Immediate or WaitForFirstConsumer). + // Control Plane VMs will still have failureDomain set, and we will set PVC annotation. zonal := len(supervisorMachineCtx.VSphereCluster.Status.FailureDomains) > 1 if zone := supervisorMachineCtx.VSphereMachine.Spec.FailureDomain; zonal && zone != nil { topology := []map[string]string{ - {kubeTopologyZoneLabelKey: *zone}, + {corev1.LabelTopologyZone: *zone}, } b, err := json.Marshal(topology) if err != nil { @@ -777,7 +943,7 @@ func (v *VmopMachineService) addVolumes(ctx context.Context, supervisorMachineCt } // getVMLabels returns the labels applied to a VirtualMachine. -func getVMLabels(supervisorMachineCtx *vmware.SupervisorMachineContext, vmLabels map[string]string) map[string]string { +func getVMLabels(supervisorMachineCtx *vmware.SupervisorMachineContext, vmLabels map[string]string, affinityInfo *affinityInfo) map[string]string { if vmLabels == nil { vmLabels = map[string]string{} } @@ -789,9 +955,16 @@ func getVMLabels(supervisorMachineCtx *vmware.SupervisorMachineContext, vmLabels vmLabels[k] = v } - // Get the labels that determine the VM's placement inside of a stretched - // cluster. - topologyLabels := getTopologyLabels(supervisorMachineCtx) + // Set the labels that determine the VM's placement. + // Note: if the failureDomain is not set, auto placement will happen according to affinity rules on VM during initial Cluster creation. + // For VM created during day-2 operation like scaling up, we should expect the failureDomain to be always set. + // Note: It is important that the value zone label is set on a vm must never change once it is set, + // because the zone in the VirtualMachineGroup might change in case this info is derived from spec.template.spec.failureDomain. + var failureDomain string + if affinityInfo != nil && affinityInfo.failureDomain != "" { + failureDomain = affinityInfo.failureDomain + } + topologyLabels := getTopologyLabels(supervisorMachineCtx, failureDomain) for k, v := range topologyLabels { vmLabels[k] = v } @@ -800,6 +973,11 @@ func getVMLabels(supervisorMachineCtx *vmware.SupervisorMachineContext, vmLabels // resources associated with the target cluster. vmLabels[clusterv1.ClusterNameLabel] = supervisorMachineCtx.GetClusterContext().Cluster.Name + // Ensure the VM has the machine deployment name label + if !infrautilv1.IsControlPlaneMachine(supervisorMachineCtx.Machine) { + vmLabels[clusterv1.MachineDeploymentNameLabel] = supervisorMachineCtx.Machine.Labels[clusterv1.MachineDeploymentNameLabel] + } + return vmLabels } @@ -809,10 +987,17 @@ func getVMLabels(supervisorMachineCtx *vmware.SupervisorMachineContext, vmLabels // // and thus the code is optimized as such. However, in the future // this function may return a more diverse topology. -func getTopologyLabels(supervisorMachineCtx *vmware.SupervisorMachineContext) map[string]string { +func getTopologyLabels(supervisorMachineCtx *vmware.SupervisorMachineContext, failureDomain string) map[string]string { + // This is for explicit placement. if fd := supervisorMachineCtx.VSphereMachine.Spec.FailureDomain; fd != nil && *fd != "" { return map[string]string{ - kubeTopologyZoneLabelKey: *fd, + corev1.LabelTopologyZone: *fd, + } + } + // This is for automatic placement. + if failureDomain != "" { + return map[string]string{ + corev1.LabelTopologyZone: failureDomain, } } return nil @@ -823,3 +1008,16 @@ func getTopologyLabels(supervisorMachineCtx *vmware.SupervisorMachineContext) ma func getMachineDeploymentNameForCluster(cluster *clusterv1.Cluster) string { return fmt.Sprintf("%s-workers-0", cluster.Name) } + +// checkVirtualMachineGroupMembership checks if the machine is in the first boot order group +// and performs logic if a match is found, as first boot order contains all the worker VMs. +func (v *VmopMachineService) checkVirtualMachineGroupMembership(vmOperatorVMGroup *vmoprv1.VirtualMachineGroup, virtualMachineName string) bool { + if len(vmOperatorVMGroup.Spec.BootOrder) > 0 { + for _, member := range vmOperatorVMGroup.Spec.BootOrder[0].Members { + if member.Name == virtualMachineName { + return true + } + } + } + return false +} diff --git a/pkg/services/vmoperator/vmopmachine_test.go b/pkg/services/vmoperator/vmopmachine_test.go index aa91556341..f2f80789ad 100644 --- a/pkg/services/vmoperator/vmopmachine_test.go +++ b/pkg/services/vmoperator/vmopmachine_test.go @@ -18,6 +18,8 @@ package vmoperator import ( "context" + "fmt" + "slices" "testing" "time" @@ -32,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/utils/ptr" clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -40,6 +43,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + "sigs.k8s.io/cluster-api-provider-vsphere/feature" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/vmware" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/network" @@ -65,6 +69,49 @@ func updateReconciledVMStatus(ctx context.Context, vmService VmopMachineService, Expect(err).ShouldNot(HaveOccurred()) } +func verifyVMAffinityRules(vmopVM *vmoprv1.VirtualMachine, machineDeploymentName string) { + Expect(vmopVM.Spec.Affinity.VMAffinity).ShouldNot(BeNil()) + Expect(vmopVM.Spec.Affinity.VMAffinity.RequiredDuringSchedulingPreferredDuringExecution).To(HaveLen(1)) + + vmAffinityTerm := vmopVM.Spec.Affinity.VMAffinity.RequiredDuringSchedulingPreferredDuringExecution[0] + Expect(vmAffinityTerm.LabelSelector.MatchLabels).To(HaveKeyWithValue(clusterv1.MachineDeploymentNameLabel, machineDeploymentName)) + Expect(vmAffinityTerm.TopologyKey).To(Equal(corev1.LabelTopologyZone)) +} + +func verifyVMAntiAffinityRules(vmopVM *vmoprv1.VirtualMachine, machineDeploymentName string, extraMDs ...string) { + Expect(vmopVM.Spec.Affinity.VMAntiAffinity).ShouldNot(BeNil()) + + expectedNumAntiAffinityTerms := 1 + if len(extraMDs) > 0 { + expectedNumAntiAffinityTerms = 2 + } + + antiAffinityTerms := vmopVM.Spec.Affinity.VMAntiAffinity.PreferredDuringSchedulingPreferredDuringExecution + Expect(antiAffinityTerms).To(HaveLen(expectedNumAntiAffinityTerms)) + + // First anti-affinity constraint - same machine deployment, different hosts + antiAffinityTerm1 := antiAffinityTerms[0] + Expect(antiAffinityTerm1.LabelSelector.MatchLabels).To(HaveKeyWithValue(clusterv1.MachineDeploymentNameLabel, machineDeploymentName)) + Expect(antiAffinityTerm1.TopologyKey).To(Equal(corev1.LabelHostname)) + + // Second anti-affinity term - different machine deployments + if len(extraMDs) > 0 { + isSortedAlphabetically := func(actual []string) (bool, error) { + return slices.IsSorted(actual), nil + } + antiAffinityTerm2 := antiAffinityTerms[1] + Expect(antiAffinityTerm2.LabelSelector.MatchExpressions).To(HaveLen(1)) + Expect(antiAffinityTerm2.LabelSelector.MatchExpressions[0].Key).To(Equal(clusterv1.MachineDeploymentNameLabel)) + Expect(antiAffinityTerm2.LabelSelector.MatchExpressions[0].Operator).To(Equal(metav1.LabelSelectorOpIn)) + + Expect(antiAffinityTerm2.LabelSelector.MatchExpressions[0].Values).To(HaveLen(len(extraMDs))) + Expect(antiAffinityTerm2.LabelSelector.MatchExpressions[0].Values).To( + WithTransform(isSortedAlphabetically, BeTrue()), + "Expected extra machine deployments to be sorted alphabetically", + ) + } +} + const ( machineName = "test-machine" clusterName = "test-cluster" @@ -81,6 +128,32 @@ const ( clusterNameLabel = clusterv1.ClusterNameLabel ) +func createMachineDeployment(name, namespace, clusterName, failureDomain string) *clusterv1.MachineDeployment { + md := &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + clusterv1.ClusterNameLabel: clusterName, + }, + }, + Spec: clusterv1.MachineDeploymentSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + // FailureDomain will be set conditionally below + }, + }, + }, + } + + // Only set failure domain if it's provided and not empty + if failureDomain != "" { + md.Spec.Template.Spec.FailureDomain = failureDomain + } + + return md +} + var _ = Describe("VirtualMachine tests", func() { var ( @@ -655,6 +728,347 @@ var _ = Describe("VirtualMachine tests", func() { Expect(vmopVM.Spec.Volumes[i]).To(BeEquivalentTo(vmVolume)) } }) + + Context("With node auto placement feature gate enabled", func() { + BeforeEach(func() { + t := GinkgoT() + featuregatetesting.SetFeatureGateDuringTest(t, feature.Gates, feature.NodeAutoPlacement, true) + }) + + // control plane machine is the machine with the control plane label set + Specify("Reconcile valid control plane Machine", func() { + // Control plane machines should not have auto placement logic applied + expectReconcileError = false + expectVMOpVM = true + expectedImageName = imageName + expectedRequeue = true + + // Provide valid bootstrap data + By("bootstrap data is created") + secretName := machine.GetName() + "-data" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: machine.GetNamespace(), + }, + Data: map[string][]byte{ + "value": []byte(bootstrapData), + }, + } + Expect(vmService.Client.Create(ctx, secret)).To(Succeed()) + + machine.Spec.Bootstrap.DataSecretName = &secretName + expectedConditions = append(expectedConditions, clusterv1beta1.Condition{ + Type: infrav1.VMProvisionedCondition, + Status: corev1.ConditionFalse, + Reason: vmwarev1.VMProvisionStartedReason, + Message: "", + }) + + By("VirtualMachine is created") + requeue, err = vmService.ReconcileNormal(ctx, supervisorMachineContext) + verifyOutput(supervisorMachineContext) + + By("Verify that control plane machine does not have affinity spec set") + vmopVM = getReconciledVM(ctx, vmService, supervisorMachineContext) + Expect(vmopVM).ShouldNot(BeNil()) + Expect(vmopVM.Spec.Affinity).To(BeNil()) + + By("Verify that control plane machine has correct labels") + Expect(vmopVM.Labels[nodeSelectorKey]).To(Equal(roleControlPlane)) + + By("Verify that machine-deployment label is not set for control plane") + Expect(vmopVM.Labels).ToNot(HaveKey(clusterv1.MachineDeploymentNameLabel)) + }) + + Context("For worker machine", func() { + var ( + machineDeploymentName string + vmGroup *vmoprv1.VirtualMachineGroup + ) + + BeforeEach(func() { + // Create a worker machine (no control plane label) + machineDeploymentName = "test-md" + workerMachineName := "test-worker-machine" + machine = util.CreateMachine(workerMachineName, clusterName, k8sVersion, false) + machine.Labels[clusterv1.MachineDeploymentNameLabel] = machineDeploymentName + + vsphereMachine = util.CreateVSphereMachine(workerMachineName, clusterName, className, imageName, storageClass, false) + + clusterContext, controllerManagerContext := util.CreateClusterContext(cluster, vsphereCluster) + supervisorMachineContext = util.CreateMachineContext(clusterContext, machine, vsphereMachine) + supervisorMachineContext.ControllerManagerContext = controllerManagerContext + + // Create a VirtualMachineGroup for the cluster + vmGroup = &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: corev1.NamespaceDefault, + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + { + Name: workerMachineName, + Kind: "VirtualMachine", + }, + }, + }, + }, + }, + } + Expect(vmService.Client.Create(ctx, vmGroup)).To(Succeed()) + + // Create a MachineDeployment for the worker + machineDeployment := createMachineDeployment(machineDeploymentName, corev1.NamespaceDefault, clusterName, "") + Expect(vmService.Client.Create(ctx, machineDeployment)).To(Succeed()) + }) + + Specify("Requeue valid Machine but not a member of the VirtualMachineGroup yet", func() { + machineDeploymentNotMemberName := "test-md-not-member" + workerMachineNotMember := "test-worker-machine-not-member" + machineNotMember := util.CreateMachine(workerMachineNotMember, clusterName, k8sVersion, false) + machineNotMember.Labels[clusterv1.MachineDeploymentNameLabel] = machineDeploymentNotMemberName + + vsphereMachineNotMember := util.CreateVSphereMachine(workerMachineNotMember, clusterName, className, imageName, storageClass, false) + + clusterContext, controllerManagerContext := util.CreateClusterContext(cluster, vsphereCluster) + supervisorMachineContext = util.CreateMachineContext(clusterContext, machineNotMember, vsphereMachineNotMember) + supervisorMachineContext.ControllerManagerContext = controllerManagerContext + + // Create a MachineDeployment for the worker + machineDeploymentNotMember := createMachineDeployment(machineDeploymentNotMemberName, corev1.NamespaceDefault, clusterName, "") + Expect(vmService.Client.Create(ctx, machineDeploymentNotMember)).To(Succeed()) + + expectReconcileError = false + expectVMOpVM = false + expectedImageName = imageName + expectedRequeue = true + + // Provide valid bootstrap data + By("bootstrap data is created") + secretName := machineNotMember.GetName() + "-data" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: machineNotMember.GetNamespace(), + }, + Data: map[string][]byte{ + "value": []byte(bootstrapData), + }, + } + Expect(vmService.Client.Create(ctx, secret)).To(Succeed()) + + machineNotMember.Spec.Bootstrap.DataSecretName = &secretName + + By("VirtualMachine is not created") + requeue, err = vmService.ReconcileNormal(ctx, supervisorMachineContext) + Expect(err).ShouldNot(HaveOccurred()) + Expect(requeue).Should(BeTrue()) + }) + + Specify("Reconcile valid Machine with no failure domain set", func() { + expectReconcileError = false + expectVMOpVM = true + expectedImageName = imageName + expectedRequeue = true + + // Provide valid bootstrap data + By("bootstrap data is created") + secretName := machine.GetName() + "-data" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: machine.GetNamespace(), + }, + Data: map[string][]byte{ + "value": []byte(bootstrapData), + }, + } + Expect(vmService.Client.Create(ctx, secret)).To(Succeed()) + + machine.Spec.Bootstrap.DataSecretName = &secretName + + By("VirtualMachine is created") + requeue, err = vmService.ReconcileNormal(ctx, supervisorMachineContext) + Expect(err).ShouldNot(HaveOccurred()) + Expect(requeue).Should(BeTrue()) + + By("Verify that worker machine has affinity spec set") + vmopVM = getReconciledVM(ctx, vmService, supervisorMachineContext) + Expect(vmopVM).ShouldNot(BeNil()) + Expect(vmopVM.Spec.Affinity).ShouldNot(BeNil()) + + By("Verify VM affinity rules are set correctly") + verifyVMAffinityRules(vmopVM, machineDeploymentName) + + By("Verify VM anti-affinity rules are set correctly") + verifyVMAntiAffinityRules(vmopVM, machineDeploymentName) + + By("Verify that worker machine has machine deployment label set") + Expect(vmopVM.Labels[clusterv1.MachineDeploymentNameLabel]).To(Equal(machineDeploymentName)) + + By("Verify that GroupName is set from VirtualMachineGroup") + Expect(vmopVM.Spec.GroupName).To(Equal(clusterName)) + }) + + Specify("Reconcile machine with failure domain set", func() { + expectReconcileError = false + expectVMOpVM = true + expectedImageName = imageName + expectedRequeue = true + + failureDomainName := "zone-1" + machineDeploymentName := "test-md-with-fd" + workerMachineName := "test-worker-machine-with-fd" + fdClusterName := "test-cluster-fd" + + // Create a separate cluster for this test to avoid VirtualMachineGroup conflicts + fdCluster := util.CreateCluster(fdClusterName) + fdVSphereCluster := util.CreateVSphereCluster(fdClusterName) + fdVSphereCluster.Status.ResourcePolicyName = resourcePolicyName + + // Create a worker machine with failure domain + machine = util.CreateMachine(workerMachineName, fdClusterName, k8sVersion, false) + machine.Labels[clusterv1.MachineDeploymentNameLabel] = machineDeploymentName + machine.Spec.FailureDomain = failureDomainName + + vsphereMachine = util.CreateVSphereMachine(workerMachineName, fdClusterName, className, imageName, storageClass, false) + + fdClusterContext, fdControllerManagerContext := util.CreateClusterContext(fdCluster, fdVSphereCluster) + supervisorMachineContext = util.CreateMachineContext(fdClusterContext, machine, vsphereMachine) + supervisorMachineContext.ControllerManagerContext = fdControllerManagerContext + + // Create a VirtualMachineGroup for the cluster with per-md zone annotation + vmGroup := &vmoprv1.VirtualMachineGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: fdClusterName, + Namespace: corev1.NamespaceDefault, + Annotations: map[string]string{ + fmt.Sprintf("%s/%s", ZoneAnnotationPrefix, machineDeploymentName): failureDomainName, + }, + }, + Spec: vmoprv1.VirtualMachineGroupSpec{ + BootOrder: []vmoprv1.VirtualMachineGroupBootOrderGroup{ + { + Members: []vmoprv1.GroupMember{ + { + Name: workerMachineName, + Kind: "VirtualMachine", + }, + }, + }, + }, + }, + } + Expect(vmService.Client.Create(ctx, vmGroup)).To(Succeed()) + + // Create a MachineDeployment for the worker with no explicit failure domain + machineDeployment := createMachineDeployment(machineDeploymentName, corev1.NamespaceDefault, fdClusterName, "") + Expect(vmService.Client.Create(ctx, machineDeployment)).To(Succeed()) + + // Provide valid bootstrap data + By("bootstrap data is created") + secretName := machine.GetName() + "-data" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: machine.GetNamespace(), + }, + Data: map[string][]byte{ + "value": []byte(bootstrapData), + }, + } + Expect(vmService.Client.Create(ctx, secret)).To(Succeed()) + + machine.Spec.Bootstrap.DataSecretName = &secretName + + By("VirtualMachine is created with auto placement and failure domain") + requeue, err = vmService.ReconcileNormal(ctx, supervisorMachineContext) + Expect(err).ShouldNot(HaveOccurred()) + Expect(requeue).Should(BeTrue()) + + By("Verify that worker machine has affinity spec set") + vmopVM = getReconciledVM(ctx, vmService, supervisorMachineContext) + Expect(vmopVM).ShouldNot(BeNil()) + Expect(vmopVM.Spec.Affinity).ShouldNot(BeNil()) + + By("Verify VM affinity rules are set correctly") + verifyVMAffinityRules(vmopVM, machineDeploymentName) + + By("Verify VM anti-affinity rules are set correctly") + verifyVMAntiAffinityRules(vmopVM, machineDeploymentName) + + By("Verify that worker machine has correct labels including topology") + Expect(vmopVM.Labels[clusterv1.MachineDeploymentNameLabel]).To(Equal(machineDeploymentName)) + Expect(vmopVM.Labels[corev1.LabelTopologyZone]).To(Equal(failureDomainName)) + + By("Verify that GroupName is set from VirtualMachineGroup") + Expect(vmopVM.Spec.GroupName).To(Equal(fdClusterName)) + }) + + Context("For multiple machine deployments", func() { + const ( + otherMdName1 = "other-md-1" + otherMdName2 = "other-md-2" + ) + + BeforeEach(func() { + otherMd1 := createMachineDeployment(otherMdName1, corev1.NamespaceDefault, clusterName, "") + Expect(vmService.Client.Create(ctx, otherMd1)).To(Succeed()) + + otherMd2 := createMachineDeployment(otherMdName2, corev1.NamespaceDefault, clusterName, "") + Expect(vmService.Client.Create(ctx, otherMd2)).To(Succeed()) + + // Create a MachineDeployment with failure domain + otherMdWithFd := createMachineDeployment("other-md-with-fd", corev1.NamespaceDefault, clusterName, "zone-1") + Expect(vmService.Client.Create(ctx, otherMdWithFd)).To(Succeed()) + }) + + Specify("Reconcile valid machine with additional anti-affinity term added", func() { + expectReconcileError = false + expectVMOpVM = true + expectedImageName = imageName + expectedRequeue = true + + // Provide valid bootstrap data + By("bootstrap data is created") + secretName := machine.GetName() + "-data" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: machine.GetNamespace(), + }, + Data: map[string][]byte{ + "value": []byte(bootstrapData), + }, + } + Expect(vmService.Client.Create(ctx, secret)).To(Succeed()) + + machine.Spec.Bootstrap.DataSecretName = &secretName + + By("VirtualMachine is created") + requeue, err = vmService.ReconcileNormal(ctx, supervisorMachineContext) + Expect(err).ShouldNot(HaveOccurred()) + Expect(requeue).Should(BeTrue()) + + By("Verify that worker machine has affinity spec set") + vmopVM = getReconciledVM(ctx, vmService, supervisorMachineContext) + Expect(vmopVM).ShouldNot(BeNil()) + Expect(vmopVM.Spec.Affinity).ShouldNot(BeNil()) + + By("Verify VM affinity rules are set correctly") + verifyVMAffinityRules(vmopVM, machineDeploymentName) + + By("Verify VM anti-affinity rules are set correctly") + verifyVMAntiAffinityRules(vmopVM, machineDeploymentName, otherMdName1, otherMdName2) + }) + }) + }) + + }) }) Context("Delete tests", func() { diff --git a/test/framework/vmoperator/vmoperator.go b/test/framework/vmoperator/vmoperator.go index c80ec76545..2c1e367b01 100644 --- a/test/framework/vmoperator/vmoperator.go +++ b/test/framework/vmoperator/vmoperator.go @@ -534,7 +534,7 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon Namespace: config.Namespace, }, Spec: vmoprv1.VirtualMachineImageSpec{ - ProviderRef: vmoprv1common.LocalObjectRef{ + ProviderRef: &vmoprv1common.LocalObjectRef{ Kind: "ContentLibraryItem", }, }, diff --git a/test/go.mod b/test/go.mod index bcab8743c0..f83674bcbd 100644 --- a/test/go.mod +++ b/test/go.mod @@ -8,15 +8,15 @@ replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.11.0-rc. replace sigs.k8s.io/cluster-api-provider-vsphere => ../ -replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels v0.0.0-20240404200847-de75746a9505 +replace github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels => github.com/vmware-tanzu/vm-operator/pkg/constants/testlabels v1.9.1-0.20251029150609-93918c59a719 -// The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-testsz -replace github.com/vmware-tanzu/vm-operator/api => github.com/vmware-tanzu/vm-operator/api v1.8.6 +// The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-tests +replace github.com/vmware-tanzu/vm-operator/api => github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719 require ( github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d // The version of vm-operator should be kept in sync with the manifests at: config/deployments/integration-tests - github.com/vmware-tanzu/vm-operator/api v1.8.6 + github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719 github.com/vmware/govmomi v0.52.0 ) diff --git a/test/go.sum b/test/go.sum index 8ac8dfd79b..bc5369f066 100644 --- a/test/go.sum +++ b/test/go.sum @@ -360,8 +360,8 @@ github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d h1:c github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d/go.mod h1:JbFOh22iDsT5BowJe0GgpMI5e2/S7cWaJlv9LdURVQM= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20241112044858-9da8637c1b0d h1:z9lrzKVtNlujduv9BilzPxuge/LE2F0N1ms3TP4JZvw= github.com/vmware-tanzu/nsx-operator/pkg/apis v0.0.0-20241112044858-9da8637c1b0d/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= -github.com/vmware-tanzu/vm-operator/api v1.8.6 h1:NIndORjcnSmIlQsCMIewpIwg/ocRVDh2lYjOroTVLrU= -github.com/vmware-tanzu/vm-operator/api v1.8.6/go.mod h1:HHA2SNI9B5Yqtyp5t+Gt9WTWBi/fIkM6+MukDDSf11A= +github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719 h1:nb/5ytRj7E/5eo9UzLfaR29JytMtbGpqMVs3hjaRwZ0= +github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20251029150609-93918c59a719/go.mod h1:nWTPpxfe4gHuuYuFcrs86+NMxfkqPk3a3IlvI8TCWak= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505 h1:y4wXx1FUFqqSgJ/xUOEM1DLS2Uu0KaeLADWpzpioGTU= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505/go.mod h1:5rqRJ9zGR+KnKbkGx373WgN8xJpvAj99kHnfoDYRO5I= github.com/vmware/govmomi v0.52.0 h1:JyxQ1IQdllrY7PJbv2am9mRsv3p9xWlIQ66bv+XnyLw=