PCP-5152: Multiple node HCP cluster losing interface reference

controllers/lxd_initializer_ds.go

@@ -6,12 +6,17 @@ import (
 	"strings"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/yaml"
 
 	"github.com/spectrocloud/cluster-api-provider-maas/pkg/maas/scope"
+	"github.com/spectrocloud/cluster-api-provider-maas/pkg/util/trust"
+	"github.com/spectrocloud/cluster-api-provider-maas/pkg/util"
 
 	// embed template
 	_ "embed"
@@ -43,7 +48,107 @@ func (r *MaasClusterReconciler) ensureLXDInitializerDS(ctx context.Context, clus
 		return fmt.Errorf("failed to get target cluster client: %v", err)
 	}
 
-	// First clean up any existing DaemonSets in this namespace
+	// If feature is off or cluster is being deleted, we're done
+	if !clusterScope.IsLXDHostEnabled() || !cluster.ObjectMeta.DeletionTimestamp.IsZero() {
+		return nil
+	}
+
+	// Gate: ensure pivot completed. Require mgmt namespace to have clusterEnv=target
+	mgmtNS := &corev1.Namespace{}
+	if err := r.Client.Get(ctx, client.ObjectKey{Name: dsNamespace}, mgmtNS); err != nil {
+		// Namespace not readable yet on management cluster; skip for now
+		return nil
+	}
+	if v := strings.TrimSpace(mgmtNS.Annotations["clusterEnv"]); v != "target" {
+		r.Log.Info("Namespace not marked as target; deferring LXD initializer", "namespace", dsNamespace, "clusterEnv", v)
+		return nil
+	}
+
+	// Gate: derive desired CP count from MaasCloudConfig; fallback to KCP
+	desiredCP := int32(1)
+	readyByKCP := int32(0)
+
+	// Prefer MaasCloudConfig.spec.machinePoolConfig[].size where isControlPlane=true (sum)
+	{
+		mccList := &unstructured.UnstructuredList{}
+		mccList.SetGroupVersionKind(schema.GroupVersionKind{Group: "cluster.spectrocloud.com", Version: "v1alpha1", Kind: "MaasCloudConfigList"})
+		if err := r.Client.List(ctx, mccList, client.InNamespace(dsNamespace)); err == nil {
+			var sum int64
+			for _, it := range mccList.Items {
+				owned := false
+				for _, or := range it.GetOwnerReferences() {
+					if or.Name == cluster.Name {
+						owned = true
+						break
+					}
+				}
+				if !owned && !strings.HasSuffix(it.GetName(), "-maas-config") {
+					continue
+				}
+				pools, found, _ := unstructured.NestedSlice(it.Object, "spec", "machinePoolConfig")
+				if !found {
+					continue
+				}
+				for _, p := range pools {
+					if mp, ok := p.(map[string]interface{}); ok {
+						isCP, _, _ := unstructured.NestedBool(mp, "isControlPlane")
+						if !isCP {
+							continue
+						}
+						if v, foundSz, _ := unstructured.NestedInt64(mp, "size"); foundSz && v > 0 {
+							sum += v
+						}
+					}
+				}
+				if sum > 0 {
+					desiredCP = util.SafeInt64ToInt32(sum)
+					break
+				}
+			}
+		}
+	}
+	// Fallback: use KCP if MCC not found
+	if desiredCP == 1 {
+		kcpList := &unstructured.UnstructuredList{}
+		kcpList.SetGroupVersionKind(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1beta1", Kind: "KubeadmControlPlaneList"})
+		if err := r.Client.List(ctx, kcpList, client.InNamespace(dsNamespace), client.MatchingLabels{
+			"cluster.x-k8s.io/cluster-name": cluster.Name,
+		}); err == nil {
+			if len(kcpList.Items) > 0 {
+				item := kcpList.Items[0]
+				if v, found, _ := unstructured.NestedInt64(item.Object, "spec", "replicas"); found && v > 0 {
+					desiredCP = util.SafeInt64ToInt32(v)
+				}
+				if v, found, _ := unstructured.NestedInt64(item.Object, "status", "readyReplicas"); found && v >= 0 {
+					readyByKCP = util.SafeInt64ToInt32(v)
+				}
+			}
+		}
+	}
+
+	nodeList := &corev1.NodeList{}
+	cpSelector := labels.SelectorFromSet(labels.Set{
+		"node-role.kubernetes.io/control-plane": "",
+	})
+	if err := remoteClient.List(ctx, nodeList, &client.ListOptions{LabelSelector: cpSelector}); err == nil {
+		ready := 0
+		for _, n := range nodeList.Items {
+			for _, c := range n.Status.Conditions {
+				if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
+					ready++
+					break
+				}
+			}
+		}
+		// Proceed when CP nodes are present and Ready, regardless of KCP readyReplicas
+		if int64(len(nodeList.Items)) < int64(desiredCP) || int64(ready) < int64(desiredCP) {
+			r.Log.Info("Not enough control-plane nodes present/ready yet; skipping DS for now", "desiredCP", desiredCP, "readyByKCP", readyByKCP, "nodeList", len(nodeList.Items), "ready", ready)
+			// Not enough control-plane nodes present/ready yet; skip DS for now
+			return nil
+		}
+	}
+
+	// Clean up any existing DaemonSets in this namespace (old naming/labels)
 	dsList := &appsv1.DaemonSetList{}
 	if err := remoteClient.List(ctx, dsList, client.InNamespace(dsNamespace), client.MatchingLabels{
 		"app": "lxd-initializer",
@@ -58,16 +163,38 @@ func (r *MaasClusterReconciler) ensureLXDInitializerDS(ctx context.Context, clus
 		}
 	}
 
-	// If feature is off or cluster is being deleted, we're done after cleanup
-	if !clusterScope.IsLXDHostEnabled() || !cluster.ObjectMeta.DeletionTimestamp.IsZero() {
-		return nil
-	}
-
 	// Ensure RBAC resources are created on target cluster
 	if err := r.ensureLXDInitializerRBACOnTarget(ctx, remoteClient, dsNamespace); err != nil {
 		return fmt.Errorf("failed to ensure LXD initializer RBAC: %v", err)
 	}
 
+	// Short-circuit deletion: if all control-plane nodes are labeled initialized, delete DS
+	shortCircuitNodes := &corev1.NodeList{}
+	shortCircuitSelector := labels.SelectorFromSet(labels.Set{
+		"node-role.kubernetes.io/control-plane": "",
+	})
+	if err := remoteClient.List(ctx, shortCircuitNodes, &client.ListOptions{LabelSelector: shortCircuitSelector}); err == nil && len(shortCircuitNodes.Items) > 0 {
+		// Count how many CP nodes are initialized
+		initCount := 0
+		for _, n := range shortCircuitNodes.Items {
+			if n.Labels != nil && n.Labels["lxdhost.cluster.com/initialized"] == "true" {
+				initCount++
+			}
+		}
+		// Delete DS only when we see at least desiredCP control-plane nodes
+		// AND desiredCP of them are initialized
+		if int64(len(shortCircuitNodes.Items)) >= int64(desiredCP) && int64(initCount) >= int64(desiredCP) {
+			// Delete existing DSs and return
+			shortCircuitDSList := &appsv1.DaemonSetList{}
+			if err := remoteClient.List(ctx, shortCircuitDSList, client.InNamespace(dsNamespace), client.MatchingLabels{"app": dsName}); err == nil {
+				for _, ds := range shortCircuitDSList.Items {
+					_ = remoteClient.Delete(ctx, &ds)
+				}
+			}
+			return nil
+		}
+	}
+
 	// pull LXD config
 	cfg := clusterScope.GetLXDConfig()
 	sb := cfg.StorageBackend
@@ -85,8 +212,12 @@ func (r *MaasClusterReconciler) ensureLXDInitializerDS(ctx context.Context, clus
 	}
 
 	nt := cfg.NICType
+	if nt == "" {
+		nt = "bridged"
+	}
 	np := cfg.NICParent
-	tp := "capmaas"
+	// Deterministic per-cluster trust password derived from cluster UID
+	tp := trust.DeriveTrustPassword(string(cluster.UID))
 
 	rendered := render(map[string]string{
 		"${STORAGE_BACKEND}":     sb,

controllers/maasmachine_controller.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/spectrocloud/maas-client-go/maasclient"
 	"k8s.io/apimachinery/pkg/runtime"
 
 	"github.com/go-logr/logr"
@@ -225,6 +226,34 @@ func (r *MaasMachineReconciler) reconcileDelete(_ context.Context, machineScope
 		// If MAAS requires VM host removal first, attempt best-effort unregister and retry once
 		if isVMHostRemovalRequiredError(err) {
 			api := clusterScope.GetMaasClientIdentity()
+
+			// For control-plane BM that backs an LXD VM host, force-delete guest VMs to unblock release
+			if clusterScope.IsLXDHostEnabled() && machineScope.IsControlPlane() {
+				ctx := context.Background()
+				client := maasclient.NewAuthenticatedClientSet(api.URL, api.Token)
+				if hosts, herr := client.VMHosts().List(ctx, nil); herr == nil {
+					for _, h := range hosts {
+						if h.HostSystemID() == m.ID {
+							if guests, gerr := h.Machines().List(ctx); gerr == nil {
+								for _, g := range guests {
+									gid := g.SystemID()
+									if gid == "" {
+										continue
+									}
+									// Fetch details to confirm and delete
+									if gm, ge := client.Machines().Machine(gid).Get(ctx); ge == nil {
+										if derr := client.Machines().Machine(gm.SystemID()).Delete(ctx); derr != nil {
+											machineScope.Error(derr, "failed to delete guest VM during host release cleanup", "guestSystemID", gm.SystemID())
+										}
+									}
+								}
+							}
+							break
+						}
+					}
+				}
+			}
+
 			// choose ExternalIP first, then InternalIP
 			nodeIP := getNodeIP(m.Addresses)
 			if nodeIP != "" {

controllers/templates/lxd_initializer_ds.yaml

@@ -73,7 +73,8 @@ spec:
           mountPropagation: HostToContainer
       containers:
       - name: lxd-initializer
-        image: "us-east1-docker.pkg.dev/spectro-images/dev/cluster-api/capmaas-lxd-initializer:v0.0.1"
+        image: us-east1-docker.pkg.dev/spectro-images/dev/amit/cluster-api/lxd-initializer:v0.6.1-spectro-4.0.0-dev-16102025-01
+        imagePullPolicy: Always
         securityContext:
           privileged: true
         env:

controllers/templates/lxd_initializer_rbac.yaml

@@ -17,7 +17,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get", "patch", "update"]
+  verbs: ["get", "list", "watch", "patch", "update"]
 - apiGroups: [""]
   resources: ["secrets"]
   verbs: ["list"]

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.36.1
 	github.com/pkg/errors v0.9.1
-	github.com/spectrocloud/maas-client-go v0.0.7-beta1
+	github.com/spectrocloud/maas-client-go v0.0.8-beta1
 	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.31.3
 	k8s.io/apiextensions-apiserver v0.31.3

go.sum

@@ -166,8 +166,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
 github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
-github.com/spectrocloud/maas-client-go v0.0.7-beta1 h1:2GryA5JSrjlsvzLaCIGyPfxcaSCPrw7fm8ixMf7aRbY=
-github.com/spectrocloud/maas-client-go v0.0.7-beta1/go.mod h1:CaqAAlh6/xfzc/cDpU8eMG0wqnwx1ODSyXcH86uV7Ww=
+github.com/spectrocloud/maas-client-go v0.0.8-beta1 h1:PCY6M3M9uXZG8dzoe0jNcMnh4nOhJuZBF2C3vsUXp9A=
+github.com/spectrocloud/maas-client-go v0.0.8-beta1/go.mod h1:CaqAAlh6/xfzc/cDpU8eMG0wqnwx1ODSyXcH86uV7Ww=
 github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=
 github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=

lxd-initializer/go.mod

@@ -4,7 +4,7 @@ go 1.24.5
 
 require (
 	github.com/canonical/lxd v0.0.0-20250730070707-c4a122e242bb
-	github.com/spectrocloud/maas-client-go v0.0.6-beta1
+	github.com/spectrocloud/maas-client-go v0.0.8-beta1
 	k8s.io/apimachinery v0.31.3
 	k8s.io/client-go v0.31.3
 )

lxd-initializer/go.sum

@@ -103,8 +103,8 @@ github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
 github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/spectrocloud/maas-client-go v0.0.6-beta1 h1:sajM2xeYEQNe/3ObyIkTxJJEsy2OM9w4loYsmDCzqio=
-github.com/spectrocloud/maas-client-go v0.0.6-beta1/go.mod h1:CaqAAlh6/xfzc/cDpU8eMG0wqnwx1ODSyXcH86uV7Ww=
+github.com/spectrocloud/maas-client-go v0.0.8-beta1 h1:PCY6M3M9uXZG8dzoe0jNcMnh4nOhJuZBF2C3vsUXp9A=
+github.com/spectrocloud/maas-client-go v0.0.8-beta1/go.mod h1:CaqAAlh6/xfzc/cDpU8eMG0wqnwx1ODSyXcH86uV7Ww=
 github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M=
 github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

lxd-initializer/lxd-initializer-daemonset.yaml

@@ -20,7 +20,7 @@ spec:
       hostPID: true
       containers:
       - name: lxd-initializer
-        image: us-east1-docker.pkg.dev/spectro-images/dev/cluster-api/capmaas-lxd-initializer:v0.0.1
+        image: us-east1-docker.pkg.dev/spectro-images/dev/amit/cluster-api/lxd-initializer:v0.6.1-spectro-4.0.0-dev-16102025-01
         imagePullPolicy: Always
         securityContext:
           privileged: true