🌱 e2e: servicediscovery controller to have an ip to discover in tests (#3405)

* e2e: servicediscovery controller to have an ip to discover in tests

* vcsim: increase default qps and burst

* fix

* fixup

* fix

Author: chrischdi

controllers/vmware/servicediscovery_controller.go

@@ -62,9 +62,7 @@ import (
 const (
 	clusterNotReadyRequeueTime = time.Minute * 2
 
-	supervisorLoadBalancerSvcNamespace = "kube-system"
-	supervisorLoadBalancerSvcName      = "kube-apiserver-lb-svc"
-	supervisorAPIServerPort            = 6443
+	supervisorAPIServerPort = 6443
 
 	supervisorHeadlessSvcNamespace = "default"
 	supervisorHeadlessSvcName      = "supervisor"

controllers/vmware/servicediscovery_controller_suite_test.go

@@ -177,8 +177,8 @@ func newTestSupervisorLBServiceWithHostnameStatus() *corev1.Service {
 func newTestSupervisorLBService() *corev1.Service {
 	return &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      supervisorLoadBalancerSvcName,
-			Namespace: supervisorLoadBalancerSvcNamespace,
+			Name:      vmwarev1.SupervisorLoadBalancerSvcName,
+			Namespace: vmwarev1.SupervisorLoadBalancerSvcNamespace,
 		},
 		Spec: corev1.ServiceSpec{
 			Type: corev1.ServiceTypeLoadBalancer,

test/framework/vmoperator/vmoperator.go

@@ -43,6 +43,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
 	topologyv1 "sigs.k8s.io/cluster-api-provider-vsphere/internal/apis/topology/v1alpha1"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/session"
 	vcsimv1 "sigs.k8s.io/cluster-api-provider-vsphere/test/infrastructure/vcsim/api/v1alpha1"
@@ -361,6 +362,63 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon
 		return retryError
 	}
 
+	// Create the supervisor service in kube-system for the servicediscovery controller to discover and set an IP address
+	// for the headless service. When using kind as management cluster, the cluster-info configmap in kube-public contains a
+	// hostname instead of an IP address which does not work for the servicediscovery controller.
+	supervisorAPIServerVIPService := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      vmwarev1.SupervisorLoadBalancerSvcName,
+			Namespace: vmwarev1.SupervisorLoadBalancerSvcNamespace,
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeLoadBalancer,
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "kube-apiserver",
+					Protocol: corev1.ProtocolTCP,
+					Port:     6443,
+				},
+			},
+		},
+	}
+	_ = wait.PollUntilContextTimeout(ctx, 250*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
+		retryError = nil
+		if err := c.Get(ctx, client.ObjectKeyFromObject(supervisorAPIServerVIPService), supervisorAPIServerVIPService); err != nil {
+			if !apierrors.IsNotFound(err) {
+				retryError = errors.Wrapf(err, "failed to get vm-operator service %s", klog.KObj(supervisorAPIServerVIPService))
+				return false, nil
+			}
+			if err := c.Create(ctx, supervisorAPIServerVIPService); err != nil {
+				retryError = errors.Wrapf(err, "failed to create vm-operator service %s", klog.KObj(supervisorAPIServerVIPService))
+				return false, nil
+			}
+			log.Info("Created vm-operator service", "Service", klog.KObj(supervisorAPIServerVIPService))
+		}
+		return true, nil
+	})
+	if retryError != nil {
+		return retryError
+	}
+	supervisorAPIServerVIPService.Status = corev1.ServiceStatus{
+		LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{
+			// Note: this creates a unusable service. During test no application should try to reach out to this.
+			// 192.0.2.2 is part of the link-local subnet 192.0.2.0/24 which is reserved for documentation and testing.
+			{IP: "192.0.2.2"},
+		}},
+	}
+	_ = wait.PollUntilContextTimeout(ctx, 250*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
+		retryError = nil
+		if err := c.Status().Update(ctx, supervisorAPIServerVIPService); err != nil {
+			retryError = errors.Wrapf(err, "failed to update vm-operator service status %s", klog.KObj(supervisorAPIServerVIPService))
+			return false, nil
+		}
+		log.Info("Updated vm-operator service status", "Service", klog.KObj(supervisorAPIServerVIPService))
+		return true, nil
+	})
+	if retryError != nil {
+		return retryError
+	}
+
 	// Create VirtualMachineClass in K8s
 	for _, vmc := range config.Spec.VirtualMachineClasses {
 		vmClass := &vmoprv1.VirtualMachineClass{

test/infrastructure/vcsim/config/rbac/role.yaml

@@ -23,6 +23,18 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - services/status
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - authentication.k8s.io
   resources:

test/infrastructure/vcsim/controllers/vmip_controller.go

@@ -112,11 +112,11 @@ func (r *vmIPReconciler) ReconcileIP(ctx context.Context) (ctrl.Result, error) {
 				MacAddress: macAddress,
 				Adapter: types.CustomizationIPSettings{
 					Ip: &types.CustomizationFixedIp{
-						IpAddress: "192.168.1.100",
+						IpAddress: "192.0.2.100",
 					},
 					SubnetMask:    "255.255.255.0",
-					Gateway:       []string{"192.168.1.1"},
-					DnsServerList: []string{"192.168.1.1"},
+					Gateway:       []string{"192.0.2.1"},
+					DnsServerList: []string{"192.0.2.1"},
 					DnsDomain:     "ad.domain",
 				},
 			},

test/infrastructure/vcsim/controllers/vmoperatordependencies_controller.go

@@ -41,6 +41,7 @@ type VMOperatorDependenciesReconciler struct {
 
 // +kubebuilder:rbac:groups=vcsim.infrastructure.cluster.x-k8s.io,resources=vmoperatordependencies,verbs=get;list;watch;patch
 // +kubebuilder:rbac:groups=vcsim.infrastructure.cluster.x-k8s.io,resources=vmoperatordependencies/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups="",resources=services;services/status,verbs=get;list;watch;create;update;patch
 
 func (r *VMOperatorDependenciesReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) {
 	// Fetch the VMOperatorDependencies instance

test/infrastructure/vcsim/main.go

@@ -159,10 +159,10 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
 
-	fs.Float32Var(&restConfigQPS, "kube-api-qps", 20,
+	fs.Float32Var(&restConfigQPS, "kube-api-qps", 100,
 		"Maximum queries per second from the controller client to the Kubernetes API server.")
 
-	fs.IntVar(&restConfigBurst, "kube-api-burst", 30,
+	fs.IntVar(&restConfigBurst, "kube-api-burst", 200,
 		"Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server.")
 
 	fs.StringVar(&healthAddr, "health-addr", ":9440",