🌱 Add e2e scalability test with fake systems

Makefile

@@ -173,6 +173,8 @@ E2E_TEMPLATES_DIR ?= $(ROOT_DIR)/test/e2e/data/infrastructure-metal3
 cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
+	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-fake > $(E2E_OUT_DIR)/cluster-template-centos-fake.yaml
+	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-metal3 > $(E2E_OUT_DIR)/clusterclass-metal3.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-md-remediation > $(E2E_OUT_DIR)/cluster-template-centos-md-remediation.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu-md-remediation > $(E2E_OUT_DIR)/cluster-template-ubuntu-md-remediation.yaml

scripts/ci-e2e.sh

@@ -34,18 +34,24 @@ export KUBERNETES_VERSION=${KUBERNETES_VERSION}
 export IMAGE_OS=${IMAGE_OS}
 export FORCE_REPO_UPDATE="false"
 EOF
-# if running a clusterctl-upgrade test skip apply bmhs in dev-env
+# if running a scalability test skip apply bmhs in dev-env and run fakeIPA
 if [[ ${GINKGO_FOCUS:-} == "clusterctl-upgrade" ]]; then
     echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
 fi
 if [[ ${GINKGO_FOCUS:-} == "features" ]]; then
     mkdir -p "$CAPI_CONFIG_FOLDER"
     echo "ENABLE_BMH_NAME_BASED_PREALLOCATION: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
 fi
-# if running a scalability test skip apply bmhs in dev-env and run fakeIPA
+# if running a scalability tests, configure dev-env with fakeIPA
 if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then
-    echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
     echo 'export NODES_PLATFORM="fake"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
+    echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
+    mkdir -p "$CAPI_CONFIG_FOLDER"
+    echo 'CLUSTER_TOPOLOGY: true' >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
+    echo 'export EPHEMERAL_CLUSTER="minikube"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
+else
+    # Don't run scalability tests if not asked for.
+    export GINKGO_SKIP="${GINKGO_SKIP:-} scalability"
 fi
 # Run make devenv to boot the source cluster
 pushd "${M3_DEV_ENV_PATH}" || exit 1

scripts/environment.sh

@@ -64,8 +64,11 @@ fi
 
 # Scalability test environment vars and config
 if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then
-  export NUM_NODES=${NUM_NODES:-"100"}
-  export BMH_BATCH_SIZE=${BMH_BATCH_SIZE:-"20"}
+  export NUM_NODES=${NUM_NODES:-"10"}
+  export BMH_BATCH_SIZE=${BMH_BATCH_SIZE:-"2"}
+  export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-"1"}
+  export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"0"}
+  export KUBERNETES_VERSION_UPGRADE_FROM=${FROM_K8S_VERSION}
 fi
 
 # Integration test environment vars and config

test/e2e/config/e2e_conf.yaml

@@ -188,8 +188,13 @@ providers:
       targetName: "cluster-template-ubuntu.yaml"
     - sourcePath: "../_out/cluster-template-centos.yaml"
       targetName: "cluster-template-centos.yaml"
+    - sourcePath: "../_out/cluster-template-centos-fake.yaml"
+      targetName: "cluster-template-centos-fake.yaml"
+    - sourcePath: "../_out/cluster-template-centos-fake.yaml"
+      targetName: "cluster-template-ubuntu-fake.yaml"
     - sourcePath: "../_out/clusterclass.yaml"
       targetName: "clusterclass-test-clusterclass.yaml"
+    - sourcePath: "../_out/clusterclass-metal3.yaml"
     - sourcePath: "../_out/cluster-template-centos-md-remediation.yaml"
       targetName: "cluster-template-centos-md-remediation.yaml"
     - sourcePath: "../_out/cluster-template-ubuntu-md-remediation.yaml"
@@ -246,7 +251,8 @@ variables:
   BMO_RELEASE_0.6: "data/bmo-deployment/overlays/release-0.6"
   BMO_RELEASE_0.8: "data/bmo-deployment/overlays/release-0.8"
   BMO_RELEASE_LATEST: "data/bmo-deployment/overlays/release-latest"
-
+  FKAS_RELEASE_LATEST: "data/fkas"
+
 intervals:
   default/wait-controllers: ["10m", "10s"]
   default/wait-cluster: ["20m", "30s"] # The second time to check the availibility of the cluster should happen late, so kcp object has time to be created

test/e2e/data/fkas/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- resources.yaml

test/e2e/data/fkas/resources.yaml

@@ -0,0 +1,77 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: fkas-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metal3-fkas-sa
+  namespace: fkas-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metal3-fkas-role
+rules:
+- apiGroups: ["metal3.io", "infrastructure.cluster.x-k8s.io"]
+  resources: ["baremetalhosts", "metal3machines"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["cluster.x-k8s.io"]
+  resources: ["machines"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metal3-fkas-rolebinding
+subjects:
+- kind: ServiceAccount
+  name: metal3-fkas-sa
+  namespace: fkas-system
+roleRef:
+  kind: ClusterRole
+  name: metal3-fkas-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metal3-fkas-system
+  namespace: fkas-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: metal3-fkas-system
+  template:
+    metadata:
+      labels:
+        app: metal3-fkas-system
+    spec:
+      serviceAccountName: metal3-fkas-sa
+      hostNetwork: true
+      containers:
+      - name: metal3-fkas-reconciler
+        image: quay.io/metal3-io/metal3-fkas:latest
+        imagePullPolicy: IfNotPresent
+        command: ["/reconciler"]
+        env:
+        - name: DEBUG
+          value: "true"
+      - image: quay.io/metal3-io/metal3-fkas:latest
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 3333
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: DEBUG
+          value: "true"
+        name: metal3-fkas

test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/cluster-with-topology.yaml

@@ -0,0 +1,107 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+    labels:
+      cni: ${CLUSTER_NAME}-crs-0
+    name: ${CLUSTER_NAME}
+    namespace: ${NAMESPACE}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks: ["${POD_CIDR}"]
+    services:
+      cidrBlocks: ["${SERVICE_CIDR}"]
+  topology:
+    class: metal3
+    version: ${KUBERNETES_VERSION}
+    controlPlane:
+      replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+    workers:
+      machineDeployments:
+      - class: worker
+        name: ${CLUSTER_NAME}-machine
+        replicas: ${WORKER_MACHINE_COUNT}
+    variables:
+    - name: image
+      value:
+        checksum: ${IMAGE_RAW_CHECKSUM}
+        checksumType: ${IMAGE_CHECKSUM_TYPE}
+        format: raw
+        url: ${IMAGE_RAW_URL}
+    - name: controlPlaneEndpoint
+      value:
+        host: CLUSTER_APIENDPOINT_HOST_HOLDER
+        port: CLUSTER_APIENDPOINT_PORT_HOLDER
+    - name: controlPlaneDataTemplate
+      value: ${CLUSTER_NAME}-controlplane-template
+---
+apiVersion: ipam.metal3.io/v1alpha1
+kind: IPPool
+metadata:
+  name: ${CLUSTER_NAME}-baremetalv4-pool
+spec:
+  clusterName: ${CLUSTER_NAME}
+  gateway: 192.168.111.1
+  namePrefix: ${CLUSTER_NAME}-bmv4
+  pools:
+  - end: 192.168.111.240
+    start: 192.168.111.201
+  prefix: 24
+---
+apiVersion: ipam.metal3.io/v1alpha1
+kind: IPPool
+metadata:
+  name: ${CLUSTER_NAME}-provisioning-pool
+spec:
+  clusterName: ${CLUSTER_NAME}
+  namePrefix: ${CLUSTER_NAME}-prov
+  pools:
+  - end: 172.22.0.240
+    start: 172.22.0.201
+  prefix: 24
+---
+apiVersion: infrastructure.cluster.x-k8s.io/${CAPM3_VERSION}
+kind: Metal3DataTemplate
+metadata:
+  name: ${CLUSTER_NAME}-controlplane-template
+spec:
+  clusterName: ${CLUSTER_NAME}
+  metaData:
+    ipAddressesFromIPPool:
+    - key: provisioningIP
+      name: ${CLUSTER_NAME}-provisioning-pool
+    objectNames:
+    - key: name
+      object: machine
+    - key: local-hostname
+      object: machine
+    - key: local_hostname
+      object: machine
+    prefixesFromIPPool:
+    - key: provisioningCIDR
+      name: ${CLUSTER_NAME}-provisioning-pool
+  networkData:
+    links:
+      ethernets:
+      - id: enp1s0
+        macAddress:
+          fromHostInterface: enp1s0
+        type: phy
+      - id: enp2s0
+        macAddress:
+          fromHostInterface: enp2s0
+        type: phy
+    networks:
+      ipv4:
+      - id: baremetalv4
+        ipAddressFromIPPool: ${CLUSTER_NAME}-baremetalv4-pool
+        link: enp2s0
+        routes:
+        - gateway:
+            fromIPPool: ${CLUSTER_NAME}-baremetalv4-pool
+          network: 0.0.0.0
+          prefix: 0
+    services:
+      dns:
+      - 8.8.8.8

test/e2e/data/infrastructure-metal3/bases/cluster-with-topology/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- cluster-with-topology.yaml

test/e2e/data/infrastructure-metal3/cluster-template-centos-fake/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- ../bases/cluster-with-topology