Skip to content

🌱Add Kubernetes N+3 upgrade test in e2e #2915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

🌱Add Kubernetes N+3 upgrade test in e2e #2915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions scripts/ci-e2e.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ if [[ "${CAPI_NIGHTLY_BUILD:-false}" == "true" ]]; then
fi

case "${GINKGO_FOCUS:-}" in
clusterctl-upgrade|k8s-upgrade|basic|integration|remediation|k8s-conformance|capi-md-tests)
# if running basic, integration, k8s upgrade, clusterctl-upgrade, remediation, k8s conformance or capi-md tests, skip apply bmhs in dev-env
clusterctl-upgrade|k8s-upgrade|k8s-upgrade-n3|basic|integration|remediation|k8s-conformance|capi-md-tests)
# if running basic, integration, k8s upgrade, k8s n+3 upgrade, clusterctl-upgrade, remediation, k8s conformance or capi-md tests, skip apply bmhs in dev-env
echo 'export SKIP_APPLY_BMH="true"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
;;

Expand Down
11 changes: 11 additions & 0 deletions scripts/environment.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,17 @@ case "${GINKGO_FOCUS:-}" in
export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"1"}
;;

# k8s n+3 upgrade vars and config
k8s-upgrade-n3)
export NUM_NODES="4"
export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-"3"}
export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"1"}
export KUBERNETES_N0_VERSION=${KUBERNETES_N0_VERSION:-"v1.31.13"}
export KUBERNETES_N1_VERSION=${KUBERNETES_N1_VERSION:-"v1.32.9"}
export KUBERNETES_N2_VERSION=${KUBERNETES_N2_VERSION:-"v1.33.5"}
export KUBERNETES_N3_VERSION=${KUBERNETES_N3_VERSION:-"v1.34.1"}
;;

# Scalability test environment vars and config
scalability)
export NUM_NODES=${NUM_NODES:-"10"}
Expand Down
102 changes: 102 additions & 0 deletions test/e2e/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1227,3 +1227,105 @@ func IsMetal3DataCountEqualToMachineCount(ctx context.Context, c client.Client,

return len(m3DataList.Items) == len(machineList.Items)
}

type UpgradeControlPlaneInput struct {
E2EConfig *clusterctl.E2EConfig
BootstrapClusterProxy framework.ClusterProxy
TargetCluster framework.ClusterProxy
SpecName string
ClusterName string
Namespace string
K8sToVersion string
K8sFromVersion string
}

func UpgradeControlPlane(ctx context.Context, inputGetter func() UpgradeControlPlaneInput) {
input := inputGetter()
e2eConfig := input.E2EConfig
clusterClient := input.BootstrapClusterProxy.GetClient()
targetClusterClient := input.TargetCluster.GetClient()
clientSet := input.TargetCluster.GetClientSet()
k8sToVersion := input.K8sToVersion
k8sFromVersion := input.K8sFromVersion
specName := input.SpecName
namespace := input.Namespace
clusterName := input.ClusterName
numberOfControlplane := int(*e2eConfig.MustGetInt32PtrVariable("CONTROL_PLANE_MACHINE_COUNT"))
var (
controlplaneTaints = []corev1.Taint{{Key: "node-role.kubernetes.io/control-plane", Effect: corev1.TaintEffectNoSchedule},
{Key: "node-role.kubernetes.io/master", Effect: corev1.TaintEffectNoSchedule}}
)
// Upgrade process starts here
// Download node image
By("Download image")
imageURL, imageChecksum := EnsureImage(k8sToVersion)

By("Create new KCP Metal3MachineTemplate with upgraded image to boot")
m3MachineTemplateName := clusterName + "-controlplane"
newM3MachineTemplateName := clusterName + "-new-controlplane"
CreateNewM3MachineTemplate(ctx, namespace, newM3MachineTemplateName, m3MachineTemplateName, clusterClient, imageURL, imageChecksum)

Byf("Update KCP to upgrade k8s version and binaries from %s to %s", k8sFromVersion, k8sToVersion)
kcpObj := framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{
Lister: clusterClient,
ClusterName: clusterName,
Namespace: namespace,
})
helper, err := v1beta1patch.NewHelper(kcpObj, clusterClient)
Expect(err).NotTo(HaveOccurred())
kcpObj.Spec.MachineTemplate.Spec.InfrastructureRef.Name = newM3MachineTemplateName
kcpObj.Spec.Version = k8sToVersion
kcpObj.Spec.Rollout.Strategy.RollingUpdate.MaxSurge.IntVal = 0
Expect(helper.Patch(ctx, kcpObj)).To(Succeed())

Byf("Wait until %d BMH(s) are in deprovisioning state", 1)
WaitForNumBmhInState(ctx, bmov1alpha1.StateDeprovisioning, WaitForNumInput{
Client: clusterClient,
Options: []client.ListOption{client.InNamespace(namespace)},
Replicas: 1,
Intervals: e2eConfig.GetIntervals(specName, "wait-bmh-deprovisioning"),
})

Byf("Wait until three Control Plane machines become running and updated with the new %s k8s version", k8sToVersion)
runningAndUpgraded := func(machine clusterv1.Machine) bool {
running := machine.Status.GetTypedPhase() == clusterv1.MachinePhaseRunning
upgraded := machine.Spec.Version == k8sToVersion
return (running && upgraded)
}
WaitForNumMachines(ctx, runningAndUpgraded, WaitForNumInput{
Client: clusterClient,
Options: []client.ListOption{client.InNamespace(namespace)},
Replicas: numberOfControlplane,
Intervals: e2eConfig.GetIntervals(specName, "wait-machine-running"),
})

By("Untaint Control Plane nodes")
controlplaneNodes := getControlplaneNodes(ctx, clientSet)
untaintNodes(ctx, targetClusterClient, controlplaneNodes, controlplaneTaints)

By("Update maxSurge field in KubeadmControlPlane back to default value(1)")
kcpObj = framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{
Lister: clusterClient,
ClusterName: clusterName,
Namespace: namespace,
})
helper, err = v1beta1patch.NewHelper(kcpObj, clusterClient)
Expect(err).NotTo(HaveOccurred())
kcpObj.Spec.Rollout.Strategy.RollingUpdate.MaxSurge.IntVal = 1
for range 3 {
err = helper.Patch(ctx, kcpObj)
if err == nil {
break
}
time.Sleep(30 * time.Second)
}

// Verify that all three control plane nodes are using the k8s version
Byf("Verify all three control plane machines become running and updated with new %s k8s version", k8sToVersion)
WaitForNumMachines(ctx, runningAndUpgraded, WaitForNumInput{
Client: clusterClient,
Options: []client.ListOption{client.InNamespace(namespace)},
Replicas: numberOfControlplane,
Intervals: e2eConfig.GetIntervals(specName, "wait-machine-running"),
})
}
153 changes: 153 additions & 0 deletions test/e2e/upgrade_kubernetes_n3_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package e2e

import (
"context"
"os"
"path/filepath"
"strings"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var _ = Describe("Kubernetes version upgrade in target nodes", Label("k8s-upgrade-n3"), func() {

var (
ctx = context.TODO()
clusterctlLogFolder string
)

BeforeEach(func() {
osType = strings.ToLower(os.Getenv("OS"))
Expect(osType).ToNot(Equal(""))
validateGlobals(specName)

// We need to override clusterctl apply log folder to avoid getting our credentials exposed.
clusterctlLogFolder = filepath.Join(os.TempDir(), "target_cluster_logs", bootstrapClusterProxy.GetName())
})

It("Should create a cluster and run kubernetes N+3 tests", func() {
By("Apply BMH for workload cluster")
ApplyBmh(ctx, e2eConfig, bootstrapClusterProxy, namespace, specName)
By("Creating target cluster")
targetCluster, _ = CreateTargetCluster(ctx, func() CreateTargetClusterInput {
return CreateTargetClusterInput{
E2EConfig: e2eConfig,
BootstrapClusterProxy: bootstrapClusterProxy,
SpecName: specName,
ClusterName: clusterName,
K8sVersion: e2eConfig.MustGetVariable("KUBERNETES_N0_VERSION"),
KCPMachineCount: int64(numberOfControlplane),
WorkerMachineCount: int64(numberOfWorkers),
ClusterctlLogFolder: clusterctlLogFolder,
ClusterctlConfigPath: clusterctlConfigPath,
OSType: osType,
Namespace: namespace,
}
})

By("Running Kubernetes Upgrade tests")
upgradeKubernetesN3(ctx, func() upgradeKubernetesN3Input {
return upgradeKubernetesN3Input{
E2EConfig: e2eConfig,
BootstrapClusterProxy: bootstrapClusterProxy,
TargetCluster: targetCluster,
SpecName: specName,
ClusterName: clusterName,
Namespace: namespace,
}
})
})

AfterEach(func() {
ListBareMetalHosts(ctx, bootstrapClusterProxy.GetClient(), client.InNamespace(namespace))
ListMetal3Machines(ctx, bootstrapClusterProxy.GetClient(), client.InNamespace(namespace))
ListMachines(ctx, bootstrapClusterProxy.GetClient(), client.InNamespace(namespace))
ListNodes(ctx, targetCluster.GetClient())
DumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, targetCluster, artifactFolder, namespace, e2eConfig.GetIntervals, clusterName, clusterctlLogFolder, skipCleanup, clusterctlConfigPath)
})

})

type upgradeKubernetesN3Input struct {
E2EConfig *clusterctl.E2EConfig
BootstrapClusterProxy framework.ClusterProxy
TargetCluster framework.ClusterProxy
SpecName string
ClusterName string
Namespace string
}

// upgradeKubernetesN3 implements a test upgrading the cluster nodes from an old k8s version to a newer version.
func upgradeKubernetesN3(ctx context.Context, inputGetter func() upgradeKubernetesN3Input) {
Logf("Starting Kubernetes upgrade tests")
input := inputGetter()
clusterClient := input.BootstrapClusterProxy.GetClient()
targetClusterClient := input.TargetCluster.GetClient()
kubernetesVersion := input.E2EConfig.MustGetVariable("KUBERNETES_N0_VERSION")
upgradedK8sVersion1 := input.E2EConfig.MustGetVariable("KUBERNETES_N1_VERSION")
upgradedK8sVersion2 := input.E2EConfig.MustGetVariable("KUBERNETES_N2_VERSION")
upgradedK8sVersion3 := input.E2EConfig.MustGetVariable("KUBERNETES_N3_VERSION")

Logf("KUBERNETES VERSION: %v", kubernetesVersion)
Logf("UPGRADED K8S VERSION: %v", upgradedK8sVersion1)

ListBareMetalHosts(ctx, clusterClient, client.InNamespace(input.Namespace))
ListMetal3Machines(ctx, clusterClient, client.InNamespace(namespace))
ListMachines(ctx, clusterClient, client.InNamespace(namespace))
ListNodes(ctx, targetClusterClient)

By("Running Kubernetes n+1 Upgrade tests")
UpgradeControlPlane(ctx, func() UpgradeControlPlaneInput {
return UpgradeControlPlaneInput{
E2EConfig: e2eConfig,
BootstrapClusterProxy: bootstrapClusterProxy,
TargetCluster: targetCluster,
SpecName: specName,
ClusterName: clusterName,
Namespace: namespace,
K8sFromVersion: kubernetesVersion,
K8sToVersion: upgradedK8sVersion1,
}
})
By("KUBERNETES UPGRADE N+1 TESTS PASSED!")
Logf("KUBERNETES VERSION: %v", upgradedK8sVersion1)
Logf("UPGRADED K8S VERSION: %v", upgradedK8sVersion2)
Logf("NUMBER OF CONTROLPLANE BMH: %v", numberOfControlplane)

By("Running Kubernetes N+2 Upgrade tests")
UpgradeControlPlane(ctx, func() UpgradeControlPlaneInput {
return UpgradeControlPlaneInput{
E2EConfig: e2eConfig,
BootstrapClusterProxy: bootstrapClusterProxy,
TargetCluster: targetCluster,
SpecName: specName,
ClusterName: clusterName,
Namespace: namespace,
K8sFromVersion: upgradedK8sVersion1,
K8sToVersion: upgradedK8sVersion2,
}
})
By("KUBERNETES UPGRADE N+2 TESTS PASSED!")
Logf("KUBERNETES VERSION: %v", upgradedK8sVersion2)
Logf("UPGRADED K8S VERSION: %v", upgradedK8sVersion3)
Logf("NUMBER OF CONTROLPLANE BMH: %v", numberOfControlplane)

By("Running Kubernetes n+3 Upgrade tests")
UpgradeControlPlane(ctx, func() UpgradeControlPlaneInput {
return UpgradeControlPlaneInput{
E2EConfig: e2eConfig,
BootstrapClusterProxy: bootstrapClusterProxy,
TargetCluster: targetCluster,
SpecName: specName,
ClusterName: clusterName,
Namespace: namespace,
K8sFromVersion: upgradedK8sVersion2,
K8sToVersion: upgradedK8sVersion3,
}
})
By("KUBERNETES UPGRADE N+3 TESTS PASSED!")
}