Skip to content

Commit 1ebdff7

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1269 from AartiJivrajani/v1beta1-cluster-update-test
add v1alpha3/v1alpha4->v1beta1 clusterctl upgrade test
2 parents d0febf0 + 91a2b5f commit 1ebdff7

File tree

15 files changed

+1990
-36
lines changed

15 files changed

+1990
-36
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
# Generated files
99
/cluster-api-provider-vsphere
1010
/examples/provider-components/provider-components*.yaml
11+
test/e2e/data/infrastructure-vsphere/kustomization/base/cluster-template.yaml
1112
test/e2e/data/infrastructure-vsphere/cluster-template.yaml
13+
test/e2e/data/infrastructure-vsphere/cluster-template-remote-management.yaml
1214

1315
# env vars file used in getting-started.md and manifests generation
1416
envvars.txt

‎Makefile‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,20 +119,34 @@ test: $(GOVC)
119119
e2e-image: ## Build the e2e manager image
120120
docker build --build-arg ldflags="$(LDFLAGS)" --tag="gcr.io/k8s-staging-cluster-api/capv-manager:e2e" .
121121

122+
.PHONY: e2e-templates
123+
e2e-templates: ## Generate e2e cluster templates
124+
$(MAKE) release-manifests
125+
cp $(RELEASE_DIR)/cluster-template.yaml $(E2E_TEMPLATE_DIR)/kustomization/base/cluster-template.yaml
126+
"$(KUSTOMIZE)" build $(E2E_TEMPLATE_DIR)/kustomization/base > $(E2E_TEMPLATE_DIR)/cluster-template.yaml
127+
"$(KUSTOMIZE)" build $(E2E_TEMPLATE_DIR)/kustomization/remote-management > $(E2E_TEMPLATE_DIR)/cluster-template-remote-management.yaml
128+
122129
.PHONY: e2e
123-
e2e: e2e-image
130+
e2e: e2e-image e2e-templates
124131
e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
125-
$(MAKE) release-manifests
126-
cp $(RELEASE_DIR)/cluster-template.yaml $(E2E_TEMPLATE_DIR)/kustomization/cluster-template.yaml
127-
"$(KUSTOMIZE)" build $(E2E_TEMPLATE_DIR)/kustomization > $(E2E_TEMPLATE_DIR)/cluster-template.yaml
128132
@echo PATH=$(PATH)
129133
@echo
130134
@echo Contents of $(TOOLS_BIN_DIR):
131135
@ls $(TOOLS_BIN_DIR)
132136
@echo
133137

134-
time $(GINKGO) -v ./test/e2e -- --e2e.config="$(E2E_CONF_FILE)"
138+
time $(GINKGO) -v -skip="ClusterAPI Upgrade Tests" ./test/e2e -- --e2e.config="$(E2E_CONF_FILE)"
139+
140+
.PHONY: e2e-upgrade
141+
e2e-upgrade: e2e-image e2e-templates
142+
e2e-upgrade: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
143+
@echo PATH=$(PATH)
144+
@echo
145+
@echo Contents of $(TOOLS_BIN_DIR):
146+
@ls $(TOOLS_BIN_DIR)
147+
@echo
135148

149+
time $(GINKGO) -v -focus="ClusterAPI Upgrade Tests" ./test/e2e -- --e2e.config="$(E2E_CONF_FILE)"
136150
## --------------------------------------
137151
## Binaries
138152
## --------------------------------------

‎hack/e2e.sh‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,23 +26,40 @@ source "${REPO_ROOT}/hack/ensure-kubectl.sh"
2626

2727
on_exit() {
2828
# release IPClaim
29-
echo "Releasing IP claim"
29+
echo "Releasing IP claims"
3030
kubectl --kubeconfig="${KUBECONFIG}" delete ipclaim "${IPCLAIM_NAME}" || true
31+
kubectl --kubeconfig="${KUBECONFIG}" delete ipclaim "${WORKLOAD_IPCLAIM_NAME}" || true
3132

3233
# kill the VPN
3334
docker kill vpn
35+
36+
# logout of gcloud
37+
if [ "${AUTH}" ]; then
38+
gcloud auth revoke
39+
fi
3440
}
3541

3642
trap on_exit EXIT
3743

44+
function login() {
45+
# If GCR_KEY_FILE is set, use that service account to login
46+
if [ "${GCR_KEY_FILE}" ]; then
47+
gcloud auth activate-service-account --key-file "${GCR_KEY_FILE}" || fatal "unable to login"
48+
AUTH=1
49+
fi
50+
}
51+
52+
AUTH=
53+
E2E_IMAGE_SHA=
54+
GCR_KEY_FILE="${GCR_KEY_FILE:-}"
3855
export VSPHERE_SERVER="${GOVC_URL}"
3956
export VSPHERE_USERNAME="${GOVC_USERNAME}"
4057
export VSPHERE_PASSWORD="${GOVC_PASSWORD}"
4158
export VSPHERE_SSH_AUTHORIZED_KEY="${VM_SSH_PUB_KEY}"
4259
export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key"
4360
export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere-ci.yaml"
4461
export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
45-
62+
export DOCKER_IMAGE_TAR="${ARTIFACTS}/tempContainers/image.tar"
4663
export GC_KIND="false"
4764

4865
# Run the vpn client in container
@@ -64,8 +81,30 @@ sed "s/IPCLAIM_NAME/${IPCLAIM_NAME}/" "${REPO_ROOT}/hack/ipclaim-template.yaml"
6481
IPADDRESS_NAME=$(kubectl --kubeconfig=${KUBECONFIG} get ipclaim "${IPCLAIM_NAME}" -o=jsonpath='{@.status.address.name}')
6582
CONTROL_PLANE_ENDPOINT_IP=$(kubectl --kubeconfig=${KUBECONFIG} get ipaddresses "${IPADDRESS_NAME}" -o=jsonpath='{@.spec.address}')
6683
export CONTROL_PLANE_ENDPOINT_IP
67-
6884
echo "Acquired Control Plane IP: $CONTROL_PLANE_ENDPOINT_IP"
6985

86+
# Retrieve an IP to be used for the workload cluster in v1a3/v1a4 -> v1b1 upgrade tests
87+
WORKLOAD_IPCLAIM_NAME="workload-ip-claim-$(date +%s)"
88+
sed "s/IPCLAIM_NAME/${WORKLOAD_IPCLAIM_NAME}/" "${REPO_ROOT}/hack/ipclaim-template.yaml" | kubectl --kubeconfig=${KUBECONFIG} create -f -
89+
WORKLOAD_IPADDRESS_NAME=$(kubectl --kubeconfig=${KUBECONFIG} get ipclaim "${WORKLOAD_IPCLAIM_NAME}" -o=jsonpath='{@.status.address.name}')
90+
WORKLOAD_CONTROL_PLANE_ENDPOINT_IP=$(kubectl --kubeconfig=${KUBECONFIG} get ipaddresses "${WORKLOAD_IPADDRESS_NAME}" -o=jsonpath='{@.spec.address}')
91+
export WORKLOAD_CONTROL_PLANE_ENDPOINT_IP
92+
echo "Acquired Workload Cluster Control Plane IP: $WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"
93+
94+
# save the docker image locally
95+
make e2e-image
96+
mkdir -p "$ARTIFACTS"/tempContainers
97+
docker save gcr.io/k8s-staging-cluster-api/capv-manager:e2e -o "$DOCKER_IMAGE_TAR"
98+
99+
# store the image on gcs
100+
login
101+
E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' gcr.io/k8s-staging-cluster-api/capv-manager:e2e)
102+
export E2E_IMAGE_SHA
103+
gsutil cp "$ARTIFACTS"/tempContainers/image.tar gs://capv-ci/"$E2E_IMAGE_SHA"
104+
105+
# Run e2e-upgrade tests
106+
# TODO: move e2e-upgrade into its own prow job
107+
make e2e-upgrade
108+
70109
# Run e2e tests
71110
make e2e

‎test/e2e/capi_upgrade_test.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
22+
. "github.com/onsi/ginkgo"
23+
24+
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
25+
)
26+
27+
var _ = Context("ClusterAPI Upgrade Tests", func() {
28+
Describe("Upgrading cluster from v1alpha3 to v1beta1 using clusterctl", func() {
29+
capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
30+
return capi_e2e.ClusterctlUpgradeSpecInput{
31+
E2EConfig: e2eConfig,
32+
ClusterctlConfigPath: clusterctlConfigPath,
33+
BootstrapClusterProxy: bootstrapClusterProxy,
34+
ArtifactFolder: artifactFolder,
35+
SkipCleanup: skipCleanup,
36+
InitWithBinary: e2eConfig.GetVariable("INIT_WITH_BINARY_V1ALPHA3"),
37+
InitWithProvidersContract: "v1alpha3",
38+
MgmtFlavor: "remote-management",
39+
}
40+
})
41+
})
42+
43+
Describe("Upgrading cluster from v1alpha4 to v1beta1 using clusterctl", func() {
44+
capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
45+
return capi_e2e.ClusterctlUpgradeSpecInput{
46+
E2EConfig: e2eConfig,
47+
ClusterctlConfigPath: clusterctlConfigPath,
48+
BootstrapClusterProxy: bootstrapClusterProxy,
49+
ArtifactFolder: artifactFolder,
50+
SkipCleanup: skipCleanup,
51+
InitWithBinary: e2eConfig.GetVariable("INIT_WITH_BINARY_V1ALPHA4"),
52+
InitWithProvidersContract: "v1alpha4",
53+
MgmtFlavor: "remote-management",
54+
}
55+
})
56+
})
57+
})

‎test/e2e/config/vsphere-ci.yaml‎

Lines changed: 79 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,24 @@ providers:
2828
- name: cluster-api
2929
type: CoreProvider
3030
versions:
31+
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
32+
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/core-components.yaml
33+
type: url
34+
contract: v1alpha3
35+
files:
36+
- sourcePath: "../data/shared/metadata.yaml"
37+
replacements:
38+
- old: "imagePullPolicy: Always"
39+
new: "imagePullPolicy: IfNotPresent"
40+
- name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
41+
contract: v1alpha4
42+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/core-components.yaml"
43+
type: "url"
44+
files:
45+
- sourcePath: "../data/shared/metadata.yaml"
46+
replacements:
47+
- old: "imagePullPolicy: Always"
48+
new: "imagePullPolicy: IfNotPresent"
3149
- name: v1.0.0
3250
# Use manifest from source files
3351
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/core-components.yaml"
@@ -37,12 +55,28 @@ providers:
3755
replacements:
3856
- old: "imagePullPolicy: Always"
3957
new: "imagePullPolicy: IfNotPresent"
40-
- old: "--enable-leader-election"
41-
new: "--enable-leader-election=false"
4258

4359
- name: kubeadm
4460
type: BootstrapProvider
4561
versions:
62+
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
63+
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/bootstrap-components.yaml
64+
type: url
65+
contract: v1alpha3
66+
files:
67+
- sourcePath: "../data/shared/metadata.yaml"
68+
replacements:
69+
- old: "imagePullPolicy: Always"
70+
new: "imagePullPolicy: IfNotPresent"
71+
- name: v0.4.4 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
72+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/bootstrap-components.yaml"
73+
type: "url"
74+
contract: v1alpha4
75+
files:
76+
- sourcePath: "../data/shared/metadata.yaml"
77+
replacements:
78+
- old: "imagePullPolicy: Always"
79+
new: "imagePullPolicy: IfNotPresent"
4680
- name: v1.0.0
4781
# Use manifest from source files
4882
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/bootstrap-components.yaml"
@@ -52,12 +86,28 @@ providers:
5286
replacements:
5387
- old: "imagePullPolicy: Always"
5488
new: "imagePullPolicy: IfNotPresent"
55-
- old: "--enable-leader-election"
56-
new: "--enable-leader-election=false"
5789

5890
- name: kubeadm
5991
type: ControlPlaneProvider
6092
versions:
93+
- name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
94+
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/control-plane-components.yaml
95+
type: url
96+
contract: v1alpha3
97+
files:
98+
- sourcePath: "../data/shared/metadata.yaml"
99+
replacements:
100+
- old: "imagePullPolicy: Always"
101+
new: "imagePullPolicy: IfNotPresent"
102+
- name: v0.4.4
103+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/control-plane-components.yaml"
104+
type: "url"
105+
contract: v1alpha4
106+
files:
107+
- sourcePath: "../data/shared/metadata.yaml"
108+
replacements:
109+
- old: "imagePullPolicy: Always"
110+
new: "imagePullPolicy: IfNotPresent"
61111
- name: v1.0.0
62112
# Use manifest from source files
63113
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/control-plane-components.yaml"
@@ -67,25 +117,38 @@ providers:
67117
replacements:
68118
- old: "imagePullPolicy: Always"
69119
new: "imagePullPolicy: IfNotPresent"
70-
- old: "--enable-leader-election"
71-
new: "--enable-leader-election=false"
72120

73121
- name: vsphere
74122
type: InfrastructureProvider
75123
versions:
124+
- name: v0.7.10
125+
value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v0.7.10/infrastructure-components.yaml
126+
type: "url"
127+
contract: v1alpha3
128+
files:
129+
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/capi-upgrades/v1alpha3/cluster-template.yaml"
130+
- sourcePath: "../../../metadata.yaml"
131+
- name: v0.8.1
132+
value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v0.8.1/infrastructure-components.yaml
133+
type: "url"
134+
contract: v1alpha4
135+
files:
136+
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/capi-upgrades/v1alpha4/cluster-template.yaml"
137+
- sourcePath: "../../../metadata.yaml"
76138
- name: v1.0.0
77139
# Use manifest from source files
78140
value: ../../../../cluster-api-provider-vsphere/config/default
141+
contract: v1beta1
79142
replacements:
80143
- old: gcr.io/cluster-api-provider-vsphere/release/manager:latest
81144
new: gcr.io/k8s-staging-cluster-api/capv-manager:e2e
82145
- old: "imagePullPolicy: Always"
83146
new: "imagePullPolicy: IfNotPresent"
84-
85-
files:
86-
# Add a cluster template
87-
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template.yaml"
88-
- sourcePath: "../../../metadata.yaml"
147+
files:
148+
# Add a cluster template
149+
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template.yaml"
150+
- sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-remote-management.yaml"
151+
- sourcePath: "../../../metadata.yaml"
89152

90153
variables:
91154
KUBERNETES_VERSION: "v1.19.1"
@@ -102,11 +165,14 @@ variables:
102165
VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy"
103166
VSPHERE_NETWORK: "sddc-cgw-network-6"
104167
VSPHERE_TEMPLATE: "ubuntu-1804-kube-v1.19.1"
168+
INIT_WITH_BINARY_V1ALPHA3: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
169+
INIT_WITH_BINARY_V1ALPHA4: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}"
170+
INIT_WITH_KUBERNETES_VERSION: "v1.19.1"
105171

106172
intervals:
107173
default/wait-controllers: ["5m", "10s"]
108174
default/wait-cluster: ["5m", "10s"]
109-
default/wait-control-plane: ["5m", "10s"]
110-
default/wait-worker-nodes: ["5m", "10s"]
175+
default/wait-control-plane: ["10m", "10s"]
176+
default/wait-worker-nodes: ["10m", "10s"]
111177
default/wait-delete-cluster: ["5m", "10s"]
112178
default/wait-machine-upgrade: ["15m", "1m"]

0 commit comments

Comments
 (0)