diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 98a2305c46..4d97f3fc41 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -11,37 +11,39 @@ parameters: hubbleEnabled: false dualstackVersion: "" cni: "cilium" + upgradeScenario: "" # Condition confirms that: # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success. # CONTROL_CNI either contains 'cniv1' or 'all'. It is not case sensitive stages: - - stage: create_${{ parameters.name }} - condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cilium') , contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) ) - variables: - ${{ if contains(parameters.clusterName, 'rdma') }}: - location: $(LOCATION_RDMA) - ${{ elseif eq(parameters.arch, 'arm64') }}: - location: $(LOCATION_ARM64) - ${{ else }}: - location: $(LOCATION_AMD64) - commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ] - dependsOn: - - setup - - build_images - displayName: "Create Cluster - ${{ parameters.clusterName }}" - jobs: - - job: create_aks_cluster_with_${{ parameters.name }} - pool: - name: "$(BUILD_POOL_NAME_DEFAULT)" - steps: - - template: ../load-test-templates/create-cluster-template.yaml - parameters: - clusterType: ${{ parameters.clusterType }} - clusterName: ${{ parameters.clusterName }}-$(commitID) - nodeCount: ${{ parameters.nodeCount }} - vmSize: ${{ parameters.vmSize }} - region: $(location) + - ${{if eq(parameters.upgradeScenario, false)}}: + - stage: create_${{ parameters.name }} + condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cilium') , contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) ) + variables: + ${{ if contains(parameters.clusterName, 'rdma') }}: + location: $(LOCATION_RDMA) + ${{ elseif eq(parameters.arch, 'arm64') }}: + location: $(LOCATION_ARM64) + ${{ else }}: + location: $(LOCATION_AMD64) + commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ] + dependsOn: + - setup + - build_images + displayName: "Create Cluster - ${{ parameters.clusterName }}" + jobs: + - job: create_aks_cluster_with_${{ parameters.name }} + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + steps: + - template: ../load-test-templates/create-cluster-template.yaml + parameters: + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + nodeCount: ${{ parameters.nodeCount }} + vmSize: ${{ parameters.vmSize }} + region: $(location) # Conditions for below E2E test scenarios confirm that: # Pipeline has not been canceled and that the previous job has reports anything other than failure(Succeeded, SuccededWithIssues, Skipped). Previous job is declared by dependsOn: @@ -60,7 +62,10 @@ stages: pool: name: "$(BUILD_POOL_NAME_DEFAULT)" dependsOn: - - create_${{ parameters.name }} + - ${{ if eq(parameters.upgradeScenario, false) }}: + - create_${{ parameters.name }} + - ${{ else }}: + - ${{ parameters.dependsOn }} - publish - setup displayName: "Cilium Test - ${{ parameters.name }}" @@ -117,27 +122,7 @@ stages: envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f - kubectl get po -owide -A - - if [ -z "$TEST_AZURE_IPAM_VERSION" ] - then - echo "TEST_AZURE_IPAM_VERSION is not set, using default value" - IPAM=$(make azure-ipam-version) - else - IPAM=$(TEST_AZURE_IPAM_VERSION) - echo "TEST_AZURE_IPAM_VERSION is set to $IPAM" - fi - if [ -z "$TEST_CNS_VERSION" ] - then - echo "TEST_CNS_VERSION is not set, using default value" - CNS=$(make cns-version) - else - CNS=$(TEST_CNS_VERSION) - echo "TEST_CNS_VERSION is set to $CNS" - fi - echo "Deploy Azure-CNS" - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) - kubectl get po -owide -A - kubectl get crd -A + - ${{if eq(parameters.hubbleEnabled, true)}}: - job: deploy_cilium_components displayName: Deploy Cilium with Hubble @@ -169,26 +154,71 @@ stages: envsubst '${CILIUM_IMAGE_REGISTRY},${CILIUM_VERSION_TAG}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f - kubectl get po -owide -A - if [ -z "$TEST_AZURE_IPAM_VERSION" ] - then - echo "TEST_AZURE_IPAM_VERSION is not set, using default value" - IPAM=$(make azure-ipam-version) - else - IPAM=$(TEST_AZURE_IPAM_VERSION) - echo "TEST_AZURE_IPAM_VERSION is set to $IPAM" - fi - if [ -z "$TEST_CNS_VERSION" ] - then - echo "TEST_CNS_VERSION is not set, using default value" - CNS=$(make cns-version) - else - CNS=$(TEST_CNS_VERSION) - echo "TEST_CNS_VERSION is set to $CNS" - fi - echo "Deploy Azure-CNS" - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) - kubectl get po -owide -A - kubectl get crd -A + - job: deploy_cns_and_ipam + displayName: "Deploy CNS and IPAM" + dependsOn: deploy_cilium_components + steps: + - task: AzureCLI@2 + displayName: "Install CNS and IPAM" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + az extension add --name aks-preview + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A + if [ ${{parameters.upgradeScenario}} = "true" ] + then + echo "Upgrade scenario is true, using upgrade azure ipam and cns version from pipeline variables" + if [ -z "$UPGRADE_AZURE_IPAM_VERSION" ] + then + echo "UPGRADE_AZURE_IPAM_VERSION is not set, using default value" + IPAM=$(make azure-ipam-version) + else + IPAM=$(UPGRADE_AZURE_IPAM_VERSION) + echo "UPGRADE_AZURE_IPAM_VERSION is set to $IPAM" + fi + + if [ -z "$UPGRADE_CNS_VERSION" ] + then + echo "UPGRADE_CNS_VERSION is not set, using default value" + CNS=$(make cns-version) + else + CNS=$(UPGRADE_CNS_VERSION) + echo "UPGRADE_CNS_VERSION is set to $CNS" + fi + IPAM_IMAGE_REPO=$(UPGRADE_IPAM_IMAGE_REPO) + CNS_IMAGE_REPO=$(UPGRADE_CNS_IMAGE_REPO) + echo Deploying with Azure Ipam version $IPAM from $IPAM_IMAGE_REPO and CNS version $CNS from $CNS_IMAGE_REPO + else + if [ -z "$TEST_AZURE_IPAM_VERSION" ] + then + echo "TEST_AZURE_IPAM_VERSION is not set, using default value" + IPAM=$(make azure-ipam-version) + else + IPAM=$(TEST_AZURE_IPAM_VERSION) + echo "TEST_AZURE_IPAM_VERSION is set to $IPAM" + fi + if [ -z "$TEST_CNS_VERSION" ] + then + echo "TEST_CNS_VERSION is not set, using default value" + CNS=$(make cns-version) + else + CNS=$(TEST_CNS_VERSION) + echo "TEST_CNS_VERSION is set to $CNS" + fi + fi + + echo "Deploy Azure-CNS" + sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) + kubectl get po -owide -A + kubectl get crd -A - job: deploy_pods condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) ) @@ -203,6 +233,8 @@ stages: iterations: ${ITERATIONS_CILIUM} nodeCount: ${{ parameters.nodeCount }} cni: cilium + ${{ if eq(parameters.upgradeScenario, 'true') }}: + logType: upgradeScaleTest - template: ../load-test-templates/validate-state-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -218,6 +250,8 @@ stages: os: ${{ parameters.os }} cni: cilium region: $(location) + ${{ if eq(parameters.upgradeScenario, 'true') }}: + logType: upgradeRestartNode - template: ../load-test-templates/validate-state-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -235,6 +269,8 @@ stages: scaleup: ${SCALEUP_CILIUM} nodeCount: ${{ parameters.nodeCount }} cni: ${{ parameters.cni }} + ${{ if eq(parameters.upgradeScenario, 'true') }}: + logType: upgradeRestartCNS - job: cni_tests displayName: "Cilium Test" dependsOn: restart_cns diff --git a/.pipelines/cni/load-test-templates/pod-deployment-template.yaml b/.pipelines/cni/load-test-templates/pod-deployment-template.yaml index 002b25ab36..d87c510914 100644 --- a/.pipelines/cni/load-test-templates/pod-deployment-template.yaml +++ b/.pipelines/cni/load-test-templates/pod-deployment-template.yaml @@ -6,6 +6,7 @@ parameters: nodeCount: 10 cni: "" jobName: "deploy_pods" + logType: "scaleTest" steps: - task: AzureCLI@2 @@ -26,7 +27,7 @@ steps: - template: ../../templates/log-template.yaml parameters: clusterName: ${{ parameters.clusterName }} - logType: scaleTest os: ${{ parameters.os }} cni: ${{ parameters.cni }} jobName: ${{ parameters.jobName }} + logType: ${{ parameters.logType }} diff --git a/.pipelines/cni/load-test-templates/restart-cns-template.yaml b/.pipelines/cni/load-test-templates/restart-cns-template.yaml index e0527c408f..f0d02cb1c7 100644 --- a/.pipelines/cni/load-test-templates/restart-cns-template.yaml +++ b/.pipelines/cni/load-test-templates/restart-cns-template.yaml @@ -5,6 +5,7 @@ parameters: nodeCount: 10 os: "" jobName: "restart_cns" + logType: "restartCNS" steps: - task: AzureCLI@2 @@ -38,7 +39,7 @@ steps: - template: ../../templates/log-template.yaml parameters: clusterName: ${{ parameters.clusterName }} - logType: restartCNS + logType: ${{ parameters.logType }} os: ${{ parameters.os }} cni: ${{ parameters.cni }} jobName: ${{ parameters.jobName }} diff --git a/.pipelines/cni/load-test-templates/restart-node-template.yaml b/.pipelines/cni/load-test-templates/restart-node-template.yaml index 2cdd4230f3..f07940a992 100644 --- a/.pipelines/cni/load-test-templates/restart-node-template.yaml +++ b/.pipelines/cni/load-test-templates/restart-node-template.yaml @@ -4,6 +4,7 @@ parameters: cni: "" jobName: "restart_nodes" region: "" + logType: "restartNode" steps: - task: AzureCLI@2 @@ -56,7 +57,7 @@ steps: - template: ../../templates/log-template.yaml parameters: clusterName: ${{ parameters.clusterName }} - logType: restartNode + logType: ${{ parameters.logType }} os: ${{ parameters.os }} cni: ${{ parameters.cni }} jobName: ${{ parameters.jobName }} diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml index 941cc4cf1b..66ff3d1087 100644 --- a/.pipelines/cni/pipeline.yaml +++ b/.pipelines/cni/pipeline.yaml @@ -5,6 +5,10 @@ trigger: - dropgz/* - azure-ipam/* - v* +parameters: + - name: upgradeScenario + type: boolean + default: false stages: - stage: setup @@ -24,7 +28,7 @@ stages: name: "SetEnvVars" displayName: "Set Environment Variables" condition: always() - + - stage: build_images dependsOn: setup displayName: "Build Images" @@ -75,29 +79,29 @@ stages: name: "$(BUILD_POOL_NAME_LINUX_ARM64)" strategy: matrix: - azure_ipam_linux_arm64: - arch: arm64 - name: azure-ipam - os: linux - cni_linux_arm64: - arch: arm64 - name: cni - os: linux - ipv6_hp_bpf_linux_arm64: - arch: arm64 - name: ipv6-hp-bpf - os: linux - npm_linux_arm64: - arch: arm64 - name: npm - os: linux + azure_ipam_linux_arm64: + arch: arm64 + name: azure-ipam + os: linux + cni_linux_arm64: + arch: arm64 + name: cni + os: linux + ipv6_hp_bpf_linux_arm64: + arch: arm64 + name: ipv6-hp-bpf + os: linux + npm_linux_arm64: + arch: arm64 + name: npm + os: linux steps: - template: ../containers/container-template.yaml parameters: arch: $(arch) name: $(name) os: $(os) - + - stage: binaries displayName: Build Binaries dependsOn: setup @@ -304,6 +308,7 @@ stages: vmSize: ${VM_SIZE_CILIUM} arch: amd64 cni: "cilium" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -315,6 +320,7 @@ stages: vmSize: ${VM_SIZE_CILIUM} arch: amd64 cni: "cilium" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -326,6 +332,7 @@ stages: arch: amd64 osSKU: AzureLinux cni: "cilium" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -336,6 +343,7 @@ stages: vmSize: Standard_D8ps_v5 arch: arm64 cni: "cilium" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -346,6 +354,7 @@ stages: vmSize: Standard_HC44-16rs arch: amd64 cni: "cilium" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -357,6 +366,7 @@ stages: arch: amd64 dualstackVersion: ${CILIUM_DUALSTACK_VERSION} cni: "cilium_dualstack" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -368,6 +378,7 @@ stages: arch: arm64 dualstackVersion: ${CILIUM_DUALSTACK_VERSION} cni: "cilium_dualstack" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -380,6 +391,7 @@ stages: osSKU: AzureLinux dualstackVersion: ${CILIUM_DUALSTACK_VERSION} cni: "cilium_dualstack" + upgradeScenario: false - template: cilium/cilium-overlay-load-test-template.yaml parameters: @@ -391,6 +403,125 @@ stages: arch: amd64 dualstackVersion: ${CILIUM_DUALSTACK_VERSION} cni: "cilium_dualstack" + upgradeScenario: false + + + ## If upgradeScenario is set, redeploy new IPAM version to existing clusters and run tests + - ${{if eq(parameters.upgradeScenario, true)}}: + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_upgrade + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cilium-over" + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: ${VM_SIZE_CILIUM} + arch: amd64 + cni: "cilium" + upgradeScenario: true + dependsOn: cilium_overlay + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_hubble_upgrade + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cil-over-hub" + hubbleEnabled: true + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: ${VM_SIZE_CILIUM} + arch: amd64 + cni: "cilium" + upgradeScenario: true + dependsOn: cilium_overlay_hubble + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_mariner_upgrade + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cil-over-mar" + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: ${VM_SIZE_CILIUM} + arch: amd64 + osSKU: AzureLinux + cni: "cilium" + upgradeScenario: true + dependsOn: cilium_overlay_mariner + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_arm_upgrade + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cil-over-arm" + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: Standard_D8ps_v5 + arch: arm64 + cni: "cilium" + upgradeScenario: true + dependsOn: cilium_overlay_arm + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_rdma_upgrade + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cil-over-rdma" + nodeCount: 2 + vmSize: Standard_HC44-16rs + arch: amd64 + cni: "cilium" + upgradeScenario: true + dependsOn: cilium_overlay_rdma + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_ds_upgrade + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cil-ds-ov" + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: ${VM_SIZE_CILIUM} + arch: amd64 + dualstackVersion: ${CILIUM_DUALSTACK_VERSION} + cni: "cilium_dualstack" + upgradeScenario: true + dependsOn: cilium_overlay_ds + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_ds_arm_upgrade + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cil-ds-arm" + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: Standard_D8ps_v5 + arch: arm64 + dualstackVersion: ${CILIUM_DUALSTACK_VERSION} + cni: "cilium_dualstack" + upgradeScenario: true + dependsOn: cilium_ds_arm + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_ds_mariner_upgrade + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cil-ds-mar" + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: ${VM_SIZE_CILIUM} + arch: amd64 + osSKU: AzureLinux + dualstackVersion: ${CILIUM_DUALSTACK_VERSION} + cni: "cilium_dualstack" + upgradeScenario: true + dependsOn: cilium_ds_mariner + + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_ds_rdma_upgrade + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cil-ds-rdma" + nodeCount: 2 + vmSize: Standard_HC44-16rs + arch: amd64 + dualstackVersion: ${CILIUM_DUALSTACK_VERSION} + cni: "cilium_dualstack" + upgradeScenario: true + dependsOn: cilium_ds_rdma - stage: delete_resources displayName: "Delete Resources" @@ -418,6 +549,16 @@ stages: - windows_overlay_HNS - windows19_overlay_HNS - setup + - ${{if eq(parameters.upgradeScenario, true)}}: + - cilium_overlay_upgrade + - cilium_overlay_hubble_upgrade + - cilium_overlay_mariner_upgrade + - cilium_overlay_arm_upgrade + - cilium_overlay_rdma_upgrade + - cilium_overlay_ds_upgrade + - cilium_ds_arm_upgrade + - cilium_ds_mariner_upgrade + - cilium_ds_rdma_upgrade variables: commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ] jobs: diff --git a/.pipelines/templates/log-template.yaml b/.pipelines/templates/log-template.yaml index fee969889f..17a9d25ab7 100644 --- a/.pipelines/templates/log-template.yaml +++ b/.pipelines/templates/log-template.yaml @@ -270,6 +270,6 @@ steps: - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt) condition: always() - artifact: ${{ parameters.clusterName }}_${{ parameters.os }}${{ parameters.jobName }}_Attempt_#$(System.StageAttempt) + artifact: ${{ parameters.clusterName }}_${{ parameters.logType }}_${{ parameters.os }}${{ parameters.jobName }}_Attempt_#$(System.StageAttempt) name: acnLogs_${{ parameters.logType }} displayName: Publish Cluster logs