Merge pull request #1060 from Nordix/erja/pre-release-testing

Add pipeline for testing with k8s' pre-release versions

Author: metal3-io-bot

jenkins/image_building/build-image.sh

@@ -16,15 +16,33 @@ trap cleanup EXIT
 # Make sure we run everything in the repo root
 cd "${REPO_ROOT}" || true
 
+export AGENT_OS="${AGENT_OS:-"ubuntu"}"
+
 export IMAGE_OS="${IMAGE_OS}"
 export IMAGE_TYPE="${IMAGE_TYPE}"
 
-# Disable needrestart interactive mode
-sudo sed -i "s/^#\$nrconf{restart} = 'i';/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf > /dev/null || true
+# Jenkins agent OS-specific package installation and configuration
+if [[ "${AGENT_OS}" == "ubuntu" ]]; then
+  # Disable needrestart interactive mode
+  sudo sed -i "s/^#\$nrconf{restart} = 'i';/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf > /dev/null || true
+
+  sudo apt-get update
+  sudo apt-get install -y python3-dev python3-pip python3-venv qemu qemu-kvm
+
+  python3 -m venv venv
+elif [[ "${AGENT_OS}" == "centos" ]]; then
+  # Install EPEL repository for additional packages
+  sudo yum install -y epel-release
 
-sudo apt-get update
-sudo apt-get install -y python3-dev python3-pip python3-venv qemu qemu-kvm
-python3 -m venv venv
+  # Install required packages
+  sudo yum install -y python3-devel python3-pip qemu-kvm
+  sudo pip3 install virtualenv
+
+  python3 -m virtualenv venv
+else
+  echo "Unsupported AGENT_OS: ${AGENT_OS}"
+  exit 1
+fi
 
 # shellcheck source=/dev/null
 . venv/bin/activate

jenkins/image_building/dib_elements/centos-node/install.d/55-install

@@ -20,6 +20,41 @@ net.bridge.bridge-nf-call-ip6tables = 1
 EOF
 
 sudo yum install -y container-selinux
-sudo yum install -y kubelet-"${KUBERNETES_BINARIES_VERSION//v}" kubeadm-"${KUBERNETES_BINARIES_VERSION//v}" kubectl-"${KUBERNETES_BINARIES_VERSION//v}" cri-o-"${CRIO_BINARIES_VERSION//v}" cri-tools-"${CRICTL_BINARIES_VERSION//v}"
+
+if [[ "${PRE_RELEASE:-}" == "true" ]]; then
+  # See https://github.com/containernetworking/plugins/releases
+  CNI_PLUGINS_VERSION="v1.7.1"
+
+  # Kubernetes release tooling
+  # See https://github.com/kubernetes/release/releases
+  RELEASE_VERSION="${RELEASE_VERSION:-v0.18.0}"
+
+  case $(uname -m) in
+    x86_64) ARCH="amd64" ;;
+    aarch64) ARCH="arm64" ;;
+    *) echo "Unsupported architecture: $(uname -m)"; exit 1 ;;
+  esac
+
+  DEST="/opt/cni/bin"
+  sudo mkdir -p "${DEST}"
+  curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${ARCH}-${CNI_PLUGINS_VERSION}.tgz" | sudo tar -C "${DEST}" -xz
+  curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | sudo tar -C /usr/local/bin -xz
+
+  sudo curl -L --remote-name-all https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/${ARCH}/{kubeadm,kubelet}
+  curl -LO "https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/${ARCH}/kubectl"
+
+  sudo install kubeadm kubelet /usr/bin/
+  sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+  curl -L --output 10-kubeadm.conf "https://github.com/kubernetes/release/raw/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf"
+  sudo mkdir -p /usr/lib/systemd/system/kubelet.service.d
+  sudo mv 10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
+  curl -L --output kubelet.service "https://github.com/kubernetes/release/raw/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service"
+  sudo mv kubelet.service /usr/lib/systemd/system/kubelet.service
+else
+  sudo yum install -y kubelet-"${KUBERNETES_BINARIES_VERSION//v}" kubeadm-"${KUBERNETES_BINARIES_VERSION//v}" kubectl-"${KUBERNETES_BINARIES_VERSION//v}" cri-tools-"${CRICTL_BINARIES_VERSION//v}"
+fi
+
+sudo yum install -y cri-o-"${CRIO_BINARIES_VERSION//v}"
 
 sudo pip install kubernetes

jenkins/image_building/dib_elements/centos-node/pre-install.d/55-setup-repos

@@ -8,14 +8,16 @@ export CRIO_MINOR_VERSION=${CRIO_VERSION%.*}
 sudo sed -i 's/enforcing/disabled/g' /etc/selinux/config /etc/selinux/config
 
 # migrate to the Kubernetes community-owned repositories
-cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
+if [[ "${PRE_RELEASE:-}" != "true" ]]; then
+  cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
 [kubernetes]
 name=Kubernetes
-baseurl=https://pkgs.k8s.io/core:/stable:/$KUBERNETES_MINOR_VERSION/rpm/
+baseurl=https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR_VERSION}/rpm/
 enabled=1
 gpgcheck=1
-gpgkey=https://pkgs.k8s.io/core:/stable:/$KUBERNETES_MINOR_VERSION/rpm/repodata/repomd.xml.key
+gpgkey=https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR_VERSION}/rpm/repodata/repomd.xml.key
 EOF
+fi
 
 cat <<EOF | tee /etc/yum.repos.d/cri-o.repo
 [cri-o]

jenkins/image_building/dib_elements/ubuntu-node/install.d/55-install

@@ -20,5 +20,35 @@ KUBERNETES_DEB_VERSION="${KUBERNETES_VERSION//v}"-\*
 CRIO_DEB_VERSION="${CRIO_VERSION//v}"-\*
 CRICTL_DEB_VERSION="${CRICTL_VERSION//v}"-\*
 
-sudo apt-get install -y kubelet="${KUBERNETES_DEB_VERSION}" kubeadm="${KUBERNETES_DEB_VERSION}" kubectl="${KUBERNETES_DEB_VERSION}" cri-o="${CRIO_DEB_VERSION}" cri-tools="${CRICTL_DEB_VERSION}"
-sudo apt-mark hold kubelet kubeadm kubectl cri-o cri-tools
+if [[ "${PRE_RELEASE:-}" == "true" ]]; then
+  # See https://github.com/containernetworking/plugins/releases
+  CNI_PLUGINS_VERSION="v1.7.1"
+
+  # Kubernetes release tooling
+  # See https://github.com/kubernetes/release/releases
+  RELEASE_VERSION="${RELEASE_VERSION:-v0.18.0}"
+
+  ARCH=$(dpkg --print-architecture)
+  DEST="/opt/cni/bin"
+  sudo mkdir -p "${DEST}"
+  curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${ARCH}-${CNI_PLUGINS_VERSION}.tgz" | sudo tar -C "${DEST}" -xz
+  curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | sudo tar -C /usr/local/bin -xz
+
+  sudo curl -L --remote-name-all "https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/${ARCH}/{kubeadm,kubelet}"
+  curl -LO "https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/${ARCH}/kubectl"
+
+  sudo install kubeadm kubelet /usr/bin/
+  sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+  curl -L --output 10-kubeadm.conf "https://github.com/kubernetes/release/raw/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf"
+  sudo mkdir -p /usr/lib/systemd/system/kubelet.service.d
+  sudo mv 10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
+  curl -L --output kubelet.service "https://github.com/kubernetes/release/raw/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service"
+  sudo mv kubelet.service /usr/lib/systemd/system/kubelet.service
+else
+  sudo apt-get install -y kubelet="${KUBERNETES_DEB_VERSION}" kubeadm="${KUBERNETES_DEB_VERSION}" kubectl="${KUBERNETES_DEB_VERSION}" cri-tools="${CRICTL_DEB_VERSION}"
+  sudo apt-mark hold kubelet kubeadm kubectl cri-tools
+fi
+
+sudo apt-get install -y cri-o="${CRIO_DEB_VERSION}"
+sudo apt-mark hold cri-o

jenkins/image_building/dib_elements/ubuntu-node/pre-install.d/55-setup-repos

@@ -5,8 +5,11 @@ set -eux
 export KUBERNETES_MINOR_VERSION=${KUBERNETES_VERSION%.*}
 export CRIO_MINOR_VERSION=${CRIO_VERSION%.*}
 
-curl -fsSL "https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR_VERSION}/deb/Release.key" | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
+if [[ "${PRE_RELEASE:-}" != "true" ]]; then
+  curl -fsSL "https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR_VERSION}/deb/Release.key" | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+  echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
+fi
+
 curl -fsSL "https://download.opensuse.org/repositories/isv:/cri-o:/stable:/${CRIO_MINOR_VERSION}/deb/Release.key" | gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg
 echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://download.opensuse.org/repositories/isv:/cri-o:/stable:/${CRIO_MINOR_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/cri-o.list
 

jenkins/jobs/image_building.pipeline

@@ -73,6 +73,7 @@ pipeline {
               echo "Building ${IMAGE_OS} ${IMAGE_TYPE} image"
               script {
                 sh """
+                def AGENT_OS = "ubuntu"
                 ./jenkins/image_building/build-image.sh
                 """
               }

jenkins/jobs/pre_release_tests.pipeline

@@ -0,0 +1,128 @@
+script {
+  UPDATED_REPO = "https://github.com/${env.REPO_OWNER}/${env.REPO_NAME}.git"
+  echo "Test triggered from ${UPDATED_REPO}"
+  ci_git_url = "https://github.com/metal3-io/project-infra.git"
+
+  if ( "${env.REPO_OWNER}" == 'metal3-io' && "${env.REPO_NAME}" == 'project-infra' ) {
+    ci_git_branch = (env.PULL_PULL_SHA) ?: 'main'
+    ci_git_base = (env.PULL_BASE_REF) ?: 'main'
+    // Fetch the base branch and the ci_git_branch when running on project-infra PR
+    refspec = '+refs/heads/' + ci_git_base + ':refs/remotes/origin/' + ci_git_base + ' ' + ci_git_branch
+  } else {
+    ci_git_branch = 'main'
+    refspec = '+refs/heads/*:refs/remotes/origin/*'
+  }
+  echo "Checkout ${ci_git_url} branch ${ci_git_branch}"
+}
+
+pipeline {
+  agent { label "metal3ci-8c16gb-${IMAGE_OS}" }
+  environment {
+    AGENT_OS = "${IMAGE_OS}"
+    REPO_ORG = "${env.REPO_OWNER}"
+    REPO_NAME = "${env.REPO_NAME}"
+    UPDATED_REPO = "${UPDATED_REPO}"
+    REPO_BRANCH = "${env.PULL_BASE_REF ?: capm3_release_branch}"
+    UPDATED_BRANCH = "${env.PULL_PULL_SHA ?: capm3_release_branch}"
+    BUILD_TAG = "${env.BUILD_TAG}"
+    PR_ID = "${env.PULL_NUMBER ?: ''}"
+    IMAGE_OS = "${IMAGE_OS}"
+    PRE_RELEASE = "true" // Affects the way k8s is installed in node image building
+    IMAGE_TYPE = "node"
+    KUBERNETES_VERSION = "v1.34.0-beta.0"
+    CRICTL_VERSION = "${CRICTL_VERSION}"
+    CRIO_VERSION = "${CRIO_VERSION}"
+    CAPM3RELEASEBRANCH = "${capm3_release_branch}"
+    BMORELEASEBRANCH = "${bmo_release_branch}"
+    CAPI_VERSION = "${CAPI_VERSION}"
+    CAPM3_VERSION = "${CAPM3_VERSION}"
+    GINKGO_FOCUS = "${GINKGO_FOCUS}"
+    GINKGO_SKIP = "${GINKGO_SKIP}"
+    NUM_NODES = "${NUM_NODES}"
+    TARGET_NODE_MEMORY = "${TARGET_NODE_MEMORY}"
+  }
+  stages {
+    stage("Checkout CI Repo") {
+      options {
+        timeout(time: 5, unit: 'MINUTES')
+      }
+      steps {
+        checkout([
+          $class: 'GitSCM',
+          branches: [[name: ci_git_branch]],
+          doGenerateSubmoduleConfigurations: false,
+          extensions: [
+            [$class: 'WipeWorkspace'],
+            [$class: 'CleanCheckout'],
+            [$class: 'CleanBeforeCheckout']
+          ],
+          submoduleCfg: [],
+          userRemoteConfigs: [[url: ci_git_url, refspec: refspec]]
+        ])
+      }
+    }
+    stage("Build disk image") {
+      options {
+        timeout(time: 1, unit: 'HOURS')
+      }
+      steps {
+        echo "Building ${IMAGE_OS} node image"
+        script {
+          sh """
+          ./jenkins/image_building/build-image.sh
+          """
+        }
+      }
+    }
+    stage("Run e2e tests") {
+      options {
+        timeout(time: 3, unit: 'HOURS')
+      }
+      steps {
+        script {
+          START_TIME = System.currentTimeMillis()
+
+          // Set image name and location as env variables
+          // to use the local image in the testing
+          def img_name = readFile("image_name.txt").trim()
+
+          env.IMAGE_NAME = "${img_name}.qcow2"
+          env.IMAGE_LOCATION = env.WORKSPACE
+
+          echo "Set IMAGE_NAME to: ${env.IMAGE_NAME}"
+          echo "Set IMAGE_LOCATION to: ${env.IMAGE_LOCATION}"
+        }
+        echo "Testing with the new ${IMAGE_OS} node image"
+        withCredentials([string(credentialsId: 'metal3-clusterctl-github-token', variable: 'GITHUB_TOKEN')]) {
+          ansiColor('xterm') {
+            timestamps {
+                sh './jenkins/scripts/dynamic_worker_workflow/e2e_tests.sh'
+            }
+          }
+        }
+      }
+      post {
+        always {
+          script {
+            END_TIME = System.currentTimeMillis()
+            if ((((END_TIME - START_TIME) / 1000) - 10800) > 0) {
+              echo 'Failed due to timeout'
+              currentBuild.result = 'FAILURE'
+            }
+            timestamps {
+              sh './jenkins/scripts/dynamic_worker_workflow/fetch_logs.sh'
+              archiveArtifacts "logs-${env.BUILD_TAG}.tgz"
+            }
+          }
+        }
+        cleanup {
+          script {
+            timestamps {
+              sh './jenkins/scripts/dynamic_worker_workflow/run_clean.sh'
+            }
+          }
+        }
+      }
+    }
+  }
+}