From 1faddf733d7ae5c8e60161457074b4365fc63aa7 Mon Sep 17 00:00:00 2001 From: peppi-lotta Date: Wed, 29 Oct 2025 15:08:27 +0200 Subject: [PATCH 1/3] Add quick-start test files (virtualized) Signed-off-by: peppi-lotta --- .gitignore | 5 + .../bmh-manifest-with-image.yaml | 17 +++ hack/quick-start-test/bml-01.yaml | 8 ++ hack/quick-start-test/bml-vm-01.yaml | 12 ++ hack/quick-start-test/bmo/ironic-password | 1 + hack/quick-start-test/bmo/ironic-username | 1 + hack/quick-start-test/bmo/ironic.env | 3 + hack/quick-start-test/bmo/kustomization.yaml | 32 +++++ hack/quick-start-test/cleanup.sh | 15 +++ hack/quick-start-test/dnsmasq.env | 7 ++ hack/quick-start-test/ironic/ironic-htpasswd | 1 + .../quick-start-test/ironic/ironic-patch.yaml | 10 ++ hack/quick-start-test/ironic/ironic_bmo.env | 7 ++ .../ironic/kustomization.yaml | 50 ++++++++ hack/quick-start-test/kind.yaml | 16 +++ hack/quick-start-test/net.xml | 15 +++ hack/quick-start-test/quick-start-test.sh | 116 ++++++++++++++++++ hack/quick-start-test/setup-bootstrap.sh | 5 + hack/quick-start-test/setup-dhcp-server.sh | 3 + .../setup-image-server-dir.sh | 14 +++ hack/quick-start-test/setup-virtual-lab.sh | 26 ++++ hack/quick-start-test/start-image-server.sh | 3 + hack/quick-start-test/sushy-tools.conf | 36 ++++++ 23 files changed, 403 insertions(+) create mode 100644 hack/quick-start-test/bmh-manifest-with-image.yaml create mode 100644 hack/quick-start-test/bml-01.yaml create mode 100644 hack/quick-start-test/bml-vm-01.yaml create mode 100644 hack/quick-start-test/bmo/ironic-password create mode 100644 hack/quick-start-test/bmo/ironic-username create mode 100644 hack/quick-start-test/bmo/ironic.env create mode 100644 hack/quick-start-test/bmo/kustomization.yaml create mode 100755 hack/quick-start-test/cleanup.sh create mode 100644 hack/quick-start-test/dnsmasq.env create mode 100644 hack/quick-start-test/ironic/ironic-htpasswd create mode 100644 hack/quick-start-test/ironic/ironic-patch.yaml create mode 100644 hack/quick-start-test/ironic/ironic_bmo.env create mode 100644 hack/quick-start-test/ironic/kustomization.yaml create mode 100644 hack/quick-start-test/kind.yaml create mode 100644 hack/quick-start-test/net.xml create mode 100755 hack/quick-start-test/quick-start-test.sh create mode 100755 hack/quick-start-test/setup-bootstrap.sh create mode 100755 hack/quick-start-test/setup-dhcp-server.sh create mode 100755 hack/quick-start-test/setup-image-server-dir.sh create mode 100755 hack/quick-start-test/setup-virtual-lab.sh create mode 100755 hack/quick-start-test/start-image-server.sh create mode 100644 hack/quick-start-test/sushy-tools.conf diff --git a/.gitignore b/.gitignore index 1a64c9170..f6ace35c9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,11 @@ .DS_Store *.swp +# Quick start test temp files +hack/quick-start-test/disk-images +hack/quick-start-test/bmh-vm-01.xml + + # Development containers .devcontainer diff --git a/hack/quick-start-test/bmh-manifest-with-image.yaml b/hack/quick-start-test/bmh-manifest-with-image.yaml new file mode 100644 index 000000000..1791a45df --- /dev/null +++ b/hack/quick-start-test/bmh-manifest-with-image.yaml @@ -0,0 +1,17 @@ +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: bml-01 +spec: + online: true + bootMACAddress: 80:c1:6e:7a:e8:10 + bootMode: legacy + bmc: + address: ilo4-virtualmedia://192.168.1.13 + credentialsName: bml-01 + disableCertificateVerification: true + image: + checksumType: sha256 + checksum: http://192.168.0.150/SHA256SUMS + format: qcow2 + url: http://192.168.0.150/jammy-server-cloudimg-amd64.img diff --git a/hack/quick-start-test/bml-01.yaml b/hack/quick-start-test/bml-01.yaml new file mode 100644 index 000000000..3571d4247 --- /dev/null +++ b/hack/quick-start-test/bml-01.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: bml-01 +type: Opaque +stringData: + username: admin #replace with desired username + password: supersecretpassword #replace with desired password diff --git a/hack/quick-start-test/bml-vm-01.yaml b/hack/quick-start-test/bml-vm-01.yaml new file mode 100644 index 000000000..afa556a0a --- /dev/null +++ b/hack/quick-start-test/bml-vm-01.yaml @@ -0,0 +1,12 @@ +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: bml-vm-01 +spec: + online: true + bootMACAddress: 00:60:2f:31:81:01 + bootMode: UEFI # use 'legacy' for Scenario 2 + hardwareProfile: libvirt + bmc: + address: redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmh-vm-01 + credentialsName: bml-01 diff --git a/hack/quick-start-test/bmo/ironic-password b/hack/quick-start-test/bmo/ironic-password new file mode 100644 index 000000000..89957080f --- /dev/null +++ b/hack/quick-start-test/bmo/ironic-password @@ -0,0 +1 @@ +supersecretpassword diff --git a/hack/quick-start-test/bmo/ironic-username b/hack/quick-start-test/bmo/ironic-username new file mode 100644 index 000000000..7fbe952b7 --- /dev/null +++ b/hack/quick-start-test/bmo/ironic-username @@ -0,0 +1 @@ +admin diff --git a/hack/quick-start-test/bmo/ironic.env b/hack/quick-start-test/bmo/ironic.env new file mode 100644 index 000000000..44147ae0f --- /dev/null +++ b/hack/quick-start-test/bmo/ironic.env @@ -0,0 +1,3 @@ +DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel +DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs +IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/ diff --git a/hack/quick-start-test/bmo/kustomization.yaml b/hack/quick-start-test/bmo/kustomization.yaml new file mode 100644 index 000000000..80299aeb8 --- /dev/null +++ b/hack/quick-start-test/bmo/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: baremetal-operator-system +# This is the kustomization that we build on. You can download it and change +# the URL to a relative path if you do not want to access it over the network. +# Note that the ref=v0.11.0 specifies the version to use. +resources: +- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=v0.11.0 +- https://github.com/metal3-io/baremetal-operator/config/base?ref=v0.11.0 +components: +- https://github.com/metal3-io/baremetal-operator/config/components/basic-auth?ref=v0.11.0 +- https://github.com/metal3-io/baremetal-operator/config/components/tls?ref=v0.11.0 +images: +- name: quay.io/metal3-io/baremetal-operator + newTag: v0.11.0 +# Create a ConfigMap from ironic.env and name it ironic. +configMapGenerator: +- name: ironic + behavior: create + envs: + - ironic.env + +# We cannot use suffix hashes since the kustomizations we build on +# cannot be aware of what suffixes we add. +generatorOptions: + disableNameSuffixHash: true +# Create secrets with the credentials for accessing Ironic. +secretGenerator: +- name: ironic-credentials + files: + - username=ironic-username + - password=ironic-password diff --git a/hack/quick-start-test/cleanup.sh b/hack/quick-start-test/cleanup.sh new file mode 100755 index 000000000..d1ab64965 --- /dev/null +++ b/hack/quick-start-test/cleanup.sh @@ -0,0 +1,15 @@ +# Delete the management cluster. +kind delete cluster + +# Stop DHCP and image servers. They are automatically removed when stopped. +docker stop dnsmasq +docker stop image-server + +# Cleanup the sushy-tools container and the VM. (For virtual-lab setup) +docker stop sushy-tools + +virsh -c qemu:///system destroy --domain bmh-vm-01 +virsh -c qemu:///system undefine --domain bmh-vm-01 --remove-all-storage --nvram + +virsh -c qemu:///system net-destroy baremetal-e2e +virsh -c qemu:///system net-undefine baremetal-e2e \ No newline at end of file diff --git a/hack/quick-start-test/dnsmasq.env b/hack/quick-start-test/dnsmasq.env new file mode 100644 index 000000000..5bf581974 --- /dev/null +++ b/hack/quick-start-test/dnsmasq.env @@ -0,0 +1,7 @@ +HTTP_PORT=6180 +DHCP_HOSTS=00:60:2f:31:81:01 +DHCP_IGNORE=tag:!known +# IP of the host from VM perspective +PROVISIONING_IP=192.168.222.1 +GATEWAY_IP=192.168.222.1 +DHCP_RANGE=192.168.222.100,192.168.222.149 diff --git a/hack/quick-start-test/ironic/ironic-htpasswd b/hack/quick-start-test/ironic/ironic-htpasswd new file mode 100644 index 000000000..dd02ae75f --- /dev/null +++ b/hack/quick-start-test/ironic/ironic-htpasswd @@ -0,0 +1 @@ +IRONIC_HTPASSWD="" diff --git a/hack/quick-start-test/ironic/ironic-patch.yaml b/hack/quick-start-test/ironic/ironic-patch.yaml new file mode 100644 index 000000000..098d21e10 --- /dev/null +++ b/hack/quick-start-test/ironic/ironic-patch.yaml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ironic +spec: + template: + spec: + containers: + - name: ironic-dnsmasq + $patch: delete diff --git a/hack/quick-start-test/ironic/ironic_bmo.env b/hack/quick-start-test/ironic/ironic_bmo.env new file mode 100644 index 000000000..07e2bb53a --- /dev/null +++ b/hack/quick-start-test/ironic/ironic_bmo.env @@ -0,0 +1,7 @@ +HTTP_PORT=6180 +PROVISIONING_INTERFACE=eth0 +CACHEURL=http://192.168.222.1/images +IRONIC_KERNEL_PARAMS=console=ttyS0 +# Docker does not allow cross-network access. If using kind to create the management +# cluster, explicitly set the external ip and use port forwarding to access ironic services. +IRONIC_EXTERNAL_IP=192.168.222.1 diff --git a/hack/quick-start-test/ironic/kustomization.yaml b/hack/quick-start-test/ironic/kustomization.yaml new file mode 100644 index 000000000..80bb2b894 --- /dev/null +++ b/hack/quick-start-test/ironic/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: baremetal-operator-system +# These are the kustomizations we build on. You can download them and change the URLs to relative +# paths if you do not want to access them over the network. +# Note that the ref=v0.11.0 specifies the version to use. +resources: +- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=v0.11.0 +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=v0.11.0 +# The kustomize components configure basic-auth and TLS +components: +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=v0.11.0 +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=v0.11.0 +images: +- name: quay.io/metal3-io/ironic + newTag: v32.0.0 +# Create a ConfigMap from ironic_bmo.env and call it ironic-bmo-configmap. +# This ConfigMap will be used to set environment variables for the containers. +configMapGenerator: +- envs: + - ironic_bmo.env + name: ironic-bmo-configmap + behavior: create + +patches: +# Patch for removing dnsmasq +- path: ironic-patch.yaml +# The TLS component adds certificates but it cannot know the exact IPs of our environment. +# Here we patch the certificates to have the correct IPs. +# - 192.168.1.7: management computer IP in out of band network +# - 172.18.0.2: kind cluster node IP. This is what Ironic will see attached to the interface. +# - 192.168.0.150: management computer IP in the other network +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + - op: add + path: /spec/ipAddresses/- + value: 172.18.0.2 + target: + kind: Certificate + name: ironic-cert +# Create secrets from the authentication configuration. +# These will be mounted or used for environment variables. +# See the basic-auth component for more details on how they are used. +secretGenerator: +- name: ironic-htpasswd + behavior: create + envs: + - ironic-htpasswd diff --git a/hack/quick-start-test/kind.yaml b/hack/quick-start-test/kind.yaml new file mode 100644 index 000000000..ddda1a204 --- /dev/null +++ b/hack/quick-start-test/kind.yaml @@ -0,0 +1,16 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +nodes: +- role: control-plane + # Open ports for Ironic + extraPortMappings: + # Ironic httpd + - containerPort: 6180 + hostPort: 6180 + listenAddress: "0.0.0.0" + protocol: TCP + # Ironic API + - containerPort: 6385 + hostPort: 6385 + listenAddress: "0.0.0.0" + protocol: TCP diff --git a/hack/quick-start-test/net.xml b/hack/quick-start-test/net.xml new file mode 100644 index 000000000..226d5a788 --- /dev/null +++ b/hack/quick-start-test/net.xml @@ -0,0 +1,15 @@ + + baremetal-e2e + + + + + + + + + + + + + diff --git a/hack/quick-start-test/quick-start-test.sh b/hack/quick-start-test/quick-start-test.sh new file mode 100755 index 000000000..da86eb206 --- /dev/null +++ b/hack/quick-start-test/quick-start-test.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +#------------------------------------------------------------------------------ +# This script sets up a quick start test environment for Metal3 by +# configuring a virtual lab, bootstrapping a Kind cluster, setting up +# DHCP and image servers, and deploying Ironic and baremetal operators. +#------------------------------------------------------------------------------ +set -eux + +export QUICK_START_BASE="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")" + +setup() { + echo "Disk images directory. If disk images are missing, they will be downloaded and prepared." + setup_disk_images_dir + + echo "Setting up virtual lab..." + "${QUICK_START_BASE}/setup-virtual-lab.sh" + # Check if the VM is running + if ! virsh -c qemu:///system dominfo bmh-vm-01 &> /dev/null; then + echo "Error: The VM bmh-vm-01 is not running." + exit 1 + fi + + echo "Bootstrapping Kind cluster..." + "${QUICK_START_BASE}/setup-bootstrap.sh" + # Wait for all nodes to be ready + if ! kubectl wait --for=condition=Ready nodes --all --timeout=300s; then + exit 1 + fi + # Wait for cert-manager and webhook to be ready + wait_for_cert_manager_ready + + echo "Setting up DHCP and image servers..." + "${QUICK_START_BASE}/start-image-server.sh" + "${QUICK_START_BASE}/setup-dhcp-server.sh" + + echo "Deploying Ironic..." + kubectl apply -k ironic + wait_for_ironic_ready + + echo "Deploying Baremetal Operator..." + kubectl apply -k bmo + wait_for_bmo_ready +} + +create_bmhs() { + kubectl apply -f "${QUICK_START_BASE}/bml-vm-01.yaml" + kubectl apply -f "${QUICK_START_BASE}/bml-01.yaml" + # Wait for BMHs to be provisioned + wait_for_bml_ready +} + +setup_disk_images_dir() { + DISK_IMAGE_DIR="${QUICK_START_BASE}/disk-images" + REQUIRED_FILES=( + "noble-server-cloudimg-amd64.img" + "CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" + "CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2" + ) + + missing_files=0 + for file in "${REQUIRED_FILES[@]}"; do + if [ ! -f "${DISK_IMAGE_DIR}/${file}" ]; then + missing_files=1 + break + fi + done + + if [ "$missing_files" -eq 1 ]; then + rm -r ${DISK_IMAGE_DIR} || true + echo "Setting up disk images directory..." + "${QUICK_START_BASE}/setup-image-server-dir.sh" + else + echo "All required disk images are present." + fi +} + +wait_for_cert_manager_ready() { + echo "Waiting for cert-manager to be ready... This may take up to 5 minutes." + if ! kubectl wait --for=condition=Available --timeout=60s deployment/cert-manager -n cert-manager; then + exit 1 + fi + if ! kubectl wait --for=condition=Available --timeout=60s deployment/cert-manager-webhook -n cert-manager; then + exit 1 + fi + if ! kubectl wait --for=condition=Available --timeout=60s deployment/cert-manager-cainjector -n cert-manager; then + exit 1 + fi + if ! kubectl wait --for=condition=Ready --timeout=600s pod -l app.kubernetes.io/name=webhook -n cert-manager; then + exit 1 + fi +} + +wait_for_bmo_ready() { + echo "Waiting for Baremetal Operator to be ready... This may take up to 5 minutes." + if ! kubectl wait --for=condition=Available --timeout=300s deployment/baremetal-operator-controller-manager -n baremetal-operator-system; then + exit 1 + fi +} + +wait_for_ironic_ready() { + echo "Waiting for Ironic to be ready... This may take up to 10 minutes." + if ! kubectl wait --for=condition=Available --timeout=600s deployment/ironic -n baremetal-operator-system; then + exit 1 + fi +} + +wait_for_bml_ready() { + echo "Waiting for BareMetalHosts to be provisioned... This may take up to 5 minutes." + if ! kubectl wait --for=condition=Available --timeout=300s baremetalhosts --all; then + exit 1 + fi +} + +setup +create_bmhs diff --git a/hack/quick-start-test/setup-bootstrap.sh b/hack/quick-start-test/setup-bootstrap.sh new file mode 100755 index 000000000..4c0834621 --- /dev/null +++ b/hack/quick-start-test/setup-bootstrap.sh @@ -0,0 +1,5 @@ +# Create a kind cluster to act as the management cluster +kind create cluster --config kind.yaml + +# Install cert-manager. It will be used to manage the certificates for Ironic +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml diff --git a/hack/quick-start-test/setup-dhcp-server.sh b/hack/quick-start-test/setup-dhcp-server.sh new file mode 100755 index 000000000..8be39c810 --- /dev/null +++ b/hack/quick-start-test/setup-dhcp-server.sh @@ -0,0 +1,3 @@ +docker run --name dnsmasq --rm -d --net=host --privileged --user 997:994 \ + --env-file dnsmasq.env --entrypoint /bin/rundnsmasq \ + quay.io/metal3-io/ironic diff --git a/hack/quick-start-test/setup-image-server-dir.sh b/hack/quick-start-test/setup-image-server-dir.sh new file mode 100755 index 000000000..21f6ce3ce --- /dev/null +++ b/hack/quick-start-test/setup-image-server-dir.sh @@ -0,0 +1,14 @@ +# Create a directory to hold the disk images: +mkdir ${QUICK_START_BASE}/disk-images + +# Download images to use for testing (pick those that you want): +pushd ${QUICK_START_BASE}/disk-images +wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img +wget https://cloud-images.ubuntu.com/noble/current/SHA256SUMS +sha256sum --ignore-missing -c SHA256SUMS +wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2 +wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM +sha256sum -c CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM +wget https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.33.0/CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2 +sha256sum CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2 +popd diff --git a/hack/quick-start-test/setup-virtual-lab.sh b/hack/quick-start-test/setup-virtual-lab.sh new file mode 100755 index 000000000..6d6d064ee --- /dev/null +++ b/hack/quick-start-test/setup-virtual-lab.sh @@ -0,0 +1,26 @@ +# use --ram=8192 for Scenario 2 + +# Define and start the baremetal-e2e network +virsh -c qemu:///system net-define net.xml +virsh -c qemu:///system net-start baremetal-e2e + +# Start the sushy-emulator container that acts as BMC +docker run --name sushy-tools --rm --network host -d \ + -v /var/run/libvirt:/var/run/libvirt \ + -v "${QUICK_START_BASE}/sushy-tools.conf:/etc/sushy/sushy-emulator.conf" \ + -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \ + quay.io/metal3-io/sushy-tools:latest sushy-emulator + +# Generate a VM definition xml file and then define the VM +virt-install \ + --connect qemu:///system \ + --name bmh-vm-01 \ + --description "Virtualized BareMetalHost" \ + --osinfo=ubuntu-lts-latest \ + --ram=4096 \ + --vcpus=2 \ + --disk size=25 \ + --boot hd,network \ + --import \ + --network network=baremetal-e2e,mac="00:60:2f:31:81:01" \ + --noautoconsole diff --git a/hack/quick-start-test/start-image-server.sh b/hack/quick-start-test/start-image-server.sh new file mode 100755 index 000000000..9a090144d --- /dev/null +++ b/hack/quick-start-test/start-image-server.sh @@ -0,0 +1,3 @@ +#Run a basic http server to expose the disk images: +docker run --name image-server --rm -d -p 80:8080 \ + -v "${QUICK_START_BASE}/disk-images:/usr/share/nginx/html" nginxinc/nginx-unprivileged diff --git a/hack/quick-start-test/sushy-tools.conf b/hack/quick-start-test/sushy-tools.conf new file mode 100644 index 000000000..fade24e2f --- /dev/null +++ b/hack/quick-start-test/sushy-tools.conf @@ -0,0 +1,36 @@ +# Listen on the local IP address 192.168.222.1 +SUSHY_EMULATOR_LISTEN_IP = u'192.168.222.1' + +# Bind to TCP port 8000 +SUSHY_EMULATOR_LISTEN_PORT = 8000 + +# Serve this SSL certificate to the clients +SUSHY_EMULATOR_SSL_CERT = None + +# If SSL certificate is being served, this is its RSA private key +SUSHY_EMULATOR_SSL_KEY = None + +# The OpenStack cloud ID to use. This option enables OpenStack driver. +SUSHY_EMULATOR_OS_CLOUD = None +# The libvirt URI to use. This option enables libvirt driver. +SUSHY_EMULATOR_LIBVIRT_URI = u'qemu:///system' + +# Instruct the libvirt driver to ignore any instructions to +# set the boot device. Allowing the UEFI firmware to instead +# rely on the EFI Boot Manager +# Note: This sets the legacy boot element to dev="fd" +# and relies on the floppy not existing, it likely wont work +# your VM has a floppy drive. +SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = False + +# The map of firmware loaders dependant on the boot mode and +# system architecture. Ideally the x86_64 loader will be capable +# of secure boot or not based on the chosen nvram. +SUSHY_EMULATOR_BOOT_LOADER_MAP = { + u'UEFI': { + u'x86_64': u'/usr/share/OVMF/OVMF_CODE_4M.secboot.fd' + }, + u'Legacy': { + u'x86_64': None + } +} From c79d57aa28695c231032c259a3d28ae1803b1479 Mon Sep 17 00:00:00 2001 From: peppi-lotta Date: Wed, 5 Nov 2025 13:30:52 +0000 Subject: [PATCH 2/3] Change to irso in quick-start-test Signed-off-by: peppi-lotta --- .github/workflows/quick-start-test.yml | 0 .gitignore | 4 + .../bmh-manifest-with-image.yaml | 17 ---- hack/quick-start-test/bml-vm-01.yaml | 2 +- hack/quick-start-test/bmo/ironic-password | 1 - hack/quick-start-test/bmo/ironic-username | 1 - hack/quick-start-test/bmo/ironic.env | 2 +- hack/quick-start-test/bmo/kustomization.yaml | 23 +---- hack/quick-start-test/cleanup.sh | 5 +- hack/quick-start-test/dnsmasq.env | 4 +- hack/quick-start-test/ironic/ironic-htpasswd | 1 - .../quick-start-test/ironic/ironic-patch.yaml | 10 -- hack/quick-start-test/ironic/ironic_bmo.env | 7 -- .../ironic/kustomization.yaml | 50 ---------- hack/quick-start-test/irso/ironic.yaml | 15 +++ hack/quick-start-test/irso/kustomization.yaml | 41 ++++++++ hack/quick-start-test/quick-start-test.sh | 96 ++++++++++++------- .../setup-image-server-dir.sh | 3 +- hack/quick-start-test/setup-virtual-lab.sh | 51 +++++++++- hack/quick-start-test/sushy-tools.conf | 2 +- 20 files changed, 187 insertions(+), 148 deletions(-) create mode 100644 .github/workflows/quick-start-test.yml delete mode 100644 hack/quick-start-test/bmh-manifest-with-image.yaml delete mode 100644 hack/quick-start-test/bmo/ironic-password delete mode 100644 hack/quick-start-test/bmo/ironic-username delete mode 100644 hack/quick-start-test/ironic/ironic-htpasswd delete mode 100644 hack/quick-start-test/ironic/ironic-patch.yaml delete mode 100644 hack/quick-start-test/ironic/ironic_bmo.env delete mode 100644 hack/quick-start-test/ironic/kustomization.yaml create mode 100644 hack/quick-start-test/irso/ironic.yaml create mode 100644 hack/quick-start-test/irso/kustomization.yaml diff --git a/.github/workflows/quick-start-test.yml b/.github/workflows/quick-start-test.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.gitignore b/.gitignore index f6ace35c9..df07277af 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ # Quick start test temp files hack/quick-start-test/disk-images hack/quick-start-test/bmh-vm-01.xml +hack/quick-start-test/bmo/ironic-username +hack/quick-start-test/bmo/ironic-password +hack/quick-start-test/irso/ironic-username +hack/quick-start-test/irso/ironic-password # Development containers diff --git a/hack/quick-start-test/bmh-manifest-with-image.yaml b/hack/quick-start-test/bmh-manifest-with-image.yaml deleted file mode 100644 index 1791a45df..000000000 --- a/hack/quick-start-test/bmh-manifest-with-image.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - name: bml-01 -spec: - online: true - bootMACAddress: 80:c1:6e:7a:e8:10 - bootMode: legacy - bmc: - address: ilo4-virtualmedia://192.168.1.13 - credentialsName: bml-01 - disableCertificateVerification: true - image: - checksumType: sha256 - checksum: http://192.168.0.150/SHA256SUMS - format: qcow2 - url: http://192.168.0.150/jammy-server-cloudimg-amd64.img diff --git a/hack/quick-start-test/bml-vm-01.yaml b/hack/quick-start-test/bml-vm-01.yaml index afa556a0a..fbc40a682 100644 --- a/hack/quick-start-test/bml-vm-01.yaml +++ b/hack/quick-start-test/bml-vm-01.yaml @@ -5,7 +5,7 @@ metadata: spec: online: true bootMACAddress: 00:60:2f:31:81:01 - bootMode: UEFI # use 'legacy' for Scenario 2 + bootMode: legacy # use 'UEFI' for Scenario 1 hardwareProfile: libvirt bmc: address: redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmh-vm-01 diff --git a/hack/quick-start-test/bmo/ironic-password b/hack/quick-start-test/bmo/ironic-password deleted file mode 100644 index 89957080f..000000000 --- a/hack/quick-start-test/bmo/ironic-password +++ /dev/null @@ -1 +0,0 @@ -supersecretpassword diff --git a/hack/quick-start-test/bmo/ironic-username b/hack/quick-start-test/bmo/ironic-username deleted file mode 100644 index 7fbe952b7..000000000 --- a/hack/quick-start-test/bmo/ironic-username +++ /dev/null @@ -1 +0,0 @@ -admin diff --git a/hack/quick-start-test/bmo/ironic.env b/hack/quick-start-test/bmo/ironic.env index 44147ae0f..8fa448ce5 100644 --- a/hack/quick-start-test/bmo/ironic.env +++ b/hack/quick-start-test/bmo/ironic.env @@ -1,3 +1,3 @@ DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/ +IRONIC_ENDPOINT=https://192.168.222.2:6385/v1/ diff --git a/hack/quick-start-test/bmo/kustomization.yaml b/hack/quick-start-test/bmo/kustomization.yaml index 80299aeb8..c5f0a3334 100644 --- a/hack/quick-start-test/bmo/kustomization.yaml +++ b/hack/quick-start-test/bmo/kustomization.yaml @@ -3,30 +3,17 @@ kind: Kustomization namespace: baremetal-operator-system # This is the kustomization that we build on. You can download it and change # the URL to a relative path if you do not want to access it over the network. -# Note that the ref=v0.11.0 specifies the version to use. +# Note that the ref=main specifies the version to use. +# We use main here simply because the integration with IrSO is not included in a release yet. resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/config/base?ref=v0.11.0 -components: -- https://github.com/metal3-io/baremetal-operator/config/components/basic-auth?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/config/components/tls?ref=v0.11.0 -images: -- name: quay.io/metal3-io/baremetal-operator - newTag: v0.11.0 -# Create a ConfigMap from ironic.env and name it ironic. -configMapGenerator: -- name: ironic - behavior: create - envs: - - ironic.env +- https://github.com/metal3-io/baremetal-operator/config/use-irso?ref=main -# We cannot use suffix hashes since the kustomizations we build on -# cannot be aware of what suffixes we add. generatorOptions: disableNameSuffixHash: true -# Create secrets with the credentials for accessing Ironic. + secretGenerator: - name: ironic-credentials + namespace: baremetal-operator-system files: - username=ironic-username - password=ironic-password diff --git a/hack/quick-start-test/cleanup.sh b/hack/quick-start-test/cleanup.sh index d1ab64965..99981530f 100755 --- a/hack/quick-start-test/cleanup.sh +++ b/hack/quick-start-test/cleanup.sh @@ -12,4 +12,7 @@ virsh -c qemu:///system destroy --domain bmh-vm-01 virsh -c qemu:///system undefine --domain bmh-vm-01 --remove-all-storage --nvram virsh -c qemu:///system net-destroy baremetal-e2e -virsh -c qemu:///system net-undefine baremetal-e2e \ No newline at end of file +virsh -c qemu:///system net-undefine baremetal-e2e + +export QUICK_START_BASE=${QUICK_START_BASE:="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")"} +rm -rf "${QUICK_START_BASE}/bmh-vm-01.xml" \ No newline at end of file diff --git a/hack/quick-start-test/dnsmasq.env b/hack/quick-start-test/dnsmasq.env index 5bf581974..774a73a46 100644 --- a/hack/quick-start-test/dnsmasq.env +++ b/hack/quick-start-test/dnsmasq.env @@ -2,6 +2,6 @@ HTTP_PORT=6180 DHCP_HOSTS=00:60:2f:31:81:01 DHCP_IGNORE=tag:!known # IP of the host from VM perspective -PROVISIONING_IP=192.168.222.1 -GATEWAY_IP=192.168.222.1 +PROVISIONING_IP=192.168.222.2 +GATEWAY_IP=192.168.222.2 DHCP_RANGE=192.168.222.100,192.168.222.149 diff --git a/hack/quick-start-test/ironic/ironic-htpasswd b/hack/quick-start-test/ironic/ironic-htpasswd deleted file mode 100644 index dd02ae75f..000000000 --- a/hack/quick-start-test/ironic/ironic-htpasswd +++ /dev/null @@ -1 +0,0 @@ -IRONIC_HTPASSWD="" diff --git a/hack/quick-start-test/ironic/ironic-patch.yaml b/hack/quick-start-test/ironic/ironic-patch.yaml deleted file mode 100644 index 098d21e10..000000000 --- a/hack/quick-start-test/ironic/ironic-patch.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ironic -spec: - template: - spec: - containers: - - name: ironic-dnsmasq - $patch: delete diff --git a/hack/quick-start-test/ironic/ironic_bmo.env b/hack/quick-start-test/ironic/ironic_bmo.env deleted file mode 100644 index 07e2bb53a..000000000 --- a/hack/quick-start-test/ironic/ironic_bmo.env +++ /dev/null @@ -1,7 +0,0 @@ -HTTP_PORT=6180 -PROVISIONING_INTERFACE=eth0 -CACHEURL=http://192.168.222.1/images -IRONIC_KERNEL_PARAMS=console=ttyS0 -# Docker does not allow cross-network access. If using kind to create the management -# cluster, explicitly set the external ip and use port forwarding to access ironic services. -IRONIC_EXTERNAL_IP=192.168.222.1 diff --git a/hack/quick-start-test/ironic/kustomization.yaml b/hack/quick-start-test/ironic/kustomization.yaml deleted file mode 100644 index 80bb2b894..000000000 --- a/hack/quick-start-test/ironic/kustomization.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: baremetal-operator-system -# These are the kustomizations we build on. You can download them and change the URLs to relative -# paths if you do not want to access them over the network. -# Note that the ref=v0.11.0 specifies the version to use. -resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=v0.11.0 -# The kustomize components configure basic-auth and TLS -components: -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=v0.11.0 -images: -- name: quay.io/metal3-io/ironic - newTag: v32.0.0 -# Create a ConfigMap from ironic_bmo.env and call it ironic-bmo-configmap. -# This ConfigMap will be used to set environment variables for the containers. -configMapGenerator: -- envs: - - ironic_bmo.env - name: ironic-bmo-configmap - behavior: create - -patches: -# Patch for removing dnsmasq -- path: ironic-patch.yaml -# The TLS component adds certificates but it cannot know the exact IPs of our environment. -# Here we patch the certificates to have the correct IPs. -# - 192.168.1.7: management computer IP in out of band network -# - 172.18.0.2: kind cluster node IP. This is what Ironic will see attached to the interface. -# - 192.168.0.150: management computer IP in the other network -- patch: |- - - op: replace - path: /spec/ipAddresses/0 - value: 192.168.222.1 - - op: add - path: /spec/ipAddresses/- - value: 172.18.0.2 - target: - kind: Certificate - name: ironic-cert -# Create secrets from the authentication configuration. -# These will be mounted or used for environment variables. -# See the basic-auth component for more details on how they are used. -secretGenerator: -- name: ironic-htpasswd - behavior: create - envs: - - ironic-htpasswd diff --git a/hack/quick-start-test/irso/ironic.yaml b/hack/quick-start-test/irso/ironic.yaml new file mode 100644 index 000000000..a42d0b61d --- /dev/null +++ b/hack/quick-start-test/irso/ironic.yaml @@ -0,0 +1,15 @@ +apiVersion: ironic.metal3.io/v1alpha1 +kind: Ironic +metadata: + name: ironic + namespace: baremetal-operator-system +spec: + apiCredentialsName: ironic-credentials + networking: + dhcp: + rangeBegin: "192.168.222.100" + rangeEnd: "192.168.222.200" + networkCIDR: "192.168.222.0/24" + interface: "eth0" + ipAddress: "192.168.222.2" + ipAddressManager: "keepalived" diff --git a/hack/quick-start-test/irso/kustomization.yaml b/hack/quick-start-test/irso/kustomization.yaml new file mode 100644 index 000000000..4a67080b9 --- /dev/null +++ b/hack/quick-start-test/irso/kustomization.yaml @@ -0,0 +1,41 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ironic-standalone-operator-system + +resources: +- https://github.com/metal3-io/ironic-standalone-operator/releases/latest/download/install.yaml + +generatorOptions: + disableNameSuffixHash: true + +secretGenerator: +- name: ironic-auth + behavior: create + files: + - username=ironic-username + - password=ironic-password + type: Opaque + +configMapGenerator: +- name: ironic-operator-config + literals: + - IPA_BASEURI=http://192.168.222.1 + +patches: +- target: + kind: Deployment + name: ironic-standalone-operator-controller-manager + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ironic-standalone-operator + spec: + template: + spec: + containers: + - name: manager + envFrom: + - configMapRef: + name: ironic-operator-config + \ No newline at end of file diff --git a/hack/quick-start-test/quick-start-test.sh b/hack/quick-start-test/quick-start-test.sh index da86eb206..3f29c6b6f 100755 --- a/hack/quick-start-test/quick-start-test.sh +++ b/hack/quick-start-test/quick-start-test.sh @@ -7,7 +7,7 @@ #------------------------------------------------------------------------------ set -eux -export QUICK_START_BASE="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")" +export QUICK_START_BASE=${QUICK_START_BASE:="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")"} setup() { echo "Disk images directory. If disk images are missing, they will be downloaded and prepared." @@ -15,37 +15,52 @@ setup() { echo "Setting up virtual lab..." "${QUICK_START_BASE}/setup-virtual-lab.sh" - # Check if the VM is running - if ! virsh -c qemu:///system dominfo bmh-vm-01 &> /dev/null; then - echo "Error: The VM bmh-vm-01 is not running." - exit 1 - fi echo "Bootstrapping Kind cluster..." "${QUICK_START_BASE}/setup-bootstrap.sh" - # Wait for all nodes to be ready - if ! kubectl wait --for=condition=Ready nodes --all --timeout=300s; then - exit 1 - fi - # Wait for cert-manager and webhook to be ready + wait_for_nodes_ready wait_for_cert_manager_ready echo "Setting up DHCP and image servers..." "${QUICK_START_BASE}/start-image-server.sh" "${QUICK_START_BASE}/setup-dhcp-server.sh" - - echo "Deploying Ironic..." - kubectl apply -k ironic - wait_for_ironic_ready + IRONIC_USERNAME="$(uuidgen)" + IRONIC_PASSWORD="$(uuidgen)" + + # These must be exported so that envsubst can pick them up below + export IRONIC_USERNAME + export IRONIC_PASSWORD + + echo "${IRONIC_USERNAME}" > "${QUICK_START_BASE}/bmo/ironic-username" + echo "${IRONIC_PASSWORD}" > "${QUICK_START_BASE}/bmo/ironic-password" + + echo "${IRONIC_USERNAME}" > "${QUICK_START_BASE}/irso/ironic-username" + echo "${IRONIC_PASSWORD}" > "${QUICK_START_BASE}/irso/ironic-password" + + # Replace in the username and password in bml-vm-01.yaml + sed -i "s/username: .*/username: ${IRONIC_USERNAME}/" ${QUICK_START_BASE}/bml-01.yaml + sed -i "s/password: .*/password: ${IRONIC_PASSWORD}/" ${QUICK_START_BASE}/bml-01.yaml + + echo "Deploying IrSO..." + # This is deploying a patch where the IPA_BASEURI are set to the local image server. + # This could be replaced with just kubectl apply -f https://github.com/metal3-io/ironic-standalone-operator/releases/latest/download/install.yaml + kubectl apply -k irso + wait_for_resource Available deployment ironic-standalone-operator-controller-manager ironic-standalone-operator-system 300 + + kubectl create namespace baremetal-operator-system echo "Deploying Baremetal Operator..." kubectl apply -k bmo - wait_for_bmo_ready + wait_for_resource Available deployment baremetal-operator-controller-manager baremetal-operator-system 300 + + # This could be replaced with just kubectl apply -f https://raw.githubusercontent.com/metal3-io/baremetal-operator/refs/heads/main/test/e2e/data/ironic-standalone-operator/ironic/base/ironic.yaml + kubectl apply -f ${QUICK_START_BASE}/irso/ironic.yaml + wait_for_resource Available deployment ironic-service baremetal-operator-system 300 } create_bmhs() { - kubectl apply -f "${QUICK_START_BASE}/bml-vm-01.yaml" kubectl apply -f "${QUICK_START_BASE}/bml-01.yaml" + kubectl apply -f "${QUICK_START_BASE}/bml-vm-01.yaml" # Wait for BMHs to be provisioned wait_for_bml_ready } @@ -56,6 +71,7 @@ setup_disk_images_dir() { "noble-server-cloudimg-amd64.img" "CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" "CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2" + "ipa-centos9-master.tar.gz" ) missing_files=0 @@ -76,38 +92,50 @@ setup_disk_images_dir() { } wait_for_cert_manager_ready() { - echo "Waiting for cert-manager to be ready... This may take up to 5 minutes." - if ! kubectl wait --for=condition=Available --timeout=60s deployment/cert-manager -n cert-manager; then - exit 1 - fi - if ! kubectl wait --for=condition=Available --timeout=60s deployment/cert-manager-webhook -n cert-manager; then + echo "Waiting for cert-manager to be ready... This may take up to 2 minutes." + if ! kubectl wait --for=condition=Available --timeout=120s deployment/cert-manager -n cert-manager; then exit 1 fi - if ! kubectl wait --for=condition=Available --timeout=60s deployment/cert-manager-cainjector -n cert-manager; then + if ! kubectl wait --for=condition=Available --timeout=120s deployment/cert-manager-webhook -n cert-manager; then exit 1 fi - if ! kubectl wait --for=condition=Ready --timeout=600s pod -l app.kubernetes.io/name=webhook -n cert-manager; then + if ! kubectl wait --for=condition=Available --timeout=120s deployment/cert-manager-cainjector -n cert-manager; then exit 1 fi } -wait_for_bmo_ready() { - echo "Waiting for Baremetal Operator to be ready... This may take up to 5 minutes." - if ! kubectl wait --for=condition=Available --timeout=300s deployment/baremetal-operator-controller-manager -n baremetal-operator-system; then +wait_for_nodes_ready() { + echo "Waiting for all nodes to be ready... This may take up to 5 minutes." + if ! kubectl wait --for=condition=Ready nodes --all --timeout=300s; then exit 1 fi } -wait_for_ironic_ready() { - echo "Waiting for Ironic to be ready... This may take up to 10 minutes." - if ! kubectl wait --for=condition=Available --timeout=600s deployment/ironic -n baremetal-operator-system; then - exit 1 - fi +wait_for_resource() { + status="$1" + resource="$2" + name="$3" + namespace="$4" + timeout="$5" + MAX_RETRIES=5 + RETRY_DELAY=2 + + echo "Waiting for ${resource} ${name} to be ready... This may take up to ${timeout} seconds." + for ((i=1; i<=MAX_RETRIES; i++)); do + if ! kubectl wait --for=condition=${status} --timeout=${timeout}s ${resource}/${name} -n ${namespace}; then + if [ $i -eq MAX_RETRIES ]; then + exit 1 + else + echo "${resource} ${name} not ready yet. Attempt $i of ${MAX_RETRIES}. Retrying..." + sleep ${RETRY_DELAY} + fi + fi + done } wait_for_bml_ready() { - echo "Waiting for BareMetalHosts to be provisioned... This may take up to 5 minutes." - if ! kubectl wait --for=condition=Available --timeout=300s baremetalhosts --all; then + echo "Waiting for BareMetalHosts to be provisioned... This may take up to 12 minutes." + if ! kubectl wait --for=condition=Available --timeout=720s baremetalhosts --all; then exit 1 fi } diff --git a/hack/quick-start-test/setup-image-server-dir.sh b/hack/quick-start-test/setup-image-server-dir.sh index 21f6ce3ce..986ee5d19 100755 --- a/hack/quick-start-test/setup-image-server-dir.sh +++ b/hack/quick-start-test/setup-image-server-dir.sh @@ -9,6 +9,7 @@ sha256sum --ignore-missing -c SHA256SUMS wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2 wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM sha256sum -c CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM -wget https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.33.0/CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2 +wget https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.34.0/CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2 sha256sum CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2 +wget https://tarballs.opendev.org/openstack/ironic-python-agent/dib/ipa-centos9-master.tar.gz popd diff --git a/hack/quick-start-test/setup-virtual-lab.sh b/hack/quick-start-test/setup-virtual-lab.sh index 6d6d064ee..dc0cbe66e 100755 --- a/hack/quick-start-test/setup-virtual-lab.sh +++ b/hack/quick-start-test/setup-virtual-lab.sh @@ -1,13 +1,53 @@ # use --ram=8192 for Scenario 2 +SERIAL_LOG_PATH="/var/log/libvirt/qemu/bmh-vm-01-serial0.log" # Define and start the baremetal-e2e network virsh -c qemu:///system net-define net.xml virsh -c qemu:///system net-start baremetal-e2e +if ! sudo virsh net-list --all | grep baremetal-e2e; then + virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml" + virsh -c qemu:///system net-start baremetal-e2e +fi + +# We need to create veth pair to connect metal3 net (defined above) and kind +# docker subnet. Let us start by creating a docker network with pre-defined +# name for bridge, so that we can configure the veth pair correctly. +# Also assume that if kind net exists, it is created by us. +if ! docker network list | grep kind; then + # These options are used by kind itself. It uses docker default mtu and + # generates ipv6 subnet ULA, but we can fix the ULA. Only addition to kind + # options is the network bridge name. + docker network create -d=bridge \ + -o com.docker.network.bridge.enable_ip_masquerade=true \ + -o com.docker.network.driver.mtu=1500 \ + -o com.docker.network.bridge.name="kind-bridge" \ + --ipv6 --subnet "fc00:f853:ccd:e793::/64" \ + kind +fi +docker network list + +# Next create the veth pair +if ! ip a | grep metalend; then + sudo ip link add metalend type veth peer name kindend + sudo ip link set metalend master metal3 + sudo ip link set kindend master kind-bridge + sudo ip link set metalend up + sudo ip link set kindend up +fi +ip a + +# Then we need to set routing rules as well +if ! sudo iptables -L FORWARD -v -n | grep kind-bridge; then + sudo iptables -I FORWARD -i kind-bridge -o metal3 -j ACCEPT + sudo iptables -I FORWARD -i metal3 -o kind-bridge -j ACCEPT +fi +sudo iptables -L FORWARD -n -v + # Start the sushy-emulator container that acts as BMC docker run --name sushy-tools --rm --network host -d \ -v /var/run/libvirt:/var/run/libvirt \ - -v "${QUICK_START_BASE}/sushy-tools.conf:/etc/sushy/sushy-emulator.conf" \ + -v "$(pwd)/sushy-tools.conf:/etc/sushy/sushy-emulator.conf" \ -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \ quay.io/metal3-io/sushy-tools:latest sushy-emulator @@ -22,5 +62,12 @@ virt-install \ --disk size=25 \ --boot hd,network \ --import \ + --serial file,path="${SERIAL_LOG_PATH}" \ + --xml "./devices/serial/@type=pty" \ + --xml "./devices/serial/log/@file=${SERIAL_LOG_PATH}" \ + --xml "./devices/serial/log/@append=on" \ --network network=baremetal-e2e,mac="00:60:2f:31:81:01" \ - --noautoconsole + --noautoconsole \ + --print-xml > ${QUICK_START_BASE}/bmh-vm-01.xml + +virsh define ${QUICK_START_BASE}/bmh-vm-01.xml diff --git a/hack/quick-start-test/sushy-tools.conf b/hack/quick-start-test/sushy-tools.conf index fade24e2f..2546fb043 100644 --- a/hack/quick-start-test/sushy-tools.conf +++ b/hack/quick-start-test/sushy-tools.conf @@ -28,7 +28,7 @@ SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = False # of secure boot or not based on the chosen nvram. SUSHY_EMULATOR_BOOT_LOADER_MAP = { u'UEFI': { - u'x86_64': u'/usr/share/OVMF/OVMF_CODE_4M.secboot.fd' + u'x86_64': u'/usr/share/OVMF/OVMF_CODE_4M.fd' }, u'Legacy': { u'x86_64': None From c91f4a4d3a0aeefdc59a953cf77019a97e1d3b35 Mon Sep 17 00:00:00 2001 From: peppi-lotta Date: Thu, 6 Nov 2025 08:20:13 +0000 Subject: [PATCH 3/3] Add github workfloW Signed-off-by: peppi-lotta --- .github/workflows/quick-start-test.yml | 41 ++++++ .gitignore | 6 +- hack/quick-start-test/bmc-secret.yaml | 8 ++ .../{bml-vm-01.yaml => bmh-01.yaml} | 2 +- hack/quick-start-test/bml-01.yaml | 8 -- hack/quick-start-test/bmo/ironic.env | 3 - hack/quick-start-test/bmo/kustomization.yaml | 10 -- hack/quick-start-test/capm3-vars.sh | 34 +++++ hack/quick-start-test/cleanup.sh | 47 +++++-- hack/quick-start-test/ironic/certificate.yaml | 44 ++++++ .../{irso => ironic}/ironic.yaml | 3 +- .../ironic/kustomization.yaml | 7 + hack/quick-start-test/ironic/namespace.yaml | 4 + hack/quick-start-test/irso/kustomization.yaml | 12 -- hack/quick-start-test/net.xml | 2 + hack/quick-start-test/quick-start-test.sh | 133 ++++++------------ hack/quick-start-test/setup-bootstrap.sh | 36 ++++- hack/quick-start-test/setup-dhcp-server.sh | 3 - .../setup-image-server-dir.sh | 24 ++-- hack/quick-start-test/setup-virtual-lab.sh | 65 ++++----- hack/quick-start-test/start-image-server.sh | 2 +- .../{sushy-tools.conf => sushy-emulator.conf} | 8 +- 22 files changed, 300 insertions(+), 202 deletions(-) create mode 100644 hack/quick-start-test/bmc-secret.yaml rename hack/quick-start-test/{bml-vm-01.yaml => bmh-01.yaml} (85%) delete mode 100644 hack/quick-start-test/bml-01.yaml delete mode 100644 hack/quick-start-test/bmo/ironic.env create mode 100755 hack/quick-start-test/capm3-vars.sh create mode 100644 hack/quick-start-test/ironic/certificate.yaml rename hack/quick-start-test/{irso => ironic}/ironic.yaml (89%) create mode 100644 hack/quick-start-test/ironic/kustomization.yaml create mode 100644 hack/quick-start-test/ironic/namespace.yaml delete mode 100755 hack/quick-start-test/setup-dhcp-server.sh rename hack/quick-start-test/{sushy-tools.conf => sushy-emulator.conf} (82%) diff --git a/.github/workflows/quick-start-test.yml b/.github/workflows/quick-start-test.yml index e69de29bb..913080e55 100644 --- a/.github/workflows/quick-start-test.yml +++ b/.github/workflows/quick-start-test.yml @@ -0,0 +1,41 @@ +name: E2E Test + +on: + workflow_call: + inputs: + runner: + type: string + default: "ubuntu-latest" + ref: + type: string + default: ${{ github.ref }} + timeout-minutes: + type: number + default: 90 + +permissions: {} + +jobs: + test: + name: Quick Start Test + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + ref: ${{ inputs.ref }} + + - name: Install libvirt + run: | + sudo apt-get update + sudo apt-get install -y libvirt-daemon-system qemu-kvm virt-manager libvirt-dev + + - name: Run Quick Start Test + # We need a new shell to pick up the new group. That is why we do the sudo -s -u $USER ... + # Remove the pre-installed go version. We install the exact version we need. + run: | + sudo usermod -a -G libvirt $USER + sudo rm /usr/bin/go + sudo -s -u $USER --preserve-env bash ${{ github.workspace }}/hack/quick-start-test/quick-start-test.sh + diff --git a/.gitignore b/.gitignore index df07277af..7a17c5096 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,10 @@ hack/quick-start-test/disk-images hack/quick-start-test/bmh-vm-01.xml hack/quick-start-test/bmo/ironic-username hack/quick-start-test/bmo/ironic-password -hack/quick-start-test/irso/ironic-username -hack/quick-start-test/irso/ironic-password +hack/quick-start-test/ironic/ironic-username +hack/quick-start-test/ironic/ironic-password +hack/quick-start-test/irso/ssh_testkey +hack/quick-start-test/irso/ssh_testkey.pub # Development containers diff --git a/hack/quick-start-test/bmc-secret.yaml b/hack/quick-start-test/bmc-secret.yaml new file mode 100644 index 000000000..870230101 --- /dev/null +++ b/hack/quick-start-test/bmc-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: bml-01 +type: Opaque +stringData: + username: replaceme + password: replaceme diff --git a/hack/quick-start-test/bml-vm-01.yaml b/hack/quick-start-test/bmh-01.yaml similarity index 85% rename from hack/quick-start-test/bml-vm-01.yaml rename to hack/quick-start-test/bmh-01.yaml index fbc40a682..bb45c77a7 100644 --- a/hack/quick-start-test/bml-vm-01.yaml +++ b/hack/quick-start-test/bmh-01.yaml @@ -5,7 +5,7 @@ metadata: spec: online: true bootMACAddress: 00:60:2f:31:81:01 - bootMode: legacy # use 'UEFI' for Scenario 1 + bootMode: UEFI hardwareProfile: libvirt bmc: address: redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmh-vm-01 diff --git a/hack/quick-start-test/bml-01.yaml b/hack/quick-start-test/bml-01.yaml deleted file mode 100644 index 3571d4247..000000000 --- a/hack/quick-start-test/bml-01.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: bml-01 -type: Opaque -stringData: - username: admin #replace with desired username - password: supersecretpassword #replace with desired password diff --git a/hack/quick-start-test/bmo/ironic.env b/hack/quick-start-test/bmo/ironic.env deleted file mode 100644 index 8fa448ce5..000000000 --- a/hack/quick-start-test/bmo/ironic.env +++ /dev/null @@ -1,3 +0,0 @@ -DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.2:6385/v1/ diff --git a/hack/quick-start-test/bmo/kustomization.yaml b/hack/quick-start-test/bmo/kustomization.yaml index c5f0a3334..7b2ead864 100644 --- a/hack/quick-start-test/bmo/kustomization.yaml +++ b/hack/quick-start-test/bmo/kustomization.yaml @@ -7,13 +7,3 @@ namespace: baremetal-operator-system # We use main here simply because the integration with IrSO is not included in a release yet. resources: - https://github.com/metal3-io/baremetal-operator/config/use-irso?ref=main - -generatorOptions: - disableNameSuffixHash: true - -secretGenerator: -- name: ironic-credentials - namespace: baremetal-operator-system - files: - - username=ironic-username - - password=ironic-password diff --git a/hack/quick-start-test/capm3-vars.sh b/hack/quick-start-test/capm3-vars.sh new file mode 100755 index 000000000..3fb202676 --- /dev/null +++ b/hack/quick-start-test/capm3-vars.sh @@ -0,0 +1,34 @@ +# Baremetal lab image variables +# export IMAGE_URL="http://192.168.0.150/CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2" +# export IMAGE_CHECKSUM="afa7e95ee6fb92b952ab85bae4d01033651e690cf04a626c668041d7b94ddd4a" +# export IMAGE_FORMAT="qcow2" +# Virtualized setup variables +export IMAGE_URL="http://192.168.222.1/CENTOS_10_NODE_IMAGE_K8S_v1.34.1.raw" +# Calculate checksum from the actual raw image file +export IMAGE_CHECKSUM=$(sha256sum ${QUICK_START_BASE}/disk-images/CENTOS_10_NODE_IMAGE_K8S_v1.34.1.raw | awk '{print $1}') +export IMAGE_FORMAT="raw" +# Common variables +export IMAGE_CHECKSUM_TYPE="sha256" +export KUBERNETES_VERSION="v1.34.1" +# Make sure this does not conflict with other networks +export POD_CIDR='["192.168.10.0/24"]' +# These can be used to add user-data +export CTLPLANE_KUBEADM_EXTRA_CONFIG=" + preKubeadmCommands: + - systemctl enable --now crio + users: + - name: user + sshAuthorizedKeys: + - ssh-ed25519 ABCD... user@example.com" +export WORKERS_KUBEADM_EXTRA_CONFIG=" + preKubeadmCommands: + - systemctl enable --now crio + users: + - name: user + sshAuthorizedKeys: + - ssh-ed25519 ABCD... user@example.com" +# NOTE! You must ensure that this is forwarded or assigned somehow to the +# server(s) that is selected for the control-plane. +# We reserved this address in the net.xml as a basic way to get a fixed IP. +export CLUSTER_APIENDPOINT_HOST="192.168.222.101" +export CLUSTER_APIENDPOINT_PORT="6443" diff --git a/hack/quick-start-test/cleanup.sh b/hack/quick-start-test/cleanup.sh index 99981530f..3b9b42b89 100755 --- a/hack/quick-start-test/cleanup.sh +++ b/hack/quick-start-test/cleanup.sh @@ -1,18 +1,39 @@ -# Delete the management cluster. -kind delete cluster +#!/usr/bin/env bash -# Stop DHCP and image servers. They are automatically removed when stopped. -docker stop dnsmasq -docker stop image-server +set -x -# Cleanup the sushy-tools container and the VM. (For virtual-lab setup) -docker stop sushy-tools +export QUICK_START_BASE=${QUICK_START_BASE:="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")"} -virsh -c qemu:///system destroy --domain bmh-vm-01 -virsh -c qemu:///system undefine --domain bmh-vm-01 --remove-all-storage --nvram +# Delete the management cluster +kind delete cluster 2>/dev/null || true -virsh -c qemu:///system net-destroy baremetal-e2e -virsh -c qemu:///system net-undefine baremetal-e2e +# Stop and remove containers +docker stop dnsmasq 2>/dev/null || true +docker rm dnsmasq 2>/dev/null || true +docker stop image-server 2>/dev/null || true +docker rm image-server 2>/dev/null || true +docker stop sushy-tools 2>/dev/null || true +docker rm sushy-tools 2>/dev/null || true -export QUICK_START_BASE=${QUICK_START_BASE:="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")"} -rm -rf "${QUICK_START_BASE}/bmh-vm-01.xml" \ No newline at end of file +# Cleanup VM from both system and session connections +virsh -c qemu:///system destroy bmh-vm-01 2>/dev/null || true +virsh -c qemu:///system undefine bmh-vm-01 --remove-all-storage --nvram 2>/dev/null || true +virsh -c qemu:///session destroy bmh-vm-01 2>/dev/null || true +virsh -c qemu:///session undefine bmh-vm-01 --remove-all-storage --nvram 2>/dev/null || true + +# Cleanup network +virsh -c qemu:///system net-destroy baremetal-e2e 2>/dev/null || true +virsh -c qemu:///system net-undefine baremetal-e2e 2>/dev/null || true + +# Remove generated files +rm -rf "${QUICK_START_BASE}/bmh-vm-01.xml" 2>/dev/null || true + +# Cleanup network interfaces and docker network +sudo ip link del metalend 2>/dev/null || true +docker network rm kind 2>/dev/null || true + +# Cleanup iptables rules +sudo iptables -D FORWARD -i kind -o metal3 -j ACCEPT 2>/dev/null || true +sudo iptables -D FORWARD -i metal3 -o kind -j ACCEPT 2>/dev/null || true + +echo "Cleanup complete" \ No newline at end of file diff --git a/hack/quick-start-test/ironic/certificate.yaml b/hack/quick-start-test/ironic/certificate.yaml new file mode 100644 index 000000000..d026b7e4d --- /dev/null +++ b/hack/quick-start-test/ironic/certificate.yaml @@ -0,0 +1,44 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: baremetal-operator-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ironic-cacert + namespace: baremetal-operator-system +spec: + commonName: ironic-ca + isCA: true + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: ironic-cacert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer + namespace: baremetal-operator-system +spec: + ca: + secretName: ironic-cacert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ironic-cert + namespace: baremetal-operator-system +spec: + ipAddresses: + - 192.168.222.2 + dnsNames: + - ironic.baremetal-operator-system.svc + issuerRef: + kind: Issuer + name: ca-issuer + secretName: ironic-cert diff --git a/hack/quick-start-test/irso/ironic.yaml b/hack/quick-start-test/ironic/ironic.yaml similarity index 89% rename from hack/quick-start-test/irso/ironic.yaml rename to hack/quick-start-test/ironic/ironic.yaml index a42d0b61d..7c8b2158a 100644 --- a/hack/quick-start-test/irso/ironic.yaml +++ b/hack/quick-start-test/ironic/ironic.yaml @@ -4,7 +4,6 @@ metadata: name: ironic namespace: baremetal-operator-system spec: - apiCredentialsName: ironic-credentials networking: dhcp: rangeBegin: "192.168.222.100" @@ -13,3 +12,5 @@ spec: interface: "eth0" ipAddress: "192.168.222.2" ipAddressManager: "keepalived" + tls: + certificateName: ironic-cert diff --git a/hack/quick-start-test/ironic/kustomization.yaml b/hack/quick-start-test/ironic/kustomization.yaml new file mode 100644 index 000000000..1aebd4024 --- /dev/null +++ b/hack/quick-start-test/ironic/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: baremetal-operator-system +resources: +- ironic.yaml +- certificate.yaml +- namespace.yaml diff --git a/hack/quick-start-test/ironic/namespace.yaml b/hack/quick-start-test/ironic/namespace.yaml new file mode 100644 index 000000000..1c0ebe64e --- /dev/null +++ b/hack/quick-start-test/ironic/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: baremetal-operator-system diff --git a/hack/quick-start-test/irso/kustomization.yaml b/hack/quick-start-test/irso/kustomization.yaml index 4a67080b9..775df52da 100644 --- a/hack/quick-start-test/irso/kustomization.yaml +++ b/hack/quick-start-test/irso/kustomization.yaml @@ -5,17 +5,6 @@ namespace: ironic-standalone-operator-system resources: - https://github.com/metal3-io/ironic-standalone-operator/releases/latest/download/install.yaml -generatorOptions: - disableNameSuffixHash: true - -secretGenerator: -- name: ironic-auth - behavior: create - files: - - username=ironic-username - - password=ironic-password - type: Opaque - configMapGenerator: - name: ironic-operator-config literals: @@ -38,4 +27,3 @@ patches: envFrom: - configMapRef: name: ironic-operator-config - \ No newline at end of file diff --git a/hack/quick-start-test/net.xml b/hack/quick-start-test/net.xml index 226d5a788..dfc520936 100644 --- a/hack/quick-start-test/net.xml +++ b/hack/quick-start-test/net.xml @@ -9,6 +9,8 @@ + + diff --git a/hack/quick-start-test/quick-start-test.sh b/hack/quick-start-test/quick-start-test.sh index 3f29c6b6f..eb60ff039 100755 --- a/hack/quick-start-test/quick-start-test.sh +++ b/hack/quick-start-test/quick-start-test.sh @@ -16,61 +16,58 @@ setup() { echo "Setting up virtual lab..." "${QUICK_START_BASE}/setup-virtual-lab.sh" - echo "Bootstrapping Kind cluster..." - "${QUICK_START_BASE}/setup-bootstrap.sh" - wait_for_nodes_ready - wait_for_cert_manager_ready - echo "Setting up DHCP and image servers..." "${QUICK_START_BASE}/start-image-server.sh" - "${QUICK_START_BASE}/setup-dhcp-server.sh" - - IRONIC_USERNAME="$(uuidgen)" - IRONIC_PASSWORD="$(uuidgen)" - - # These must be exported so that envsubst can pick them up below - export IRONIC_USERNAME - export IRONIC_PASSWORD - - echo "${IRONIC_USERNAME}" > "${QUICK_START_BASE}/bmo/ironic-username" - echo "${IRONIC_PASSWORD}" > "${QUICK_START_BASE}/bmo/ironic-password" - echo "${IRONIC_USERNAME}" > "${QUICK_START_BASE}/irso/ironic-username" - echo "${IRONIC_PASSWORD}" > "${QUICK_START_BASE}/irso/ironic-password" - - # Replace in the username and password in bml-vm-01.yaml - sed -i "s/username: .*/username: ${IRONIC_USERNAME}/" ${QUICK_START_BASE}/bml-01.yaml - sed -i "s/password: .*/password: ${IRONIC_PASSWORD}/" ${QUICK_START_BASE}/bml-01.yaml + echo "Bootstrapping Kind cluster..." + "${QUICK_START_BASE}/setup-bootstrap.sh" + if ! kubectl -n baremetal-operator-system wait --for=condition=Available --timeout=300s deployment --all; then + exit 1 + fi +} - echo "Deploying IrSO..." - # This is deploying a patch where the IPA_BASEURI are set to the local image server. - # This could be replaced with just kubectl apply -f https://github.com/metal3-io/ironic-standalone-operator/releases/latest/download/install.yaml - kubectl apply -k irso - wait_for_resource Available deployment ironic-standalone-operator-controller-manager ironic-standalone-operator-system 300 +create_bmhs() { + kubectl apply -f "${QUICK_START_BASE}/bmc-secret.yaml" + kubectl apply -f "${QUICK_START_BASE}/bmh-01.yaml" + # Wait for BMHs to be available + if ! kubectl wait --for=jsonpath='{.status.provisioning.state}'=available --timeout=600s bmh --all; then + echo "ERROR: One or more BMHs failed to reach 'available' state within timeout." + exit 1 + fi +} - kubectl create namespace baremetal-operator-system - echo "Deploying Baremetal Operator..." - kubectl apply -k bmo - wait_for_resource Available deployment baremetal-operator-controller-manager baremetal-operator-system 300 +scenario_2() { + echo "Running Scenario 2: ..." + # "clusterctl init --infrastructure metal3 --ipam=metal3" has already been run. + # Define env variables + source "${QUICK_START_BASE}/capm3-vars.sh" + + # Render and apply manifests + clusterctl generate cluster test-cluster --control-plane-machine-count 1 --worker-machine-count 0 | kubectl apply -f - + + # Wait for bml-vm-01 to be provisioned + if ! kubectl wait --for=jsonpath='{.status.provisioning.state}'=provisioned --timeout=1800s bmh bml-vm-01; then + echo "ERROR: bml-vm-01 failed to reach 'provisioned' state within timeout." + exit 1 + fi - # This could be replaced with just kubectl apply -f https://raw.githubusercontent.com/metal3-io/baremetal-operator/refs/heads/main/test/e2e/data/ironic-standalone-operator/ironic/base/ironic.yaml - kubectl apply -f ${QUICK_START_BASE}/irso/ironic.yaml - wait_for_resource Available deployment ironic-service baremetal-operator-system 300 -} + # Get kubeconfig for the workload cluster and install CNI + clusterctl get kubeconfig test-cluster > test-cluster-kubeconfig.yaml + kubectl --kubeconfig=test-cluster-kubeconfig.yaml apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.31.0/manifests/calico.yaml -create_bmhs() { - kubectl apply -f "${QUICK_START_BASE}/bml-01.yaml" - kubectl apply -f "${QUICK_START_BASE}/bml-vm-01.yaml" - # Wait for BMHs to be provisioned - wait_for_bml_ready + # Wait for the control plane machine to be ready + if ! kubectl wait --for=condition=Ready --timeout=600s machine --all; then + echo "ERROR: Machine failed to reach 'Ready' state within timeout." + exit 1 + fi } setup_disk_images_dir() { DISK_IMAGE_DIR="${QUICK_START_BASE}/disk-images" REQUIRED_FILES=( - "noble-server-cloudimg-amd64.img" - "CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" - "CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2" + "jammy-server-cloudimg-amd64.img" + "CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2" + "CENTOS_10_NODE_IMAGE_K8S_v1.34.1.raw" "ipa-centos9-master.tar.gz" ) @@ -91,54 +88,6 @@ setup_disk_images_dir() { fi } -wait_for_cert_manager_ready() { - echo "Waiting for cert-manager to be ready... This may take up to 2 minutes." - if ! kubectl wait --for=condition=Available --timeout=120s deployment/cert-manager -n cert-manager; then - exit 1 - fi - if ! kubectl wait --for=condition=Available --timeout=120s deployment/cert-manager-webhook -n cert-manager; then - exit 1 - fi - if ! kubectl wait --for=condition=Available --timeout=120s deployment/cert-manager-cainjector -n cert-manager; then - exit 1 - fi -} - -wait_for_nodes_ready() { - echo "Waiting for all nodes to be ready... This may take up to 5 minutes." - if ! kubectl wait --for=condition=Ready nodes --all --timeout=300s; then - exit 1 - fi -} - -wait_for_resource() { - status="$1" - resource="$2" - name="$3" - namespace="$4" - timeout="$5" - MAX_RETRIES=5 - RETRY_DELAY=2 - - echo "Waiting for ${resource} ${name} to be ready... This may take up to ${timeout} seconds." - for ((i=1; i<=MAX_RETRIES; i++)); do - if ! kubectl wait --for=condition=${status} --timeout=${timeout}s ${resource}/${name} -n ${namespace}; then - if [ $i -eq MAX_RETRIES ]; then - exit 1 - else - echo "${resource} ${name} not ready yet. Attempt $i of ${MAX_RETRIES}. Retrying..." - sleep ${RETRY_DELAY} - fi - fi - done -} - -wait_for_bml_ready() { - echo "Waiting for BareMetalHosts to be provisioned... This may take up to 12 minutes." - if ! kubectl wait --for=condition=Available --timeout=720s baremetalhosts --all; then - exit 1 - fi -} - setup create_bmhs +scenario_2 \ No newline at end of file diff --git a/hack/quick-start-test/setup-bootstrap.sh b/hack/quick-start-test/setup-bootstrap.sh index 4c0834621..7052b3e8b 100755 --- a/hack/quick-start-test/setup-bootstrap.sh +++ b/hack/quick-start-test/setup-bootstrap.sh @@ -1,5 +1,35 @@ -# Create a kind cluster to act as the management cluster +#!/usr/bin/env bash + kind create cluster --config kind.yaml -# Install cert-manager. It will be used to manage the certificates for Ironic -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml +# (Optional) Initialize CAPM3. This is only needed for scenario 2, but it also installs +# cert-manager, which is needed for pretty much everything else. +# If you skip this, make sure you install cert-manager separately! +clusterctl init --infrastructure=metal3 --ipam=metal3 + +kubectl apply -k irso +kubectl -n ironic-standalone-operator-system wait --for=condition=Available --timeout=300s deploy/ironic-standalone-operator-controller-manager + +# Now we can deploy Ironic and BMO +# Apply Ironic with retry logic (up to 5 attempts with 10 second delays). +# The IrSO webhook is not guaranteed to be ready when the IrSO deployment is, +# so some retries may be needed. +MAX_RETRIES=5 +RETRY_DELAY=10 +RETRY_COUNT=0 +echo "Applying Ironic configuration..." +while [[ "${RETRY_COUNT}" -lt "${MAX_RETRIES}" ]]; do + if kubectl apply -k ironic; then + echo "Successfully applied Ironic configuration" + break + else + RETRY_COUNT=$((RETRY_COUNT + 1)) + echo "Failed to apply Ironic configuration. Retrying in ${RETRY_DELAY} seconds... (Attempt ${RETRY_COUNT}/${MAX_RETRIES})" + sleep ${RETRY_DELAY} + fi +done +if [[ "${RETRY_COUNT}" -eq "${MAX_RETRIES}" ]]; then + echo "ERROR: Failed to apply Ironic configuration after ${MAX_RETRIES} attempts. Exiting." + exit 1 +fi +kubectl apply -k bmo diff --git a/hack/quick-start-test/setup-dhcp-server.sh b/hack/quick-start-test/setup-dhcp-server.sh deleted file mode 100755 index 8be39c810..000000000 --- a/hack/quick-start-test/setup-dhcp-server.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker run --name dnsmasq --rm -d --net=host --privileged --user 997:994 \ - --env-file dnsmasq.env --entrypoint /bin/rundnsmasq \ - quay.io/metal3-io/ironic diff --git a/hack/quick-start-test/setup-image-server-dir.sh b/hack/quick-start-test/setup-image-server-dir.sh index 986ee5d19..fea0cc580 100755 --- a/hack/quick-start-test/setup-image-server-dir.sh +++ b/hack/quick-start-test/setup-image-server-dir.sh @@ -1,15 +1,17 @@ -# Create a directory to hold the disk images: +#!/usr/bin/env bash + mkdir ${QUICK_START_BASE}/disk-images -# Download images to use for testing (pick those that you want): -pushd ${QUICK_START_BASE}/disk-images -wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -wget https://cloud-images.ubuntu.com/noble/current/SHA256SUMS +pushd ${QUICK_START_BASE}/disk-images || exit +wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img +wget https://cloud-images.ubuntu.com/jammy/current/SHA256SUMS sha256sum --ignore-missing -c SHA256SUMS -wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2 -wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM -sha256sum -c CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM -wget https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.34.0/CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2 -sha256sum CENTOS_9_NODE_IMAGE_K8S_v1.34.0.qcow2 +wget https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.34.1/CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2 +sha256sum CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2 +# Convert to raw. +# This helps lower memory requirements, since the raw image can be streamed to disk +# instead of first loaded to memory by IPA for conversion. +qemu-img convert -f qcow2 -O raw CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2 CENTOS_10_NODE_IMAGE_K8S_v1.34.1.raw +# Local cache of IPA wget https://tarballs.opendev.org/openstack/ironic-python-agent/dib/ipa-centos9-master.tar.gz -popd +popd || exit diff --git a/hack/quick-start-test/setup-virtual-lab.sh b/hack/quick-start-test/setup-virtual-lab.sh index dc0cbe66e..6e72b5f70 100755 --- a/hack/quick-start-test/setup-virtual-lab.sh +++ b/hack/quick-start-test/setup-virtual-lab.sh @@ -1,57 +1,46 @@ -# use --ram=8192 for Scenario 2 -SERIAL_LOG_PATH="/var/log/libvirt/qemu/bmh-vm-01-serial0.log" +#!/usr/bin/env bash # Define and start the baremetal-e2e network virsh -c qemu:///system net-define net.xml virsh -c qemu:///system net-start baremetal-e2e -if ! sudo virsh net-list --all | grep baremetal-e2e; then - virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml" - virsh -c qemu:///system net-start baremetal-e2e -fi - -# We need to create veth pair to connect metal3 net (defined above) and kind -# docker subnet. Let us start by creating a docker network with pre-defined -# name for bridge, so that we can configure the veth pair correctly. -# Also assume that if kind net exists, it is created by us. -if ! docker network list | grep kind; then - # These options are used by kind itself. It uses docker default mtu and - # generates ipv6 subnet ULA, but we can fix the ULA. Only addition to kind - # options is the network bridge name. - docker network create -d=bridge \ - -o com.docker.network.bridge.enable_ip_masquerade=true \ - -o com.docker.network.driver.mtu=1500 \ - -o com.docker.network.bridge.name="kind-bridge" \ - --ipv6 --subnet "fc00:f853:ccd:e793::/64" \ - kind -fi -docker network list +# We need to create veth pair to connect the baremetal-e2e net (defined above) +# and the docker network used by kind. This is to allow controllers in +# the kind cluster to communicate with the VMs and vice versa. +# For example, Ironic needs to communicate with IPA. +# These options are the same as what kind creates by default, +# except that we hard code the IPv6 subnet and specify a bridge name. +# +# NOTE! If you used kind before, you already have this network but +# without the fixed bridge name. Please remove it first in that case! +# docker network rm kind +docker network create -d=bridge \ + -o com.docker.network.bridge.enable_ip_masquerade=true \ + -o com.docker.network.driver.mtu=1500 \ + -o com.docker.network.bridge.name="kind" \ + --ipv6 --subnet "fc00:f853:ccd:e793::/64" \ + kind # Next create the veth pair -if ! ip a | grep metalend; then - sudo ip link add metalend type veth peer name kindend - sudo ip link set metalend master metal3 - sudo ip link set kindend master kind-bridge - sudo ip link set metalend up - sudo ip link set kindend up -fi -ip a +sudo ip link add metalend type veth peer name kindend +sudo ip link set metalend master metal3 +sudo ip link set kindend master kind +sudo ip link set metalend up +sudo ip link set kindend up # Then we need to set routing rules as well -if ! sudo iptables -L FORWARD -v -n | grep kind-bridge; then - sudo iptables -I FORWARD -i kind-bridge -o metal3 -j ACCEPT - sudo iptables -I FORWARD -i metal3 -o kind-bridge -j ACCEPT -fi -sudo iptables -L FORWARD -n -v +sudo iptables -I FORWARD -i kind -o metal3 -j ACCEPT +sudo iptables -I FORWARD -i metal3 -o kind -j ACCEPT # Start the sushy-emulator container that acts as BMC docker run --name sushy-tools --rm --network host -d \ -v /var/run/libvirt:/var/run/libvirt \ - -v "$(pwd)/sushy-tools.conf:/etc/sushy/sushy-emulator.conf" \ + -v "$(pwd)/sushy-emulator.conf:/etc/sushy/sushy-emulator.conf" \ -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \ quay.io/metal3-io/sushy-tools:latest sushy-emulator # Generate a VM definition xml file and then define the VM +# use --ram=8192 for Scenario 2 virt-install \ --connect qemu:///system \ --name bmh-vm-01 \ @@ -60,7 +49,7 @@ virt-install \ --ram=4096 \ --vcpus=2 \ --disk size=25 \ - --boot hd,network \ + --boot uefi,hd,network \ --import \ --serial file,path="${SERIAL_LOG_PATH}" \ --xml "./devices/serial/@type=pty" \ diff --git a/hack/quick-start-test/start-image-server.sh b/hack/quick-start-test/start-image-server.sh index 9a090144d..f6e1bdf12 100755 --- a/hack/quick-start-test/start-image-server.sh +++ b/hack/quick-start-test/start-image-server.sh @@ -1,3 +1,3 @@ -#Run a basic http server to expose the disk images: +#!/usr/bin/env bash docker run --name image-server --rm -d -p 80:8080 \ -v "${QUICK_START_BASE}/disk-images:/usr/share/nginx/html" nginxinc/nginx-unprivileged diff --git a/hack/quick-start-test/sushy-tools.conf b/hack/quick-start-test/sushy-emulator.conf similarity index 82% rename from hack/quick-start-test/sushy-tools.conf rename to hack/quick-start-test/sushy-emulator.conf index 2546fb043..58da1a357 100644 --- a/hack/quick-start-test/sushy-tools.conf +++ b/hack/quick-start-test/sushy-emulator.conf @@ -19,16 +19,16 @@ SUSHY_EMULATOR_LIBVIRT_URI = u'qemu:///system' # set the boot device. Allowing the UEFI firmware to instead # rely on the EFI Boot Manager # Note: This sets the legacy boot element to dev="fd" -# and relies on the floppy not existing, it likely wont work -# your VM has a floppy drive. +# and relies on the floppy not existing, it likely won't work +# if your VM has a floppy drive. SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = False -# The map of firmware loaders dependant on the boot mode and +# The map of firmware loaders dependent on the boot mode and # system architecture. Ideally the x86_64 loader will be capable # of secure boot or not based on the chosen nvram. SUSHY_EMULATOR_BOOT_LOADER_MAP = { u'UEFI': { - u'x86_64': u'/usr/share/OVMF/OVMF_CODE_4M.fd' + u'x86_64': u'/usr/share/OVMF/OVMF_CODE.secboot.fd' }, u'Legacy': { u'x86_64': None