diff --git a/docs/user-guide/examples/bmc-secret.yaml b/docs/user-guide/examples/bmc-secret.yaml new file mode 100644 index 00000000..87023010 --- /dev/null +++ b/docs/user-guide/examples/bmc-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: bml-01 +type: Opaque +stringData: + username: replaceme + password: replaceme diff --git a/docs/user-guide/examples/bmh-01-provision.yaml b/docs/user-guide/examples/bmh-01-provision.yaml new file mode 100644 index 00000000..2ebd1245 --- /dev/null +++ b/docs/user-guide/examples/bmh-01-provision.yaml @@ -0,0 +1,17 @@ +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: bml-vm-01 +spec: + online: true + bootMACAddress: 00:60:2f:31:81:01 + bootMode: UEFI + hardwareProfile: libvirt + bmc: + address: redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmh-vm-01 + credentialsName: bml-01 + image: + checksumType: sha256 + checksum: http://192.168.222.1/SHA256SUMS + format: qcow2 + url: http://192.168.222.1/jammy-server-cloudimg-amd64.img diff --git a/docs/user-guide/examples/bmh-01.yaml b/docs/user-guide/examples/bmh-01.yaml new file mode 100644 index 00000000..bb45c77a --- /dev/null +++ b/docs/user-guide/examples/bmh-01.yaml @@ -0,0 +1,12 @@ +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: bml-vm-01 +spec: + online: true + bootMACAddress: 00:60:2f:31:81:01 + bootMode: UEFI + hardwareProfile: libvirt + bmc: + address: redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmh-vm-01 + credentialsName: bml-01 diff --git a/docs/user-guide/examples/bmo/kustomization.yaml b/docs/user-guide/examples/bmo/kustomization.yaml new file mode 100644 index 00000000..7b2ead86 --- /dev/null +++ b/docs/user-guide/examples/bmo/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: baremetal-operator-system +# This is the kustomization that we build on. You can download it and change +# the URL to a relative path if you do not want to access it over the network. +# Note that the ref=main specifies the version to use. +# We use main here simply because the integration with IrSO is not included in a release yet. +resources: +- https://github.com/metal3-io/baremetal-operator/config/use-irso?ref=main diff --git a/docs/user-guide/examples/capm3-vars.sh b/docs/user-guide/examples/capm3-vars.sh new file mode 100644 index 00000000..0f4cbe86 --- /dev/null +++ b/docs/user-guide/examples/capm3-vars.sh @@ -0,0 +1,33 @@ +# Baremetal lab image variables +# export IMAGE_URL="http://192.168.0.150/CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2" +# export IMAGE_CHECKSUM="afa7e95ee6fb92b952ab85bae4d01033651e690cf04a626c668041d7b94ddd4a" +# export IMAGE_FORMAT="qcow2" +# Virtualized setup variables +export IMAGE_URL="http://192.168.222.1/CENTOS_10_NODE_IMAGE_K8S_v1.34.1.raw" +export IMAGE_CHECKSUM="20537529c0588e1c3d1929981207ef6fac73df7b2500b84f462d09badcc670ea" +export IMAGE_FORMAT="raw" +# Common variables +export IMAGE_CHECKSUM_TYPE="sha256" +export KUBERNETES_VERSION="v1.34.1" +# Make sure this does not conflict with other networks +export POD_CIDR='["192.168.10.0/24"]' +# These can be used to add user-data +export CTLPLANE_KUBEADM_EXTRA_CONFIG=" + preKubeadmCommands: + - systemctl enable --now crio + users: + - name: user + sshAuthorizedKeys: + - ssh-ed25519 ABCD... user@example.com" +export WORKERS_KUBEADM_EXTRA_CONFIG=" + preKubeadmCommands: + - systemctl enable --now crio + users: + - name: user + sshAuthorizedKeys: + - ssh-ed25519 ABCD... user@example.com" +# NOTE! You must ensure that this is forwarded or assigned somehow to the +# server(s) that is selected for the control-plane. +# We reserved this address in the net.xml as a basic way to get a fixed IP. +export CLUSTER_APIENDPOINT_HOST="192.168.222.101" +export CLUSTER_APIENDPOINT_PORT="6443" diff --git a/docs/user-guide/examples/cleanup-virtual-lab.sh b/docs/user-guide/examples/cleanup-virtual-lab.sh new file mode 100755 index 00000000..f5d4bdea --- /dev/null +++ b/docs/user-guide/examples/cleanup-virtual-lab.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +docker rm -f sushy-tools + +virsh -c qemu:///system destroy --domain "bmh-vm-01" +virsh -c qemu:///system undefine --domain "bmh-vm-01" --remove-all-storage --nvram + +# Clear network +virsh -c qemu:///system net-destroy baremetal-e2e +virsh -c qemu:///system net-undefine baremetal-e2e + +sudo iptables -D FORWARD -i kind -o metal3 -j ACCEPT +sudo iptables -D FORWARD -i metal3 -o kind -j ACCEPT + +sudo ip link delete metalend type veth diff --git a/docs/user-guide/examples/image-server.sh b/docs/user-guide/examples/image-server.sh new file mode 100755 index 00000000..098618c6 --- /dev/null +++ b/docs/user-guide/examples/image-server.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +mkdir disk-images + +pushd disk-images || exit +wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img +wget https://cloud-images.ubuntu.com/jammy/current/SHA256SUMS +sha256sum --ignore-missing -c SHA256SUMS +wget https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.34.1/CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2 +sha256sum CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2 +# Convert to raw. +# This helps lower memory requirements, since the raw image can be streamed to disk +# instead of first loaded to memory by IPA for conversion. +qemu-img convert -f qcow2 -O raw CENTOS_10_NODE_IMAGE_K8S_v1.34.1.qcow2 CENTOS_10_NODE_IMAGE_K8S_v1.34.1.raw +# Local cache of IPA +wget https://tarballs.opendev.org/openstack/ironic-python-agent/dib/ipa-centos9-master.tar.gz +popd || exit + +docker run --name image-server --rm -d -p 80:8080 \ + -v "$(pwd)/disk-images:/usr/share/nginx/html" nginxinc/nginx-unprivileged diff --git a/docs/user-guide/examples/ironic/certificate.yaml b/docs/user-guide/examples/ironic/certificate.yaml new file mode 100644 index 00000000..d026b7e4 --- /dev/null +++ b/docs/user-guide/examples/ironic/certificate.yaml @@ -0,0 +1,44 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: baremetal-operator-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ironic-cacert + namespace: baremetal-operator-system +spec: + commonName: ironic-ca + isCA: true + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: ironic-cacert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer + namespace: baremetal-operator-system +spec: + ca: + secretName: ironic-cacert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ironic-cert + namespace: baremetal-operator-system +spec: + ipAddresses: + - 192.168.222.2 + dnsNames: + - ironic.baremetal-operator-system.svc + issuerRef: + kind: Issuer + name: ca-issuer + secretName: ironic-cert diff --git a/docs/user-guide/examples/ironic/ironic.yaml b/docs/user-guide/examples/ironic/ironic.yaml new file mode 100644 index 00000000..7c8b2158 --- /dev/null +++ b/docs/user-guide/examples/ironic/ironic.yaml @@ -0,0 +1,16 @@ +apiVersion: ironic.metal3.io/v1alpha1 +kind: Ironic +metadata: + name: ironic + namespace: baremetal-operator-system +spec: + networking: + dhcp: + rangeBegin: "192.168.222.100" + rangeEnd: "192.168.222.200" + networkCIDR: "192.168.222.0/24" + interface: "eth0" + ipAddress: "192.168.222.2" + ipAddressManager: "keepalived" + tls: + certificateName: ironic-cert diff --git a/docs/user-guide/examples/ironic/kustomization.yaml b/docs/user-guide/examples/ironic/kustomization.yaml new file mode 100644 index 00000000..abeea828 --- /dev/null +++ b/docs/user-guide/examples/ironic/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: baremetal-operator-system +resources: +- ironic.yaml +- certificate.yaml diff --git a/docs/user-guide/examples/kind.yaml b/docs/user-guide/examples/kind.yaml new file mode 100644 index 00000000..ddda1a20 --- /dev/null +++ b/docs/user-guide/examples/kind.yaml @@ -0,0 +1,16 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +nodes: +- role: control-plane + # Open ports for Ironic + extraPortMappings: + # Ironic httpd + - containerPort: 6180 + hostPort: 6180 + listenAddress: "0.0.0.0" + protocol: TCP + # Ironic API + - containerPort: 6385 + hostPort: 6385 + listenAddress: "0.0.0.0" + protocol: TCP diff --git a/docs/user-guide/examples/net.xml b/docs/user-guide/examples/net.xml new file mode 100644 index 00000000..dfc52093 --- /dev/null +++ b/docs/user-guide/examples/net.xml @@ -0,0 +1,17 @@ + + baremetal-e2e + + + + + + + + + + + + + + + diff --git a/docs/user-guide/examples/setup-bootstrap.sh b/docs/user-guide/examples/setup-bootstrap.sh new file mode 100755 index 00000000..44d44946 --- /dev/null +++ b/docs/user-guide/examples/setup-bootstrap.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +kind create cluster --config kind.yaml + +# (Optional) Initialize CAPM3. This is only needed for scenario 2, but it also installs +# cert-manager, which is needed for pretty much everything else. +# If you skip this, make sure you install cert-manager separately! +clusterctl init --infrastructure=metal3 --ipam=metal3 + +kubectl apply -f https://github.com/metal3-io/ironic-standalone-operator/releases/latest/download/install.yaml +kubectl -n ironic-standalone-operator-system wait --for=condition=Available deploy/ironic-standalone-operator-controller-manager + +# Now we can deploy Ironic and BMO +kubectl create ns baremetal-operator-system +# Apply Ironic with retry logic (up to 5 attempts with 10 second delays). +# The IrSO webhook is not guaranteed to be ready when the IrSO deployment is, +# so some retries may be needed. +MAX_RETRIES=5 +RETRY_DELAY=10 +RETRY_COUNT=0 +echo "Applying Ironic configuration..." +while [ ${RETRY_COUNT} -lt ${MAX_RETRIES} ]; do + if kubectl apply -k ironic; then + echo "Successfully applied Ironic configuration" + break + else + RETRY_COUNT=$((RETRY_COUNT + 1)) + echo "Failed to apply Ironic configuration. Retrying in ${RETRY_DELAY} seconds... (Attempt ${RETRY_COUNT}/${MAX_RETRIES})" + sleep ${RETRY_DELAY} + fi +done +if [ ${RETRY_COUNT} -eq ${MAX_RETRIES} ]; then + echo "ERROR: Failed to apply Ironic configuration after ${MAX_RETRIES} attempts. Exiting." + exit 1 +fi +kubectl apply -k bmo diff --git a/docs/user-guide/examples/setup-virtual-lab.sh b/docs/user-guide/examples/setup-virtual-lab.sh new file mode 100755 index 00000000..25af16db --- /dev/null +++ b/docs/user-guide/examples/setup-virtual-lab.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Define and start the baremetal-e2e network +virsh -c qemu:///system net-define net.xml +virsh -c qemu:///system net-start baremetal-e2e + +# We need to create veth pair to connect the baremetal-e2e net (defined above) +# and the docker network used by kind. This is to allow controllers in +# the kind cluster to communicate with the VMs and vice versa. +# For example, Ironic needs to communicate with IPA. +# These options are the same as what kind creates by default, +# except that we hard code the IPv6 subnet and specify a bridge name. +# +# NOTE! If you used kind before, you already have this network but +# without the fixed bridge name. Please remove it first in that case! +# docker network rm kind +docker network create -d=bridge \ + -o com.docker.network.bridge.enable_ip_masquerade=true \ + -o com.docker.network.driver.mtu=1500 \ + -o com.docker.network.bridge.name="kind" \ + --ipv6 --subnet "fc00:f853:ccd:e793::/64" \ + kind + +# Next create the veth pair +sudo ip link add metalend type veth peer name kindend +sudo ip link set metalend master metal3 +sudo ip link set kindend master kind +sudo ip link set metalend up +sudo ip link set kindend up + +# Then we need to set routing rules as well +sudo iptables -I FORWARD -i kind -o metal3 -j ACCEPT +sudo iptables -I FORWARD -i metal3 -o kind -j ACCEPT + +# Start the sushy-emulator container that acts as BMC +docker run --name sushy-tools --rm --network host -d \ + -v /var/run/libvirt:/var/run/libvirt \ + -v "$(pwd)/sushy-emulator.conf:/etc/sushy/sushy-emulator.conf" \ + -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \ + quay.io/metal3-io/sushy-tools:latest sushy-emulator + +# Generate a VM definition xml file and then define the VM +# use --ram=8192 for Scenario 2 +virt-install \ + --connect qemu:///system \ + --name bmh-vm-01 \ + --description "Virtualized BareMetalHost" \ + --osinfo=ubuntu-lts-latest \ + --ram=4096 \ + --vcpus=2 \ + --disk size=25 \ + --boot uefi,hd,network \ + --import \ + --network network=baremetal-e2e,mac="00:60:2f:31:81:01" \ + --noautoconsole \ + --print-xml > bmh-vm-01.xml +virsh define bmh-vm-01.xml +rm bmh-vm-01.xml diff --git a/docs/user-guide/examples/sushy-emulator.conf b/docs/user-guide/examples/sushy-emulator.conf new file mode 100644 index 00000000..58da1a35 --- /dev/null +++ b/docs/user-guide/examples/sushy-emulator.conf @@ -0,0 +1,36 @@ +# Listen on the local IP address 192.168.222.1 +SUSHY_EMULATOR_LISTEN_IP = u'192.168.222.1' + +# Bind to TCP port 8000 +SUSHY_EMULATOR_LISTEN_PORT = 8000 + +# Serve this SSL certificate to the clients +SUSHY_EMULATOR_SSL_CERT = None + +# If SSL certificate is being served, this is its RSA private key +SUSHY_EMULATOR_SSL_KEY = None + +# The OpenStack cloud ID to use. This option enables OpenStack driver. +SUSHY_EMULATOR_OS_CLOUD = None +# The libvirt URI to use. This option enables libvirt driver. +SUSHY_EMULATOR_LIBVIRT_URI = u'qemu:///system' + +# Instruct the libvirt driver to ignore any instructions to +# set the boot device. Allowing the UEFI firmware to instead +# rely on the EFI Boot Manager +# Note: This sets the legacy boot element to dev="fd" +# and relies on the floppy not existing, it likely won't work +# if your VM has a floppy drive. +SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = False + +# The map of firmware loaders dependent on the boot mode and +# system architecture. Ideally the x86_64 loader will be capable +# of secure boot or not based on the chosen nvram. +SUSHY_EMULATOR_BOOT_LOADER_MAP = { + u'UEFI': { + u'x86_64': u'/usr/share/OVMF/OVMF_CODE.secboot.fd' + }, + u'Legacy': { + u'x86_64': None + } +} diff --git a/docs/user-guide/src/quick-start.md b/docs/user-guide/src/quick-start.md index 5d6e5cf9..fa9d9019 100644 --- a/docs/user-guide/src/quick-start.md +++ b/docs/user-guide/src/quick-start.md @@ -2,7 +2,7 @@ -This guide has been tested on Ubuntu server 22.04. It should be seen as an +This guide has been tested on Ubuntu server 24.04. It should be seen as an example rather than the absolute truth about how to deploy and use Metal3. We will cover two environments and two scenarios. The environments are @@ -18,11 +18,9 @@ scenarios. In a nut-shell, this is what we will do: -1. [Setup a management cluster](#management-cluster) -1. [Setup a DHCP server](#dhcp-server) +1. [(Optional) Setup virtualized lab environment](#virtualized-configuration) 1. [Setup a disk image server](#image-server) -1. [Deploy Ironic](#deploy-ironic) -1. [Deploy Bare Metal Operator](#deploy-bare-metal-operator) +1. [Setup a management cluster](#management-cluster) 1. [Create BareMetalHosts to represent the servers](#create-baremetalhosts) 1. [(Scenario 1) Provision the BareMetalHosts](#scenario-1-provision-baremetalhosts) 1. [(Scenario 2) Deploy Cluster API and turn the BareMetalHosts into a Kubernetes cluster](#scenario-2-metal3-and-cluster-api) @@ -46,7 +44,7 @@ as a management computer where we will set up Metal3. The servers are equipped with iLO 4 BMCs. These BMCs are connected to an "out of band" network (`192.168.1.0/24`) and they have the following IP addresses. -- bml-01: 192.168.1.13 +- bml-01: 192.168.1.28 - bml-02: 192.168.1.14 There is a separate network for the servers (`192.168.0.0/24`). The management @@ -56,7 +54,7 @@ and `192.168.0.150` respectively. Finally, we will need the MAC addresses of the servers to keep track of which is which. -- bml-01: 80:c1:6e:7a:e8:10 +- bml-01: 9C:63:C0:AC:10:42 - bml-02: 80:c1:6e:7a:5a:a8 ## Virtualized configuration @@ -79,70 +77,29 @@ and the MAC address: Start by defining a libvirt network: ```xml - - baremetal - - - - - - - - - -``` - -Save this as `net.xml`, define it and start it. - -```bash -virsh -c qemu:///system net-define net.xml -virsh -c qemu:///system net-start baremetal +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/net.xml"}} ``` -Next, we will create a virtual machine. Feel free to adjust at as you see fit, -but make sure to note the MAC address. That will be needed later. You can also -create more than one if you like. - -```bash -# use --ram=8192 for Scenario 2 -virt-install \ - --connect qemu:///system \ - --name bmh-vm-01 \ - --description "Virtualized BareMetalHost" \ - --osinfo=ubuntu-lts-latest \ - --ram=4096 \ - --vcpus=2 \ - --disk size=25 \ - --graphics=none \ - --console pty \ - --serial pty \ - --pxe \ - --network network=baremetal,mac="00:60:2f:31:81:01" \ - --noautoconsole -``` - -### Sushy-tools - AKA the BMC +Save this as `net.xml`. Metal3 relies on baseboard management controllers to manage the baremetal servers, so we need something similar for our virtual machines. This comes in the form of [sushy-tools](https://docs.openstack.org/sushy/latest/). -We need to create configuration file first: +We need to create a configuration file for sushy-tools: ```conf -# Listen on 192.168.222.1:8000 -SUSHY_EMULATOR_LISTEN_IP = u'192.168.222.1' -SUSHY_EMULATOR_LISTEN_PORT = 8000 -# The libvirt URI to use. This option enables libvirt driver. -SUSHY_EMULATOR_LIBVIRT_URI = u'qemu:///system' +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/sushy-emulator.conf"}} ``` +Finally, we start up the virtual baremetal lab and create VMs to simulate the +servers. Feel free to adjust things as you see fit, but make sure to note the +MAC address. That will be needed later. You can choose how many VMs to create. +At least one is needed, although more could be nice for scenario 2, to have more +than one node in the cluster. + ```bash -docker run --name sushy-tools --rm --network host -d \ - -v /var/run/libvirt:/var/run/libvirt \ - -v "$(pwd)/sushy-tools.conf:/etc/sushy/sushy-emulator.conf" \ - -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \ - quay.io/metal3-io/sushy-tools:latest sushy-emulator +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/setup-virtual-lab.sh"}} ``` ## Common setup @@ -152,462 +109,73 @@ environment. Specific configuration will always differ between environments though. We will go through how to configure and deploy Ironic and Baremetal Operator. -### Management cluster - -If you already have a Kubernetes cluster that you want to use, go ahead and use -that. Please ensure that it is connected to the relevant networks so that Ironic -can reach the BMCs and so that the BareMetalHosts can reach Ironic. - -If you do not have an cluster already, you can create one using kind. Please -note that this is absolutely not intended for production environments. - -We will use the following configuration file for kind, save it as `kind.yaml`: - -```yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - # Open ports for Ironic - extraPortMappings: - # Ironic httpd - - containerPort: 6180 - hostPort: 6180 - listenAddress: "0.0.0.0" - protocol: TCP - # Ironic API - - containerPort: 6385 - hostPort: 6385 - listenAddress: "0.0.0.0" - protocol: TCP -``` - -As you can see, it has a few ports forwarded from the host. This is to make -Ironic reachable when it is running inside the kind cluster. - -Now go ahead and create the cluster: - -```bash -kind create cluster --config kind.yaml -``` +### Image server -We will need to install cert-manager also. It will be used to manage the -certificates for Ironic later. +In order to do anything useful, we will need a server for hosting disk images +that can be used to provision the servers. In this guide, we will use an nginx +container for this. We also download some images that will be used later. ```bash -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.3/cert-manager.yaml +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/image-server.sh"}} ``` ### DHCP server The BareMetalHosts must be able to call back to Ironic when going through the inspection phase. This means that they must have IP addresses in a network where -they can reach Ironic. We will set up a DHCP server for this purpose. - -Any DHCP server can be used for this. We will here use the Ironic container -image that incudes dnsmasq and some scripts for configuring it. - -Create a configuration file and save it as `dnsmasq.env`. - -Baremetal lab: - -```bash -# The same HTTP port must be provided to all containers! -HTTP_PORT=6180 -# Specify the MAC addresses (separated by ;) of the hosts we know about and want to use -DHCP_HOSTS=80:c1:6e:7a:e8:10;80:c1:6e:7a:5a:a8 -# Ignore unknown hosts so we don't accidentally give out IP addresses to other hosts in the network -DHCP_IGNORE=tag:!known -# Listen on this IP (management computer) -PROVISIONING_IP=192.168.0.150 -# Give out IP addresses in this range -DHCP_RANGE=192.168.0.100,192.168.0.149 -GATEWAY_IP=192.168.0.1 -``` - -Virtualized environment: - -```bash -HTTP_PORT=6180 -DHCP_HOSTS=00:60:2f:31:81:01 -DHCP_IGNORE=tag:!known -# IP of the host from VM perspective -PROVISIONING_IP=192.168.222.1 -GATEWAY_IP=192.168.222.1 -DHCP_RANGE=192.168.222.100,192.168.222.149 -``` - -You can now run the DHCP server like this: +they can reach Ironic. Any DHCP server can be used for this. We use the Ironic +container image that includes dnsmasq. It is deployed automatically together +with Ironic. -```bash -docker run --name dnsmasq --rm -d --net=host --privileged --user 997:994 \ - --env-file dnsmasq.env --entrypoint /bin/rundnsmasq \ - quay.io/metal3-io/ironic -``` - -### Image server - -In order to do anything useful, we will need a server for hosting disk images -that can be used to provision the servers. - -Create a directory to hold the disk images: - -```bash -mkdir disk-images -``` - -Download images to use for testing (pick those that you want): - -```bash -pushd disk-images -wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img -wget https://cloud-images.ubuntu.com/jammy/current/SHA256SUMS -sha256sum --ignore-missing -c SHA256SUMS -wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2 -wget https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM -sha256sum -c CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM -wget https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.33.0/CENTOS_9_NODE_IMAGE_K8S_v1.33.0.qcow2 -sha256sum CENTOS_9_NODE_IMAGE_K8S_v1.33.0.qcow2 -popd -``` - -Run a basic http server to expose the disk images: - -```bash -docker run --name image-server --rm -d -p 80:8080 \ - -v "$(pwd)/disk-images:/usr/share/nginx/html" nginxinc/nginx-unprivileged -``` - -### Deploy Ironic - -In this section we will create a -[kustomization](https://kubectl.docs.kubernetes.io/references/kustomize/glossary/#kustomization) -containing configuration and credentials for deploying Ironic. - -Create a folder to hold the kustomization: - -```bash -mkdir ironic -``` +### Management cluster -#### Authentication configuration +If you already have a Kubernetes cluster that you want to use, go ahead and use +that. Please ensure that it is connected to the relevant networks so that Ironic +can reach the BMCs and so that the BareMetalHosts can reach Ironic. -Create authentication configuration for Ironic. You will need to -generate a username and password for it. We will here refer to them as -`IRONIC_USERNAME` and `IRONIC_PASSWORD`. +If you do not have a cluster already, you can create one using kind. Please note +that this is absolutely not intended for production environments. -To enable basic auth, we need to create a secret containing the key -`IRONIC_HTPASSWD` with values generated from the credentials using htpasswd. We -will do this by creating a file `ironic-htpasswd` with the following content. +We will use the following configuration file for kind, save it as `kind.yaml`: -```bash -IRONIC_HTPASSWD="" +```yaml +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/kind.yaml"}} ``` -#### Ironic environment variables - -In this section we will create a file containing environment variables used to -configure Ironic and related components. We will call the file `ironic_bmo.env`. -It looks like this for the baremetal lab: - -```bash -# Same port as exposed in kind.yaml -HTTP_PORT=6180 -# This is the interface inside the container -PROVISIONING_INTERFACE=eth0 -# URL where the http server is exposed (IP of management computer) -CACHEURL=http://192.168.0.150 -IRONIC_IP=192.168.0.150 -IRONIC_KERNEL_PARAMS=console=ttyS0 -# IP where the BMCs can access Ironic to get the virtualmedia boot image. -# This is the IP of the management computer in the out of band network. -IRONIC_EXTERNAL_IP=192.168.1.7 -# URLs where the servers can callback during inspection. -# IP of management computer in the other network and same ports as in kind.yaml -IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.0.150:6385 -IRONIC_INSPECTOR_CALLBACK_ENDPOINT_OVERRIDE=https://192.168.0.150:5050 -``` +As you can see, it has a few ports forwarded from the host. This is to make +Ironic reachable when it is running inside the kind cluster. -For the virtualized environment it looks like this: +We will also need to install cert-manager and Ironic Standalone Operator. +Finally, we deploy Ironic and Bare Metal Operator. ```bash -HTTP_PORT=6180 -PROVISIONING_INTERFACE=eth0 -CACHEURL=http://192.168.222.1/images -IRONIC_KERNEL_PARAMS=console=ttyS0 -# Docker does not allow cross-network access. If using kind to create the management -# cluster, explicitly set the external ip and use port forwarding to access ironic services. -IRONIC_EXTERNAL_IP=192.168.222.1 +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/setup-bootstrap.sh"}} ``` -For more details on available variables, see the -[ironic-image repository](https://github.com/metal3-io/ironic-image/tree/main). - -#### Patch Ironic Deployment - -The Ironic kustomization that we build on includes a dnsmasq container used for -DHCP and PXE booting. However, we already set this up separately, because it is -tricky to expose a DHCP server running inside kind. This means that we do not -need the dnsmasq container that comes with the kustomization by default. - -We will create a patch for removing it. It looks like this: +We use the following manifest to deploy Ironic. Feel free to adjust as needed +for your environment. ```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ironic -spec: - template: - spec: - containers: - - name: ironic-dnsmasq - $patch: delete +# kustomization.yaml +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/ironic/kustomization.yaml"}} ``` -Save it as `ironic-patch.yaml`. - -#### Ironic kustomization - -Time to tie it all together by creating a `kustomization.yaml`. At this point -you should have a file structure like this: - -```text -ironic/ -├── ironic-htpasswd -├── ironic-patch.yaml -├── ironic_bmo.env -└── kustomization.yaml -``` - -Here is a commented `kustomization.yaml`. Check carefully the IP addresses as -these will always differ depending on environment. - ```yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: baremetal-operator-system -# These are the kustomizations we build on. You can download them and change the URLs to relative -# paths if you do not want to access them over the network. -# Note that the ref=v0.11.0 specifies the version to use. -resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=v0.11.0 -# The kustomize components configure basic-auth and TLS -components: -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=v0.11.0 -images: -- name: quay.io/metal3-io/ironic - newTag: v32.0.0 -# Create a ConfigMap from ironic_bmo.env and call it ironic-bmo-configmap. -# This ConfigMap will be used to set environment variables for the containers. -configMapGenerator: -- envs: - - ironic_bmo.env - name: ironic-bmo-configmap - behavior: create - -patches: -# Patch for removing dnsmasq -- path: ironic-patch.yaml -# The TLS component adds certificates but it cannot know the exact IPs of our environment. -# Here we patch the certificates to have the correct IPs. -# - 192.168.1.7: management computer IP in out of band network -# - 172.18.0.2: kind cluster node IP. This is what Ironic will see attached to the interface. -# - 192.168.0.150: management computer IP in the other network -- patch: |- - - op: replace - path: /spec/ipAddresses/0 - value: 192.168.1.7 - - op: add - path: /spec/ipAddresses/- - value: 172.18.0.2 - - op: add - path: /spec/ipAddresses/- - value: 192.168.0.150 - # The same patch in the virtualized environment looks like this: - # - op: replace - # path: /spec/ipAddresses/0 - # value: 192.168.222.1 - # - op: add - # path: /spec/ipAddresses/- - # value: 172.18.0.2 - target: - kind: Certificate - name: ironic-cert -# The CA certificate should not have any IP address so we remove it. -- patch: |- - - op: remove - path: /spec/ipAddresses - target: - kind: Certificate - name: ironic-cacert -# Create secrets from the authentication configuration. -# These will be mounted or used for environment variables. -# See the basic-auth component for more details on how they are used. -secretGenerator: -- name: ironic-htpasswd - behavior: create - envs: - - ironic-htpasswd -``` - -You can check that it works and inspect the resulting manifest by running this: - -```bash -kubectl create -k ironic --dry-run=client -o yaml -``` - -When you are happy with the output, apply it in the cluster: - -```bash -kubectl apply -k ironic -``` - -### Deploy Bare Metal Operator - -Similar to Ironic, we will create a kustomization for deploying Baremetal -Operator. It will include credentials for accessing Ironic. Start with creating -a folder for the kustomization: - -```bash -mkdir bmo -``` - -Create files containing the credentials for Ironic: - -- ironic-username -- ironic-password - -We will use kustomize to create secrets from these that Bare Metal Operator can -use to access Ironic. - -Next, create a file for environment variables. We will call it `ironic.env`. The -content looks like this for the baremetal lab: - -```bash -DEPLOY_KERNEL_URL=http://192.168.0.150:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.0.150:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.0.150:6385/v1/ +# ironic.yaml +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/ironic/ironic.yaml"}} ``` -The IP address is that of the management computer. The same in the virtualized -environment looks like this: - -```bash -DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/ -``` - -Finally, create the `kustomization.yaml` with this content: - ```yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: baremetal-operator-system -# This is the kustomization that we build on. You can download it and change -# the URL to a relative path if you do not want to access it over the network. -# Note that the ref=v0.11.0 specifies the version to use. -resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/config/base?ref=v0.11.0 -components: -- https://github.com/metal3-io/baremetal-operator/config/components/basic-auth?ref=v0.11.0 -- https://github.com/metal3-io/baremetal-operator/config/components/tls?ref=v0.11.0 -images: -- name: quay.io/metal3-io/baremetal-operator - newTag: v0.11.0 -# Create a ConfigMap from ironic.env and name it ironic. -configMapGenerator: -- name: ironic - behavior: create - envs: - - ironic.env - -# We cannot use suffix hashes since the kustomizations we build on -# cannot be aware of what suffixes we add. -generatorOptions: - disableNameSuffixHash: true -# Create secrets with the credentials for accessing Ironic. -secretGenerator: -- name: ironic-credentials - files: - - username=ironic-username - - password=ironic-password +# certificate.yaml +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/ironic/certificate.yaml"}} ``` -At this point, you should have a folder structure like this: +For the Bare Metal Operator, we use a kustomization that looks like this: -```text -bmo/ -├── ironic-password -├── ironic-username -├── ironic.env -└── kustomization.yaml -``` - -You can check that the kustomization works and inspect the resulting manifest by -running this: - -```bash -kubectl create -k bmo --dry-run=client -o yaml -``` - -When you are happy with the output, apply it in the cluster: - -```bash -kubectl apply -k bmo +```yaml +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/bmo/kustomization.yaml"}} ``` -## Deployment summary - -You are not expected to go through all the above steps each time you want to -deploy Metal3. Store the configuration and reuse it the next time. - -Here is a summary of the deploy steps when all configuration is already in -place. - -1. Create the management cluster. - - ```bash - kind create cluster --config kind.yaml - ``` - -1. Deploy cert-manager. - - ```bash - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.3/cert-manager.yaml - ``` - -1. Start the DHCP server. - - ```bash - docker run --name dnsmasq --rm -d --net=host --privileged --user 997:994 \ - --env-file dnsmasq.env --entrypoint /bin/rundnsmasq \ - quay.io/metal3-io/ironic - ``` - -1. Start the image server. - - ```bash - docker run --name image-server --rm -d -p 80:8080 \ - -v "$(pwd)/disk-images:/usr/share/nginx/html" nginxinc/nginx-unprivileged - ``` - -1. Deploy Ironic. - - ```bash - kubectl apply -k ironic - ``` - -1. Deploy Bare Metal Operator. - - ```bash - kubectl apply -k bmo - ``` - ## Create BareMetalHosts Now that we have Bare Metal Operator deployed, let's put it to use by creating @@ -620,14 +188,7 @@ accessing its BMC. No credentials are needed in the virtualized setup but you still need to create the secret with some values. Here is an example: ```yaml -apiVersion: v1 -kind: Secret -metadata: - name: bml-01 -type: Opaque -stringData: - username: replaceme - password: replaceme +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/bmc-secret.yaml"}} ``` Then continue by creating the BareMetalHost manifest. You can put it in the same @@ -645,11 +206,10 @@ metadata: name: bml-01 spec: online: true - bootMACAddress: 80:c1:6e:7a:e8:10 - # This particular hardware does not support UEFI so we use legacy - bootMode: legacy + bootMACAddress: 9C:63:C0:AC:10:42 + bootMode: UEFI bmc: - address: ilo4-virtualmedia://192.168.1.13 + address: idrac-virtualmedia://192.168.1.28 credentialsName: bml-01 disableCertificateVerification: true ``` @@ -657,18 +217,7 @@ spec: Here is the same for the virtualized BareMetalHost: ```yaml -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - name: bml-vm-01 -spec: - online: true - bootMACAddress: 00:60:2f:31:81:01 - bootMode: UEFI # use 'legacy' for Scenario 2 - hardwareProfile: libvirt - bmc: - address: redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmh-vm-01 - credentialsName: bml-01 +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/bmh-01.yaml"}} ``` Apply these in the cluster with `kubectl apply -f path/to/file`. @@ -692,23 +241,7 @@ Edit the BareMetalHost to add details of what image you want to provision it with. For example: ```yaml -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - name: bml-01 -spec: - online: true - bootMACAddress: 80:c1:6e:7a:e8:10 - bootMode: legacy - bmc: - address: ilo4-virtualmedia://192.168.1.13 - credentialsName: bml-01 - disableCertificateVerification: true - image: - checksumType: sha256 - checksum: http://192.168.0.150/SHA256SUMS - format: qcow2 - url: http://192.168.0.150/jammy-server-cloudimg-amd64.img +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/bmh-01-provision.yaml"}} ``` Note that the URL for the disk image is _not_ using the out of band network. @@ -768,10 +301,10 @@ consider using Cluster API and the infrastructure provider for Metal3. In this section we will show how to do it. Initialize the Cluster API core components and the infrastructure provider for -Metal3: +Metal3 (if you didn't already do it): ```bash -clusterctl init --infrastructure metal3 +clusterctl init --infrastructure metal3 --ipam=metal3 ``` Now we need to set some environment variables that will be used to render the @@ -790,31 +323,7 @@ options: control-plane nodes. ```bash -export IMAGE_CHECKSUM="ab54897a1bcae83581512cdeeda787f009846cfd7a63b298e472c1bd6c522d23" -export IMAGE_CHECKSUM_TYPE="sha256" -export IMAGE_FORMAT="qcow2" -# Baremetal lab IMAGE_URL -export IMAGE_URL="http://192.168.0.150/CENTOS_9_NODE_IMAGE_K8S_v1.33.0.qcow2" -# Virtualized setup IMAGE_URL -export IMAGE_URL="http://192.168.222.1/CENTOS_9_NODE_IMAGE_K8S_v1.33.0.qcow2" -export KUBERNETES_VERSION="v1.33.0" -# Make sure this does not conflict with other networks -export POD_CIDR='["192.168.10.0/24"]' -# These can be used to add user-data -export CTLPLANE_KUBEADM_EXTRA_CONFIG=" - users: - - name: user - sshAuthorizedKeys: - - ssh-ed25519 ABCD... user@example.com" -export WORKERS_KUBEADM_EXTRA_CONFIG=" - users: - - name: user - sshAuthorizedKeys: - - ssh-ed25519 ABCD... user@example.com" -# NOTE! You must ensure that this is forwarded or assigned somehow to the -# server(s) that is selected for the control-plane. -export CLUSTER_APIENDPOINT_HOST="192.168.0.101" -export CLUSTER_APIENDPOINT_PORT="6443" +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/capm3-vars.sh"}} ``` With the variables in place, we can render the manifests and apply: @@ -832,15 +341,24 @@ bml-02 provisioned my-cluster-controlplane-8z46n true 68m ``` If all goes well and the API endpoint is correctly configured, you should -eventually see a healthy cluster. Check with -`clusterctl describe cluster my-cluster`: +eventually get a working cluster. Note that it will not become fully healthy +until a CNI is deployed. + +Deploy Calico as CNI: + +```bash +clusterctl get kubeconfig my-cluster > kubeconfig.yaml +kubectl --kubeconfig=kubeconfig.yaml apply --server-side -f https://raw.githubusercontent.com/projectcalico/calico/v3.31.0/manifests/calico.yaml +``` + +Check cluster health with `clusterctl describe cluster my-cluster`: ```text -NAME READY SEVERITY REASON SINCE MESSAGE -Cluster/my-cluster True 76s -├─ClusterInfrastructure - Metal3Cluster/my-cluster True 15m -└─ControlPlane - KubeadmControlPlane/my-cluster True 76s - └─Machine/my-cluster-cj5zt True 76s +NAME REPLICAS AVAILABLE READY UP TO DATE STATUS REASON SINCE MESSAGE +Cluster/my-cluster 1/1 1 1 1 Available: True Available 48s +├─ClusterInfrastructure - Metal3Cluster/my-cluster Ready: True NoReasonReported 32m +└─ControlPlane - KubeadmControlPlane/my-cluster 1/1 1 1 1 + └─Machine/my-cluster-2zc9x 1 1 1 1 Ready: True Ready 48s ``` ## Cleanup @@ -860,22 +378,21 @@ Delete the management cluster. kind delete cluster ``` -Stop DHCP and image servers. They are automatically removed when stopped. +Stop image server. It is automatically removed when stopped. ```bash -docker stop dnsmasq docker stop image-server ``` -If you did the virtualized setup you will also need to cleanup the sushy-tools -container and the VM. +You may also want to delete the disk images: ```bash -docker stop sushy-tools +rm -r disk-images +``` -virsh -c qemu:///system destroy --domain bmh-vm-01 -virsh -c qemu:///system undefine --domain bmh-vm-01 --remove-all-storage --nvram +If you did the virtualized setup you will also need to cleanup the sushy-tools +container and the VM(s). -virsh -c qemu:///system net-destroy baremetal -virsh -c qemu:///system net-undefine baremetal +```bash +{{#embed-github repo:"Nordix/metal3-docs" branch:"lentzi90/quick-revision" path:"docs/user-guide/examples/cleanup-virtual-lab.sh"}} ```