diff --git a/common/library/module_utils/input_validation/common_utils/config.py b/common/library/module_utils/input_validation/common_utils/config.py index 9240080e1b..52f6c49edd 100644 --- a/common/library/module_utils/input_validation/common_utils/config.py +++ b/common/library/module_utils/input_validation/common_utils/config.py @@ -38,6 +38,7 @@ "x86_64": [ "slurm_control_node", "slurm_node", "login_node", "login_compiler_node", "service_kube_control_plane", "service_kube_control_plane_first", "service_kube_node", + "compute_kube_control_plane", "compute_kube_control_plane_first", "compute_kube_node", "os" ], "aarch64": [ @@ -96,6 +97,11 @@ files["storage_config"], files["high_availability_config"], ], + "compute_rke2": [ + files["omnia_config"], + files["storage_config"], + files["high_availability_config"], + ], "storage": [files["storage_config"]], "prepare_oim": [ files["network_spec"], @@ -191,6 +197,9 @@ "service_kube_control_plane_first_x86_64": "management", "service_kube_control_plane_x86_64": "management", "service_kube_node_x86_64": "management", + "compute_kube_control_plane_first_x86_64": "compute", + "compute_kube_control_plane_x86_64": "compute", + "compute_kube_node_x86_64": "compute", "login_node_x86_64": "management", "login_node_aarch64": "management", "login_compiler_node_x86_64": "management", diff --git a/common/library/module_utils/input_validation/schema/omnia_config.json b/common/library/module_utils/input_validation/schema/omnia_config.json index 09fdc3f75e..2da69f9f15 100644 --- a/common/library/module_utils/input_validation/schema/omnia_config.json +++ b/common/library/module_utils/input_validation/schema/omnia_config.json @@ -176,6 +176,126 @@ } ] } + }, + "service_rke2_k8s_cluster": { + "type": "array", + "description": "List of RKE2 Kubernetes cluster configurations.", + "items": { + "type": "object", + "properties": { + "cluster_name": { + "type": "string", + "minLength": 1, + "description": "Unique name for the RKE2 service cluster." + }, + "deployment": { + "type": "boolean" + }, + "k8s_cni": { + "enum": ["calico", "canal", "cilium", "flannel"], + "description": "K8s CNI plugin for the RKE2 cluster." + }, + "pod_external_ip_range": { + "description": "IP range for LoadBalancer external IPs.", + "type": "string", + "pattern": "^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)-(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$|^$" + }, + "k8s_service_addresses": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s service IPs." + }, + "k8s_pod_network_cidr": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s pod network." + }, + "nfs_storage_name": { + "type": "string", + "description": "The NFS client server name mentioned in storage_config.yml" + }, + "csi_powerscale_driver_secret_file_path": { + "description": "Absolute file path for the secret.yaml file.", + "type": "string", + "pattern": "^(|/?([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+\\.yaml)$" + }, + "csi_powerscale_driver_values_file_path": { + "description": "File path for the values.yaml file.", + "type": "string", + "pattern": "^(|/?([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+\\.yaml)$" + } + }, + "required": [ + "cluster_name", + "k8s_cni", + "k8s_service_addresses", + "k8s_pod_network_cidr", + "pod_external_ip_range" + ], + "allOf": [ + { + "if": { + "properties": { + "csi_powerscale_driver_secret_file_path": { + "type": "string", + "minLength": 1 + } + }, + "required": ["csi_powerscale_driver_secret_file_path"] + }, + "then": { + "required": ["csi_powerscale_driver_values_file_path"] + } + } + ] + } + }, + "compute_rke2_k8s_cluster": { + "type": "array", + "description": "List of RKE2-based compute Kubernetes cluster configurations.", + "items": { + "type": "object", + "properties": { + "cluster_name": { + "type": "string", + "minLength": 1, + "description": "Unique name for the RKE2 compute cluster." + }, + "deployment": { + "type": "boolean" + }, + "k8s_cni": { + "enum": ["calico", "canal", "cilium", "flannel"], + "description": "K8s CNI plugin for the compute RKE2 cluster." + }, + "pod_external_ip_range": { + "description": "IP range for LoadBalancer external IPs.", + "type": "string", + "pattern": "^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)-(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$|^$" + }, + "k8s_service_addresses": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s service IPs." + }, + "k8s_pod_network_cidr": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s pod network." + }, + "nfs_storage_name": { + "type": "string", + "description": "The NFS client server name mentioned in storage_config.yml" + } + }, + "required": [ + "cluster_name", + "k8s_cni", + "k8s_service_addresses", + "k8s_pod_network_cidr", + "pod_external_ip_range" + ] + } } }, "required": [ diff --git a/common/library/module_utils/input_validation/validation_flows/common_validation.py b/common/library/module_utils/input_validation/validation_flows/common_validation.py index b1aef8e919..37f1e0b1e6 100644 --- a/common/library/module_utils/input_validation/validation_flows/common_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/common_validation.py @@ -214,8 +214,35 @@ def validate_software_config( ) ) - # Ensure ldms is not configured without service_k8s in softwares - if "ldms" in software_names and "service_k8s" not in software_names: + # Ensure service_k8s and service_rke2 are not both present + if "service_k8s" in software_names and "service_rke2" in software_names: + errors.append( + create_error_msg( + "Validation Error: ", + "service_k8s and service_rke2", + "cannot both be present in softwares. Use service_k8s for kubeadm or service_rke2 for RKE2, not both." + ) + ) + + # RKE2 packages are only available for x86_64; reject any other arch explicitly + for rke2_sw_name in ["service_rke2", "compute_rke2"]: + if rke2_sw_name in software_names: + rke2_entry = next( + (s for s in data.get("softwares", []) if s.get("name") == rke2_sw_name), {} + ) + rke2_arch = rke2_entry.get("arch", ["x86_64"]) + unsupported_archs = [a for a in rke2_arch if a != "x86_64"] + if unsupported_archs: + errors.append( + create_error_msg( + "Validation Error: ", + rke2_sw_name, + f"is only supported on x86_64. Unsupported arch(es): {', '.join(unsupported_archs)}." + ) + ) + + # Ensure ldms is not configured without service_k8s or service_rke2 in softwares + if "ldms" in software_names and "service_k8s" not in software_names and "service_rke2" not in software_names: errors.append( create_error_msg( "Validation Error: ", @@ -234,7 +261,7 @@ def validate_software_config( ) # Check for required subgroups when specific software names are present - software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s"] + software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s", "service_rke2", "compute_rke2"] for software_name in software_requiring_subgroups: if software_name in software_names: if software_name not in data or not data[software_name]: @@ -553,6 +580,22 @@ def get_matching_clusters_for_nfs(nfs_name, omnia_config): ): matching_clusters["service_k8s_cluster"] = svc + # Service RKE2 + for svc in omnia_config.get("service_rke2_k8s_cluster", []): + if ( + svc.get("nfs_storage_name") == nfs_name + and svc.get("deployment") is True + ): + matching_clusters["service_rke2_k8s_cluster"] = svc + + # Compute RKE2 + for compute in omnia_config.get("compute_rke2_k8s_cluster", []): + if ( + compute.get("nfs_storage_name") == nfs_name + and compute.get("deployment") is True + ): + matching_clusters["compute_rke2_k8s_cluster"] = compute + # Slurm for slurm in omnia_config.get("slurm_cluster", []): if slurm.get("nfs_storage_name") == nfs_name: @@ -1463,6 +1506,14 @@ def validate_k8s(data, admin_networks, softwares, ha_config, tag_names, errors, cluster_set["service_k8s_cluster"] = data.get( "service_k8s_cluster", []) + if "service_rke2" in softwares and "service_rke2" in tag_names: + cluster_set["service_rke2_k8s_cluster"] = data.get( + "service_rke2_k8s_cluster", []) + + if "compute_rke2" in softwares and "compute_rke2" in tag_names: + cluster_set["compute_rke2_k8s_cluster"] = data.get( + "compute_rke2_k8s_cluster", []) + for k8s_cluster_type, k8s_clusters in cluster_set.items(): deployments_list = [k.get('deployment', False) for k in k8s_clusters] true_count = deployments_list.count(True) diff --git a/common/library/modules/image_package_collector.py b/common/library/modules/image_package_collector.py index 1f40fffbf7..9904bdbcaf 100644 --- a/common/library/modules/image_package_collector.py +++ b/common/library/modules/image_package_collector.py @@ -109,8 +109,14 @@ def collect_packages_from_json(sw_data, fg_name=None, elif service_k8s_defined: fg_name = fg_name.replace("_aarch64", "").replace("_x86_64", "") - if "service_k8s" in sw_data and "cluster" in sw_data["service_k8s"]: - for entry in sw_data["service_k8s"]["cluster"]: + k8s_top_key = ( + "compute_rke2" if "compute_rke2" in sw_data + else "service_rke2" if "service_rke2" in sw_data + else "service_k8s" + ) + + if k8s_top_key in sw_data and "cluster" in sw_data[k8s_top_key]: + for entry in sw_data[k8s_top_key]["cluster"]: if entry.get("type") == "rpm" and "package" in entry: packages.append(entry["package"]) @@ -157,6 +163,7 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir, # Remove version suffix for versioned files (e.g., service_k8s_v1.35.1 -> service_k8s) if sw_name.startswith("service_k8s_v"): sw_name = "service_k8s" + # compute_rke2.json -> compute_rke2 (already handled by replace above) if sw_name not in allowed_softwares: continue @@ -175,8 +182,8 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir, sw_data, fg_name=fg_name, slurm_defined=True ) ) - elif json_file.startswith("service_k8s_v"): - # Handle versioned service_k8s_v.json files + elif json_file.startswith("service_k8s_v") or json_file == "service_rke2.json" or json_file == "compute_rke2.json": + # Handle versioned service_k8s_v.json or service_rke2.json packages.extend( collect_packages_from_json( sw_data, fg_name=fg_name, service_k8s_defined=True @@ -251,17 +258,36 @@ def run_module(): "service_kube_control_plane_x86_64" } + # Compute RKE2 functional groups + compute_k8s_functional_groups = { + "compute_kube_node_x86_64", + "compute_kube_control_plane_first_x86_64", + "compute_kube_control_plane_x86_64" + } + # Check if any k8s functional groups are being processed needs_service_k8s = any(fg in k8s_functional_groups for fg in functional_groups) + needs_compute_rke2 = any(fg in compute_k8s_functional_groups for fg in functional_groups) # Only validate service_k8s version if k8s functional groups are present service_k8s_json = None - if needs_service_k8s: + if needs_service_k8s and "service_rke2" not in allowed_softwares: if not service_k8s_version: module.fail_json(msg="service_k8s version not found in software_config.json") service_k8s_json = f"service_k8s_v{service_k8s_version}.json" # pylint: disable=line-too-long + # Determine which K8s package JSON to use based on software_config.json + if "service_rke2" in allowed_softwares: + k8s_json = "service_rke2.json" + elif service_k8s_json: + k8s_json = service_k8s_json + else: + k8s_json = None + + # Determine compute RKE2 package JSON + compute_rke2_json = "compute_rke2.json" if "compute_rke2" in allowed_softwares else None + # Functional group → json files mapping software_map = { "os_x86_64": ["default_packages.json", "ldms.json"], @@ -280,12 +306,18 @@ def run_module(): ], } - # Add k8s functional groups to software_map only if service_k8s_json is available - if service_k8s_json: + if k8s_json: + software_map.update({ + "service_kube_node_x86_64": [k8s_json], + "service_kube_control_plane_first_x86_64": [k8s_json], + "service_kube_control_plane_x86_64": [k8s_json], + }) + + if compute_rke2_json: software_map.update({ - "service_kube_node_x86_64": [service_k8s_json], - "service_kube_control_plane_first_x86_64": [service_k8s_json], - "service_kube_control_plane_x86_64": [service_k8s_json], + "compute_kube_node_x86_64": [compute_rke2_json], + "compute_kube_control_plane_first_x86_64": [compute_rke2_json], + "compute_kube_control_plane_x86_64": [compute_rke2_json], }) compute_images_dict = {} diff --git a/input/config/x86_64/rhel/10.0/compute_rke2.json b/input/config/x86_64/rhel/10.0/compute_rke2.json new file mode 100644 index 0000000000..263b728b5b --- /dev/null +++ b/input/config/x86_64/rhel/10.0/compute_rke2.json @@ -0,0 +1,56 @@ +{ + "compute_rke2": { + "cluster": [ + { "package": "docker.io/library/busybox", "type": "image", "tag": "1.36" }, + { "package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "vim", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2-common" }, + { "package": "rke2-images-core.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-core.linux-amd64.tar.zst" }, + { "package": "rke2-images-calico.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-calico.linux-amd64.tar.zst" }, + { "package": "rke2-images-canal.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-canal.linux-amd64.tar.zst" }, + { "package": "rke2-images-cilium.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-cilium.linux-amd64.tar.zst" }, + { "package": "rke2-images-flannel.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-flannel.linux-amd64.tar.zst" }, + { "package": "apptainer", "type": "rpm", "repo_name": "epel" }, + { "package": "doca-ofed", "type": "iso", "url": "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.1/host/doca-host-3.2.1-044000_25.10_rhel10.x86_64.rpm"} + ] + }, + "compute_kube_control_plane": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" } + ] + }, + "compute_kube_control_plane_first": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { + "package": "metallb-native-v0.15.2", + "type": "manifest", + "url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml" + }, + { "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" }, + { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" } + ] + }, + "compute_kube_node": { + "cluster": [ + { "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.15.2", "type": "image" } + ] + } +} diff --git a/input/config/x86_64/rhel/10.0/service_rke2.json b/input/config/x86_64/rhel/10.0/service_rke2.json new file mode 100644 index 0000000000..d74d76cad7 --- /dev/null +++ b/input/config/x86_64/rhel/10.0/service_rke2.json @@ -0,0 +1,78 @@ +{ + "service_rke2": { + "cluster": [ + { "package": "docker.io/library/busybox", "type": "image", "tag": "1.36" }, + { "package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "vim", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2-common" }, + { "package": "rke2-images-core.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-core.linux-amd64.tar.zst" }, + { "package": "rke2-images-calico.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-calico.linux-amd64.tar.zst" }, + { "package": "rke2-images-canal.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-canal.linux-amd64.tar.zst" }, + { "package": "rke2-images-cilium.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-cilium.linux-amd64.tar.zst" }, + { "package": "rke2-images-flannel.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-flannel.linux-amd64.tar.zst" }, + { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.128.0" }, + { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.128.0" }, + { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/victoriametrics/vminsert", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/victoriametrics/vmselect", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/curlimages/curl", "type": "image", "tag": "8.17.0" }, + { "package": "docker.io/rmohr/activemq", "type": "image", "tag": "5.15.9" }, + { "package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0" }, + { "package": "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "type": "image", "tag": "1.0" }, + { "package": "docker.io/dellhpcomniaaisolution/kafkapump", "type": "image", "tag": "1.0" }, + { "package": "docker.io/dellhpcomniaaisolution/victoriapump", "type": "image", "tag": "1.0" }, + { "package": "cryptography==45.0.7", "type": "pip_module" }, + { "package": "omsdk==1.2.518", "type": "pip_module" }, + { "package": "cffi==1.17.1", "type": "pip_module" }, + { "package": "quay.io/strimzi/operator", "tag": "0.48.0", "type": "image" }, + { "package": "quay.io/strimzi/kafka", "tag": "0.48.0-kafka-4.1.0", "type": "image" }, + { "package": "docker.io/dellhpcomniaaisolution/ubuntu-ldms", "tag": "1.0", "type": "image" }, + { "package": "strimzi-kafka-operator-helm-3-chart-0.48.0", "type": "tarball", "url": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" }, + { "package": "quay.io/strimzi/kafka-bridge", "tag": "0.33.1", "type": "image" }, + { "package": "apptainer", "type": "rpm", "repo_name": "epel" }, + { "package": "doca-ofed", "type": "iso", "url": "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.1/host/doca-host-3.2.1-044000_25.10_rhel10.x86_64.rpm"} + ] + }, + "service_kube_control_plane": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" }, + { "package": "PyMySQL==1.1.2", "type": "pip_module" } + ] + }, + "service_kube_control_plane_first": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { + "package": "metallb-native-v0.15.2", + "type": "manifest", + "url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml" + }, + { "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" }, + { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" }, + { "package": "PyMySQL==1.1.2", "type": "pip_module" } + ] + }, + + "service_kube_node": { + "cluster": [ + { "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.15.2", "type": "image" } + ] + } +} diff --git a/input/local_repo_config.yml b/input/local_repo_config.yml index 2a0b0d5276..f61e7583ff 100644 --- a/input/local_repo_config.yml +++ b/input/local_repo_config.yml @@ -67,7 +67,7 @@ # sslcacert : Path to SSL CA certificate (if using SSL) # sslclientkey: Path to SSL client key (if using SSL) # sslclientcert: Path to SSL client certificate (if using SSL) -# policy : Repository policy if mentioned allowed values (always, partial). +# policy : Repository policy if mentioned allowed values (always, partial). # If not provided, uses repo_config from software_config.json # caching : Enable or disable local caching. Allowed values: true, false (OPTIONAL) # If not provided, defaults to true @@ -84,9 +84,9 @@ # # 6. rhel_subscription_repo_config_x86_64 #------------------------------------------- -# Optional configuration for overriding policy and caching settings for RHEL +# Optional configuration for overriding policy and caching settings for RHEL # subscription-based repositories on x86_64 architecture. -# When subscription is enabled, this config takes precedence over dynamically +# When subscription is enabled, this config takes precedence over dynamically # generated URLs for matching repositories and adds any additional repositories. # Fields: # url : Base URL of the repository (REQUIRED) @@ -195,6 +195,10 @@ omnia_repo_url_rhel_x86_64: - { url: "https://dl.fedoraproject.org/pub/epel/10/Everything/x86_64/", gpgkey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-10", name: "epel"} - { url: "https://pkgs.k8s.io/core:/stable:/v1.35/rpm/", gpgkey: "https://pkgs.k8s.io/core:/stable:/v1.35/rpm/repodata/repomd.xml.key", name: "kubernetes-v1-35"} - { url: "https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/", gpgkey: "https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/repodata/repomd.xml.key", name: "cri-o-v1-35"} + # Channel (stable/1.35) must match the minor version in service_rke2.json and software_config.json. + - { url: "https://rpm.rancher.io/rke2/stable/1.35/centos/10/x86_64/", gpgkey: "https://rpm.rancher.io/public.key", name: "rke2"} + # noarch repo providing rke2-selinux + - { url: "https://rpm.rancher.io/rke2/stable/common/centos/10/noarch/", gpgkey: "https://rpm.rancher.io/public.key", name: "rke2-common"} - { url: "https://linux.mellanox.com/public/repo/doca/3.2.1/rhel10/x86_64/", gpgkey: "https://linux.mellanox.com/public/repo/doca/3.2.1/rhel10/x86_64/repodata/repomd.xml.key", name: "doca"} - { url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel10/x86_64/", gpgkey: "https://developer.download.nvidia.com/compute/cuda/repos/rhel10/x86_64/repodata/repomd.xml.key", name: "cuda"} omnia_repo_url_rhel_aarch64: diff --git a/input/omnia_config.yml b/input/omnia_config.yml index 3086e491e4..4b6e86b9d7 100644 --- a/input/omnia_config.yml +++ b/input/omnia_config.yml @@ -201,3 +201,40 @@ service_k8s_cluster: k8s_crio_storage_size: "20G" csi_powerscale_driver_secret_file_path: "" csi_powerscale_driver_values_file_path: "" + +# ----------------------------SERVICE RKE2 K8S-------------------------------------------------- +# To deploy RKE2 instead of vanilla Kubernetes (kubeadm), configure this block +# and add "service_rke2" to software_config.json softwares list instead of "service_k8s". +# RKE2 bundles its own container runtime (containerd) and manages the CNI lifecycle. +# k8s_cni supports "calico", "canal", "cilium", and "flannel". +# Only one of service_k8s_cluster or service_rke2_k8s_cluster should have deployment: true. +# Please ensure corresponding cluster entry is added to high_availability_config.yml. + +# service_rke2_k8s_cluster: +# - cluster_name: service_cluster +# deployment: true +# k8s_cni: "calico" +# pod_external_ip_range: "172.16.107.170-172.16.107.200" +# k8s_service_addresses: "10.233.0.0/18" +# k8s_pod_network_cidr: "10.233.64.0/18" +# nfs_storage_name: "nfs_k8s" +# csi_powerscale_driver_secret_file_path: "" +# csi_powerscale_driver_values_file_path: "" + +# ----------------------------COMPUTE RKE2 K8S------------------------------------------------- +# To deploy a separate RKE2-based compute Kubernetes cluster, configure this block +# and add "compute_rke2" to software_config.json softwares list. +# The compute RKE2 cluster is independent of the service cluster and can coexist +# with service_k8s (kubeadm), service_rke2, and slurm clusters. +# k8s_cni supports "calico", "canal", "cilium", and "flannel". +# Please ensure a corresponding cluster entry is added to high_availability_config.yml. +# CIDRs must not overlap with the service cluster CIDRs. + +# compute_rke2_k8s_cluster: +# - cluster_name: compute_cluster +# deployment: false +# k8s_cni: "calico" +# pod_external_ip_range: "" +# k8s_service_addresses: "10.234.0.0/18" +# k8s_pod_network_cidr: "10.234.64.0/18" +# nfs_storage_name: "nfs_compute_k8s" diff --git a/input/software_config.json b/input/software_config.json index 877ba0c3dc..554120d802 100644 --- a/input/software_config.json +++ b/input/software_config.json @@ -23,10 +23,18 @@ {"name": "service_kube_control_plane"}, {"name": "service_kube_node"} ], + "compute_rke2": [ + {"name": "compute_kube_control_plane_first"}, + {"name": "compute_kube_control_plane"}, + {"name": "compute_kube_node"} + ], "additional_packages":[ {"name": "service_kube_control_plane_first"}, {"name": "service_kube_control_plane"}, {"name": "service_kube_node"}, + {"name": "compute_kube_control_plane_first"}, + {"name": "compute_kube_control_plane"}, + {"name": "compute_kube_node"}, {"name": "slurm_control_node"}, {"name": "slurm_node"}, {"name": "login_node"}, diff --git a/local_repo/roles/validation/tasks/validate_software_config_json.yml b/local_repo/roles/validation/tasks/validate_software_config_json.yml index 190904eb95..21e1cad807 100644 --- a/local_repo/roles/validation/tasks/validate_software_config_json.yml +++ b/local_repo/roles/validation/tasks/validate_software_config_json.yml @@ -31,13 +31,25 @@ - name: Check if service k8s support is true ansible.builtin.set_fact: service_k8s_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | list | length > 0 }}" + service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}" + compute_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'compute_rke2') | list | length > 0 }}" software_names: "{{ software_config.softwares | map(attribute='name') | select('defined') | list }}" software_json_list: "{{ software_names | map('regex_replace', '$', '.json') | list }}" +- name: Set service_k8s_support true when rke2 is selected + ansible.builtin.set_fact: + service_k8s_support: true + when: service_rke2_support | bool + - name: Get k8s archs ansible.builtin.set_fact: service_k8s_arch: "{{ (software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | first).get('arch', default_archs) }}" - when: service_k8s_support + when: service_k8s_support and not (service_rke2_support | bool) + +- name: Get rke2 archs + ansible.builtin.set_fact: + service_k8s_arch: "{{ (software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | first).get('arch', default_archs) }}" + when: service_rke2_support | bool - name: Get k8s archs ansible.builtin.set_fact: @@ -91,4 +103,5 @@ msg: "{{ fail_msg }}" when: - service_k8s_support + - not (service_rke2_support | default(false) | bool) - service_k8s_version not in supported_k8s_versions diff --git a/provision/provision.yml b/provision/provision.yml index 5aac96f3dc..f354b850a1 100644 --- a/provision/provision.yml +++ b/provision/provision.yml @@ -67,9 +67,19 @@ ( ['service_k8s'] if ( 'service_k8s' in ( + lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json') + | from_json).softwares | map(attribute='name') | list + or 'service_rke2' in ( lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json') | from_json).softwares | map(attribute='name') | list ) else [] + ) + + ( + ['compute_rke2'] + if 'compute_rke2' in ( + lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json') + | from_json).softwares | map(attribute='name') | list + else [] ) ) | unique }} diff --git a/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml b/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml index a999c8b9ef..1ac7a24fdf 100644 --- a/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml +++ b/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml @@ -26,11 +26,23 @@ failed_when: false when: not (hostvars['localhost']['upgrade_mode'] | default(false) | bool) +- name: Determine cloud-init template path based on k8s_distro # noqa: yaml[line-length] + ansible.builtin.set_fact: + ci_group_template_src: >- + {%- if k8s_distro | default('kubeadm') == 'rke2' and + 'service_kube' in functional_group_name -%} + cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2 + {%- elif 'compute_kube' in functional_group_name -%} + cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2 + {%- else -%} + cloud_init/ci-group-{{ functional_group_name }}.yaml.j2 + {%- endif -%} + - name: Load ci group template block: - name: Load ci group template - {{ functional_group_name }} ansible.builtin.template: - src: "cloud_init/ci-group-{{ functional_group_name }}.yaml.j2" + src: "{{ ci_group_template_src | trim }}" dest: "{{ cloud_init_dir }}/ci-group-{{ functional_group_name }}.yaml" mode: "{{ hostvars['localhost']['file_permissions_644'] }}" rescue: diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_first_x86_64.yaml.j2 new file mode 100644 index 0000000000..7e978e74af --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_first_x86_64.yaml.j2 @@ -0,0 +1,523 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host * + IdentityFile {{ compute_k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/chrony.conf + permissions: '0644' + content: | + server {{ cluster_boot_ip }} iburst + + driftfile /var/lib/chrony/drift + rtcsync + makestep 1.0 3 + logdir /var/log/chrony + cmdport 0 + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + + - path: /etc/fstab + content: | + {{ compute_k8s_nfs_server_path }} {{ compute_k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ compute_k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/config.yaml + owner: root:root + permissions: '0600' + content: | + tls-san: + - {{ compute_kube_vip }} + - {% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} + node-name: {% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} + write-kubeconfig-mode: "0644" + cluster-cidr: {{ compute_k8s_pod_network_cidr }} + service-cidr: {{ compute_k8s_service_addresses }} + cni: {{ compute_k8s_cni }} + disable: + - rke2-ingress-nginx + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + - path: /tmp/kube-vip.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: vip_interface + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: {{ compute_kube_vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.8.9 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + status: {} + + - path: /usr/local/bin/install-helm.sh + permissions: '0755' + content: | + #!/bin/bash + set -e + ARCH="amd64" + cp {{ compute_k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + - path: /tmp/ipaddress_pool.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: first-pool + namespace: metallb-system + spec: + addresses: + - {{ compute_pod_external_ip_range }} + + - path: /tmp/l2advertisement.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + metadata: + name: default + namespace: metallb-system + spec: + ipAddressPools: + - first-pool + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports (Calico BGP, VXLAN, WireGuard, NodePort, MetalLB) + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ compute_k8s_client_mount_path }} /var/log/pods /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ compute_k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ compute_k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - mv /tmp/generate-control-plane-join.sh {{ compute_k8s_client_mount_path }} || true + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.compute_cluster_initialized" + export KUBECONFIG="/etc/rancher/rke2/rke2.yaml" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + if [ ! -f "$MARKER" ]; then + # FIRST BOOT - CLUSTER INIT + echo "Initial boot - initializing RKE2 first server for compute_kube_control_plane_first_x86_64" + mv /tmp/ipaddress_pool.yaml {{ compute_k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + mv /tmp/l2advertisement.yaml {{ compute_k8s_client_mount_path }}/metallb/l2advertisement.yaml + + # Determine the VIP interface + NODE_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + VIP_IFACE=$(ip -o addr show | awk -v ip="$NODE_IP" '$4 ~ ip {print $2}') + sed -i "s/value: vip_interface/value: ${VIP_IFACE}/" /tmp/kube-vip.yaml + + # Start RKE2 server + echo "Starting RKE2 server..." + systemctl enable rke2-server.service + systemctl start rke2-server.service + + # Wait for RKE2 to generate kubeconfig + echo "Waiting for RKE2 kubeconfig..." + while [ ! -f /etc/rancher/rke2/rke2.yaml ]; do + sleep 5 + done + + # Set up kubectl access + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + chmod 600 $HOME/.kube/config + + # Place kube-vip as a kubelet static pod now that rke2.yaml exists. + cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/agent/pod-manifests/kube-vip.yaml + + echo "Waiting for one Ready control plane node ..." + while true; do + control_plane_ready=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l) + if [ "$control_plane_ready" -ge 1 ]; then + echo "Found $control_plane_ready Ready node(s)!" + break + else + echo "No Ready node yet, waiting 5s ..." + sleep 5 + fi + done + + # Wait for all pods to be ready + echo "Waiting for all pods to be Ready (Running/Completed)..." + while true; do + not_ready=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '{ print $4 }' | grep -vE '^(Running|Completed)$' | wc -l) + if [ "${not_ready}" -eq 0 ]; then + echo "All pods are Running or Completed." + break + else + echo "$not_ready pods not yet ready, waiting 5s ..." + sleep 5 + fi + done + + # Save the server token to NFS for other nodes to join + K8S_CLIENT_MOUNT_PATH="{{ compute_k8s_client_mount_path }}" + while [ ! -f /var/lib/rancher/rke2/server/node-token ]; do + echo "Waiting for RKE2 node token..." + sleep 5 + done + cp /var/lib/rancher/rke2/server/node-token "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + chmod 644 "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + echo "RKE2 server token saved to NFS." + + # Update kubeconfig to use VIP + KUBE_VIP="{{ compute_kube_vip }}" + KUBE_PORT="6443" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" /etc/rancher/rke2/rke2.yaml + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + + # Also save kubeconfig to NFS for other control planes + cp /etc/rancher/rke2/rke2.yaml "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + + echo "Waiting for at least one READY Kubernetes worker node ..." + while true; do + if kubectl get nodes --no-headers | grep -Ev 'control-plane|master' | grep ' Ready '; then + echo "Worker node(s) present and Ready." + break + else + echo "No Ready worker node detected yet. Retrying in 10 seconds..." + sleep 10 + fi + done + + echo "Installing plugins" + echo "Installing nfs-client-provisioner" + /usr/local/bin/helm install nfs-client {{ compute_k8s_client_mount_path }}/nfs-client-provisioner/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz \ + --namespace default --create-namespace \ + --set nfs.server={{ compute_k8s_nfs_server_path.split(':')[0] }} \ + --set nfs.path={{ compute_k8s_nfs_server_path.split(':')[1] }} \ + --set storageClass.defaultClass=true \ + --set storageClass.reclaimPolicy=Retain + echo "Waiting for nfs-subdir-external-provisioner pods to appear..." + sleep 15 + + if kubectl get pods -n default -l app=nfs-subdir-external-provisioner | grep -q nfs-subdir; then + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + else + echo "Pods not yet created, retrying after 10 seconds..." + sleep 10 + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + fi + + echo "Installing MetalLB" + kubectl create -f {{ compute_k8s_client_mount_path }}/metallb/{{ compute_metallb_package }}.yml + echo "Waiting for MetalLB pods to be ready..." + kubectl wait --namespace metallb-system --for=condition=Ready pods --all --timeout=300s + echo "Waiting for MetalLB webhook to be ready..." + until kubectl get endpoints metallb-webhook-service -n metallb-system \ + -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -qE '[0-9]'; do + echo "Webhook endpoints not ready yet. Retrying in 5s..." + sleep 5 + done + echo "MetalLB webhook is ready." + echo "Deploy ipaddress pool" + kubectl create -f {{ compute_k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + echo "Deploy Layer2 Configuration" + kubectl create -f {{ compute_k8s_client_mount_path }}/metallb/l2advertisement.yaml + + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + + systemctl restart nfs-client.target + systemctl restart rpcbind + + echo "Display nodes and pods status after deploying plugins" + echo "Listing all Kubernetes nodes:" + kubectl get nodes -o wide + + echo "Listing all Kubernetes pods in all namespaces:" + kubectl get pods --all-namespaces -o wide + + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP INIT + echo "compute_kube_control_plane_first_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_x86_64.yaml.j2 new file mode 100644 index 0000000000..39053e6815 --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_x86_64.yaml.j2 @@ -0,0 +1,414 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host * + IdentityFile {{ compute_k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ compute_k8s_nfs_server_path }} {{ compute_k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ compute_k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + - path: /tmp/kube-vip.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: vip_interface + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: {{ compute_kube_vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.8.9 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + status: {} + + - path: /usr/local/bin/install-helm.sh + permissions: '0755' + content: | + #!/bin/bash + set -e + ARCH="amd64" + cp {{ compute_k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ compute_k8s_client_mount_path }} /var/log/pods /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ compute_k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ compute_k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + set -e + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.compute_cluster_initialized" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + if [ ! -f "$MARKER" ]; then + # Join RKE2 cluster as additional server + echo "Initial boot - joining compute RKE2 cluster as additional server for compute_kube_control_plane_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ compute_k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ compute_kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the compute RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "Compute RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable." + + # Write RKE2 config for joining as server + cat > /etc/rancher/rke2/config.yaml </dev/null 2>&1; do + sleep 10 + done + kubectl get nodes -o wide + + systemctl restart nfs-client.target + systemctl restart rpcbind + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP JOIN + echo "compute_kube_control_plane_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_node_x86_64.yaml.j2 new file mode 100644 index 0000000000..b1cbe20075 --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_node_x86_64.yaml.j2 @@ -0,0 +1,260 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host * + IdentityFile {{ compute_k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ compute_k8s_nfs_server_path }} {{ compute_k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ compute_k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open required ports for RKE2 agent + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ compute_k8s_client_mount_path }} /var/log/pods /etc/rancher/rke2 /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - systemctl restart rpcbind + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ compute_k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ compute_k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - | + set -e + MARKER="/etc/rancher/rke2/.compute_cluster_initialized" + if [ ! -f "$MARKER" ]; then + # Join compute RKE2 cluster as agent (worker node) + echo "Initial boot - joining compute RKE2 cluster as agent for compute_kube_node_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ compute_k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ compute_kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the compute RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "Compute RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable. Joining this compute_kube_node_x86_64 to the cluster now." + + # Write RKE2 agent config + cat > /etc/rancher/rke2/config.yaml < /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + - path: /tmp/ipaddress_pool.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: first-pool + namespace: metallb-system + spec: + addresses: + - {{ pod_external_ip_range }} + + - path: /tmp/l2advertisement.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + metadata: + name: default + namespace: metallb-system + spec: + ipAddressPools: + - first-pool + +{% if hostvars['localhost']['idrac_telemetry_support'] or hostvars['localhost']['ldms_support'] %} + - path: /root/telemetry.sh + owner: root:root + permissions: '0755' + content: | + {{ lookup('template', 'templates/telemetry/telemetry.sh.j2') | indent(12) }} +{% endif %} + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports (Calico BGP, VXLAN, WireGuard, NodePort, MetalLB) + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - mv /tmp/generate-control-plane-join.sh {{ k8s_client_mount_path }} || true + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.cluster_initialized" + export KUBECONFIG="/etc/rancher/rke2/rke2.yaml" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + if [ ! -f "$MARKER" ]; then + # FIRST BOOT - CLUSTER INIT + echo "Initial boot - initializing RKE2 first server for service_kube_control_plane_first_x86_64" + mv /tmp/ipaddress_pool.yaml {{ k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + mv /tmp/l2advertisement.yaml {{ k8s_client_mount_path }}/metallb/l2advertisement.yaml + + # Determine the VIP interface + NODE_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + VIP_IFACE=$(ip -o addr show | awk -v ip="$NODE_IP" '$4 ~ ip {print $2}') + sed -i "s/value: vip_interface/value: ${VIP_IFACE}/" /tmp/kube-vip.yaml + + # Start RKE2 server + echo "Starting RKE2 server..." + systemctl enable rke2-server.service + systemctl start rke2-server.service + + # Wait for RKE2 to generate kubeconfig + echo "Waiting for RKE2 kubeconfig..." + while [ ! -f /etc/rancher/rke2/rke2.yaml ]; do + sleep 5 + done + + # Set up kubectl access + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + chmod 600 $HOME/.kube/config + + # Place kube-vip as a kubelet static pod now that rke2.yaml exists. + # /var/lib/rancher/rke2/agent/pod-manifests is RKE2's kubelet + # staticPodPath; deploying after the kubeconfig is written avoids + # the hostPath mount creating a directory at the rke2.yaml path. + cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/agent/pod-manifests/kube-vip.yaml + + echo "Waiting for one Ready control plane node ..." + while true; do + control_plane_ready=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l) + if [ "$control_plane_ready" -ge 1 ]; then + echo "Found $control_plane_ready Ready node(s)!" + break + else + echo "No Ready node yet, waiting 5s ..." + sleep 5 + fi + done + + # Wait for all pods to be ready + echo "Waiting for all pods to be Ready (Running/Completed)..." + while true; do + not_ready=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '{ print $4 }' | grep -vE '^(Running|Completed)$' | wc -l) + if [ "${not_ready}" -eq 0 ]; then + echo "All pods are Running or Completed." + break + else + echo "$not_ready pods not yet ready, waiting 5s ..." + sleep 5 + fi + done + + # Save the server token to NFS for other nodes to join + K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}" + while [ ! -f /var/lib/rancher/rke2/server/node-token ]; do + echo "Waiting for RKE2 node token..." + sleep 5 + done + cp /var/lib/rancher/rke2/server/node-token "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + chmod 644 "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + echo "RKE2 server token saved to NFS." + + # Update kubeconfig to use VIP + KUBE_VIP="{{ kube_vip }}" + KUBE_PORT="6443" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" /etc/rancher/rke2/rke2.yaml + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + + # Also save kubeconfig to NFS for other control planes + cp /etc/rancher/rke2/rke2.yaml "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + + echo "Waiting for at least one READY Kubernetes worker node ..." + while true; do + if kubectl get nodes --no-headers | grep -Ev 'control-plane|master' | grep ' Ready '; then + echo "Worker node(s) present and Ready." + break + else + echo "No Ready worker node detected yet. Retrying in 10 seconds..." + sleep 10 + fi + done + + echo "Installing plugins" + echo "Installing nfs-client-provisioner" + /usr/local/bin/helm install nfs-client {{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz \ + --namespace default --create-namespace \ + --set nfs.server={{ k8s_nfs_server_path.split(':')[0] }} \ + --set nfs.path={{ k8s_nfs_server_path.split(':')[1] }} \ + --set storageClass.defaultClass=true \ + --set storageClass.reclaimPolicy=Retain + echo "Waiting for nfs-subdir-external-provisioner pods to appear..." + sleep 15 + + if kubectl get pods -n default -l app=nfs-subdir-external-provisioner | grep -q nfs-subdir; then + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + else + echo "Pods not yet created, retrying after 10 seconds..." + sleep 10 + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + fi + + echo "Installing MetalLB" + kubectl create -f {{ k8s_client_mount_path }}/metallb/{{ metallb_package }}.yml + echo "Waiting for MetalLB pods to be ready..." + kubectl wait --namespace metallb-system --for=condition=Ready pods --all --timeout=300s + echo "Waiting for MetalLB webhook to be ready..." + until kubectl get endpoints metallb-webhook-service -n metallb-system \ + -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -qE '[0-9]'; do + echo "Webhook endpoints not ready yet. Retrying in 5s..." + sleep 5 + done + echo "MetalLB webhook is ready." + echo "Deploy ipaddress pool" + kubectl create -f {{ k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + echo "Deploy Layer2 Configuration" + kubectl create -f {{ k8s_client_mount_path }}/metallb/l2advertisement.yaml + + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + + CSI_DRIVER_SUPPORT="{{ csi_driver_powerscale_support | lower }}" + echo "===== Checking if PowerScale CSI driver support is enabled =====" + + if [ "$CSI_DRIVER_SUPPORT" != "true" ]; then + echo "PowerScale CSI driver support is disabled. Skipping deployment." + true + else + echo "PowerScale CSI driver support is enabled. Proceeding with deployment." + echo "===== Copying CSI PowerScale driver from NFS-mounted path =====" + mkdir -p /opt/omnia + POWERSCALE_DEPLOYMENT_FAILED=0 + + if cp -rp {{ k8s_client_mount_path }}/csi-driver-powerscale /opt/omnia/; then + echo "Copied CSI PowerScale driver to /opt/omnia successfully." + else + echo "ERROR: Failed to copy PowerScale driver. Skipping PowerScale deployment." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + SECRET_FILE="/opt/omnia/csi-driver-powerscale/secret.yaml" + echo "Checking if creds are provided by user" + + if [[ -f "$SECRET_FILE" ]]; then + echo "Found secret file at $SECRET_FILE" + csi_username=$(grep -v '^[[:space:]]*#' "$SECRET_FILE" | grep 'username:' | head -1 | awk -F':' '{gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' | base64 --decode 2>/dev/null) + csi_password=$(grep -v '^[[:space:]]*#' "$SECRET_FILE" | grep 'password:' | head -1 | awk -F':' '{gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' | base64 --decode 2>/dev/null) + + if [ -z "${csi_username}" ] || [ -z "${csi_password}" ]; then + echo " ERROR: CSI credentials not defined in secret.yaml." + POWERSCALE_DEPLOYMENT_FAILED=1 + else + export csi_username + export csi_password + fi + else + echo "ERROR: secret.yaml not found at $SECRET_FILE." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Checking if PowerScale driver is deployed =====" + if kubectl get pods -n isilon --no-headers 2>/dev/null | grep -q '^isilon-'; then + echo "PowerScale driver is already deployed on the cluster." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -ne 0 ]; then + echo "PowerScale prerequisites failed. Skipping remaining deployment steps." + true + else + echo "===== Ensuring 'isilon' namespace exists =====" + kubectl create namespace isilon --dry-run=client -o yaml | kubectl apply -f - || { + echo "ERROR: Failed to create or verify 'isilon' namespace." + POWERSCALE_DEPLOYMENT_FAILED=1 + } + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Creating and patching isilon-creds secret =====" + kubectl delete secret isilon-creds -n isilon >/dev/null 2>&1 || true + kubectl create secret generic isilon-creds -n isilon \ + --from-file=config="$SECRET_FILE" >/dev/null 2>&1 || POWERSCALE_DEPLOYMENT_FAILED=1 + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Applying empty certificate secret =====" + if [ -f "/opt/omnia/csi-driver-powerscale/empty_isilon-certs.yaml" ]; then + kubectl apply -f /opt/omnia/csi-driver-powerscale/empty_isilon-certs.yaml || { + echo "Failed to apply empty certs secret. Continuing..." + POWERSCALE_DEPLOYMENT_FAILED=1 + } + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Deploying External Snapshotter CRDs =====" + if [ -d "/opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/client/config/crd" ]; then + kubectl apply -f /opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/client/config/crd/ >/dev/null 2>&1 || true + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Deploying Snapshot Controller =====" + if [ -d "/opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/deploy/kubernetes/snapshot-controller" ]; then + kubectl apply -f /opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/deploy/kubernetes/snapshot-controller/ >/dev/null 2>&1 || true + kubectl set image deployment/snapshot-controller snapshot-controller=registry.k8s.io/sig-storage/snapshot-controller:v8.3.0 -n kube-system >/dev/null 2>&1 || true + kubectl rollout status deployment/snapshot-controller -n kube-system --timeout=300s >/dev/null 2>&1 || true + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Running CSI PowerScale installation script =====" + INSTALL_SCRIPT="/opt/omnia/csi-driver-powerscale/csi-powerscale/dell-csi-helm-installer/csi-install.sh" + if [ -x "$INSTALL_SCRIPT" ]; then + cd "$(dirname "$INSTALL_SCRIPT")" || true + ./csi-install.sh --namespace isilon --values /opt/omnia/csi-driver-powerscale/values.yaml & + CSI_PID=$! + wait $CSI_PID + else + echo "ERROR: CSI install script not found." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + fi + fi + fi + systemctl restart nfs-client.target + systemctl restart rpcbind + +{% if hostvars['localhost']['idrac_telemetry_support'] or hostvars['localhost']['ldms_support'] %} + echo "Applying Telemetry Kubernetes deployments" + /root/telemetry.sh +{% endif %} + + echo "Display nodes and pods status after deploying plugins" + echo "Listing all Kubernetes nodes:" + kubectl get nodes -o wide + + echo "Listing all Kubernetes pods in all namespaces:" + kubectl get pods --all-namespaces -o wide + + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP INIT + echo "service_kube_control_plane_first_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 new file mode 100644 index 0000000000..9233925c6d --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 @@ -0,0 +1,418 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host {{ k8s_control_ssh_patterns }} + IdentityFile {{ k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ k8s_nfs_server_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + - path: /tmp/kube-vip.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: vip_interface + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: {{ kube_vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.8.9 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + status: {} + + - path: /usr/local/bin/install-helm.sh + permissions: '0755' + content: | + #!/bin/bash + set -e + ARCH="amd64" + cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + set -e + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.cluster_initialized" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + if [ ! -f "$MARKER" ]; then + # Join RKE2 cluster as additional server + echo "Initial boot - joining RKE2 cluster as additional server for service_kube_control_plane_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable." + + # Write RKE2 config for joining as server + cat > /etc/rancher/rke2/config.yaml </dev/null 2>&1; do + sleep 10 + done + kubectl get nodes -o wide + + systemctl restart nfs-client.target + systemctl restart rpcbind + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP JOIN + echo "service_kube_control_plane_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 new file mode 100644 index 0000000000..98c56efcb8 --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 @@ -0,0 +1,261 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host {{ k8s_control_ssh_patterns }} + IdentityFile {{ k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ k8s_nfs_server_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open required ports for RKE2 agent + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - systemctl restart rpcbind + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - | + set -e + MARKER="/etc/rancher/rke2/.cluster_initialized" + if [ ! -f "$MARKER" ]; then + # Join RKE2 cluster as agent (worker node) + echo "Initial boot - joining RKE2 cluster as agent for service_kube_node_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable. Joining this service_kube_node_x86_64 to the cluster now." + + # Write RKE2 agent config + cat > /etc/rancher/rke2/config.yaml <- + {{ cp_first_cluster | selectattr('type', 'equalto', 'manifest') + | selectattr('package', 'search', 'metallb-native') + | map(attribute='package') | join }} + compute_helm_package: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'helm') + | map(attribute='package') | join }} + compute_nfs_subdir_external_provisioner_pkg: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'nfs-subdir-external-provisioner') + | map(attribute='package') | join }} + +- name: Extract compute RKE2 airgap image tarball package names # noqa: yaml[line-length] + vars: + rke2_cluster: "{{ compute_k8s_packages_json['compute_rke2']['cluster'] }}" + ansible.builtin.set_fact: + compute_rke2_images_core_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-core') + | map(attribute='package') | join }} + compute_rke2_images_cni_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-' ~ compute_k8s_cni) + | map(attribute='package') | join }} + +- name: Copy pulp webserver certificate to target host + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ anchors_path }}" + mode: "{{ file_mode }}" + become: true + +- name: Update CA trust on target host + ansible.builtin.command: update-ca-trust + register: update_ca + changed_when: false + +# metallb +- name: Download metallb-native manifest for compute cluster + ansible.builtin.get_url: + url: "{{ offline_manifest_path }}/{{ compute_metallb_package }}/{{ compute_metallb_package }}.yml" + dest: "{{ compute_k8s_client_mount_path }}/metallb/{{ compute_metallb_package }}.yml" + mode: "{{ file_mode }}" + +# helm +- name: Download helm tarball for compute cluster + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ compute_helm_package }}/{{ compute_helm_package }}.tar.gz" + dest: "{{ compute_k8s_client_mount_path }}/helm/{{ compute_helm_package }}.tar.gz" + mode: "{{ file_mode }}" + +- name: Untar helm tarball repo for compute cluster + ansible.builtin.unarchive: + src: "{{ compute_k8s_client_mount_path }}/helm/{{ compute_helm_package }}.tar.gz" + dest: "{{ compute_k8s_client_mount_path }}/helm/" + remote_src: true + +# nfs client provisioner +- name: Download nfs-client-provisioner tarball for compute cluster + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ compute_nfs_subdir_external_provisioner_pkg }}/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz" + dest: "{{ compute_k8s_client_mount_path }}/nfs-client-provisioner/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz" + mode: "{{ file_mode }}" + +# RKE2 airgap system images (core + selected CNI) +- name: Download RKE2 airgap image tarballs to compute NFS share + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.zst" + dest: "{{ compute_k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" + mode: "{{ file_mode }}" + loop: + - "{{ compute_rke2_images_core_pkg }}" + - "{{ compute_rke2_images_cni_pkg }}" + +- name: Copy pulp webserver certificate to compute client_share_path + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ compute_k8s_client_mount_path }}" + mode: "{{ file_mode }}" + become: true diff --git a/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml new file mode 100644 index 0000000000..490a99fa2d --- /dev/null +++ b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml @@ -0,0 +1,292 @@ +# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +- name: Include variable file omnia_config.yml + ansible.builtin.include_vars: "{{ input_project_dir }}/omnia_config.yml" + +- name: Include storage vars + ansible.builtin.include_vars: "{{ input_project_dir }}/storage_config.yml" + +- name: Include include_high_availability_config vars + ansible.builtin.include_vars: "{{ ha_config_file }}" + +- name: Set facts for service_rke2_k8s_cluster + ansible.builtin.set_fact: + cluster_name: "{{ service_rke2_k8s_cluster[0].cluster_name }}" + k8s_distro: "rke2" + k8s_cni: "{{ service_rke2_k8s_cluster[0].k8s_cni }}" + pod_external_ip_range: "{{ service_rke2_k8s_cluster[0].pod_external_ip_range }}" + k8s_service_addresses: "{{ service_rke2_k8s_cluster[0].k8s_service_addresses }}" + k8s_pod_network_cidr: "{{ service_rke2_k8s_cluster[0].k8s_pod_network_cidr }}" + csi_powerscale_driver_secret_file_path: "{{ service_rke2_k8s_cluster[0].csi_powerscale_driver_secret_file_path }}" + csi_powerscale_driver_values_file_path: "{{ service_rke2_k8s_cluster[0].csi_powerscale_driver_values_file_path }}" + nfs_storage_name: "{{ service_rke2_k8s_cluster[0].nfs_storage_name }}" + +- name: Read the service_k8s mount point + ansible.builtin.set_fact: + k8s_client_mount_path: "{{ (mounts | selectattr('name', 'equalto', nfs_storage_name) | first).mount_point }}" + k8s_nfs_server_ip: "{{ (mounts | selectattr('name', 'equalto', nfs_storage_name) | first).source.split(':')[0] }}" + k8s_nfs_server_path: "{{ (mounts | selectattr('name', 'equalto', nfs_storage_name) | first).source }}" + +- name: Ensure SSH key directory exists on K8s share + ansible.builtin.file: + path: "{{ k8s_client_mount_path }}/ssh" + state: directory + owner: root + group: root + mode: '0700' + +- name: Copy OIM private key to K8s share for node-to-node SSH + ansible.builtin.copy: + src: "{{ ssh_private_key_path }}" + dest: "{{ k8s_client_mount_path }}/ssh/oim_rsa" + owner: root + group: root + mode: '0600' + +- name: Set admin network nic and ip + ansible.builtin.set_fact: + admin_nic_ip: "{{ hostvars['localhost']['admin_nic_ip'] }}" + admin_netmask_bits: "{{ hostvars['localhost']['admin_netmask_bits'] }}" + +- name: Set admin network CIDR + ansible.builtin.set_fact: + admin_nic_cidr: "{{ (admin_nic_ip + '/' + admin_netmask_bits) | ansible.utils.ipaddr('network/prefix') }}" + +- name: Fetch server_ip and server_share_path from list when nfs server is localhost + ansible.builtin.set_fact: + nfs_server_ip: "{{ hostvars['127.0.0.1']['admin_nic_ip'] }}" + when: k8s_nfs_server_ip == "localhost" + +- name: Set HA-related facts + ansible.builtin.set_fact: + enable_k8s_ha: "{{ service_k8s_cluster_ha[0].enable_k8s_ha | default(false) }}" + kube_vip: "{{ service_k8s_cluster_ha[0].virtual_ip_address | default('') }}" + +- name: Set fact for pulp mirror + ansible.builtin.set_fact: + pulp_mirror: "{{ hostvars['localhost']['admin_nic_ip'] }}:2225" + +- name: Set fact for dns, csi driver support and service_k8s_version + ansible.builtin.set_fact: + dns: "{{ hostvars['localhost']['dns'] }}" + csi_driver_powerscale_support: "{{ hostvars['localhost']['csi_driver_powerscale_support'] | string | lower }}" + service_k8s_version: "{{ hostvars['localhost']['service_k8s_version'] }}" + +- name: Create required share directories on NFS for RKE2 + block: + - name: Create subdirectories on NFS share + ansible.builtin.file: + path: "{{ k8s_client_mount_path }}/{{ item }}" + state: directory + mode: "{{ folder_mode }}" + owner: root + group: root + loop: + - metallb + - nfs-client-provisioner + - helm + - rke2 + - rke2/images + + - name: Create subdirectories on NFS share for CSI + ansible.builtin.file: + path: "{{ k8s_client_mount_path }}/{{ item }}" + state: directory + mode: "{{ folder_mode }}" + owner: root + group: root + loop: + - csi-driver-powerscale + when: hostvars['localhost']['csi_driver_powerscale_support'] + + rescue: + - name: Fail with NFS export advice + ansible.builtin.fail: + msg: "{{ nfs_export_help_msg }}" + +- name: Creating the persist folders in nfs share + ansible.builtin.include_tasks: create_node_dir.yml + +# additional packages +- name: Create x86_64 package base directory + ansible.builtin.file: + path: "{{ packages_base_dir_x86_64 }}" + state: directory + mode: '{{ common_mode }}' + +- name: Create aarch64 package base directory + ansible.builtin.file: + path: "{{ packages_base_dir_aarch64 }}" + state: directory + mode: '{{ common_mode }}' + +- name: Create x86_64 package layout directories + ansible.builtin.file: + path: "{{ packages_base_dir_x86_64 }}/{{ item }}" + state: directory + mode: '{{ common_mode }}' + loop: "{{ packages_layout_x86_64 }}" + +- name: Create aarch64 package layout directories + ansible.builtin.file: + path: "{{ packages_base_dir_aarch64 }}/{{ item }}" + state: directory + mode: '{{ common_mode }}' + loop: "{{ packages_layout_aarch64 }}" + +- name: Print copy paths for x86_64 + ansible.builtin.debug: + msg: "{{ print_copy_msg }}" + loop: "{{ offline_path_x86_64 | default([]) }}" + +- name: Print copy paths for aarch64 + ansible.builtin.debug: + msg: "{{ print_copy_msg }}" + loop: "{{ offline_path_aarch64 | default([]) }}" + +- name: Check x86_64 offline package sources + ansible.builtin.stat: + path: "{{ item.source_path }}" + loop: "{{ offline_path_x86_64 | default([]) }}" + register: x86_64_offline_pkg_sources + +- name: Check aarch64 offline package sources + ansible.builtin.stat: + path: "{{ item.source_path }}" + loop: "{{ offline_path_aarch64 | default([]) }}" + register: aarch64_offline_pkg_sources + +- name: Copy x86_64 offline packages + ansible.builtin.copy: + src: "{{ item.item.source_path }}/" + dest: "{{ item.item.dest_path }}/" + remote_src: true + mode: preserve + loop: "{{ x86_64_offline_pkg_sources.results | default([]) }}" + when: + - item.stat.exists + - item.item.source_path | length > 0 + - item.item.dest_path | length > 0 + +- name: Copy aarch64 offline packages + ansible.builtin.copy: + src: "{{ item.item.source_path }}/" + dest: "{{ item.item.dest_path }}/" + remote_src: true + mode: preserve + loop: "{{ aarch64_offline_pkg_sources.results | default([]) }}" + when: + - item.stat.exists + - item.item.source_path | length > 0 + - item.item.dest_path | length > 0 + +- name: Include local repo access variable file + ansible.builtin.include_vars: "{{ local_repo_access_config_file }}" + +- name: Load service_rke2.json + ansible.builtin.set_fact: + k8s_packages_json: "{{ lookup('file', rke2_packages_file) | from_json }}" + +- name: Extract and set facts for tarball URLs (RKE2) # noqa: yaml[line-length] + vars: + cp_first_cluster: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] }}" + ansible.builtin.set_fact: + metallb_package: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'manifest') + | selectattr('package', 'search', 'metallb-native') + | map(attribute='package') | join }} + helm_package: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'helm') + | map(attribute='package') | join }} + nfs_subdir_external_provisioner_pkg: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'nfs-subdir-external-provisioner') + | map(attribute='package') | join }} + +- name: Extract RKE2 airgap image tarball package names # noqa: yaml[line-length] + vars: + rke2_cluster: "{{ k8s_packages_json['service_rke2']['cluster'] }}" + ansible.builtin.set_fact: + rke2_images_core_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-core') + | map(attribute='package') | join }} + rke2_images_cni_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-' ~ k8s_cni) + | map(attribute='package') | join }} + +- name: Copy pulp webserver certificate to target host + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ anchors_path }}" + mode: "{{ file_mode }}" + become: true + +- name: Update CA trust on target host + ansible.builtin.command: update-ca-trust + register: update_ca + changed_when: false + +# metallb +- name: Download metallb-native manifest + ansible.builtin.get_url: + url: "{{ metallb_manifest_yaml_url }}" + dest: "{{ k8s_client_mount_path }}/metallb/{{ metallb_package }}.yml" + mode: "{{ file_mode }}" + +# helm +- name: Download helm tarball + ansible.builtin.get_url: + url: "{{ helm_tarball_url }}" + dest: "{{ k8s_client_mount_path }}/helm/{{ helm_package }}.tar.gz" + mode: "{{ file_mode }}" + +- name: Untar helm tarball repo + ansible.builtin.unarchive: + src: "{{ k8s_client_mount_path }}/helm/{{ helm_package }}.tar.gz" + dest: "{{ k8s_client_mount_path }}/helm/" + remote_src: true + +# nfs client provisioner +- name: Download nfs-client-provisioner tarball + ansible.builtin.get_url: + url: "{{ nfs_client_provisioner_tarball_url }}" + dest: "{{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz" + mode: "{{ file_mode }}" + +# RKE2 airgap system images (core + selected CNI) +# RKE2 auto-imports any *.tar.zst found in /var/lib/rancher/rke2/agent/images/ +# on the nodes; the cloud-init templates copy these from the NFS share. +- name: Download RKE2 airgap image tarballs to NFS share + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.zst" + dest: "{{ k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" + mode: "{{ file_mode }}" + loop: + - "{{ rke2_images_core_pkg }}" + - "{{ rke2_images_cni_pkg }}" + +- name: Copy pulp webserver certificate to client_share_path + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ k8s_client_mount_path }}" + mode: "{{ file_mode }}" + become: true + +- name: Include PowerScale CSI dependency tasks + ansible.builtin.include_tasks: get_powerscale_dependencies.yml + when: hostvars['localhost']['csi_driver_powerscale_support'] | bool diff --git a/provision/roles/k8s_config/tasks/main.yml b/provision/roles/k8s_config/tasks/main.yml index 2595c99080..b342df0007 100644 --- a/provision/roles/k8s_config/tasks/main.yml +++ b/provision/roles/k8s_config/tasks/main.yml @@ -13,6 +13,24 @@ # limitations under the License. --- -- name: Creating the configuration files required for service_k8s in nfs share - ansible.builtin.include_tasks: create_k8s_config_nfs.yml +- name: Set k8s_distro based on service_rke2_support + ansible.builtin.set_fact: + k8s_distro: "{{ 'rke2' if (hostvars['localhost']['service_rke2_support'] | default(false) | bool) else 'kubeadm' }}" when: hostvars['localhost']['service_k8s_support'] + +- name: Creating the configuration files required for kubeadm service_k8s in nfs share + ansible.builtin.include_tasks: create_k8s_config_nfs.yml + when: + - hostvars['localhost']['service_k8s_support'] + - k8s_distro | default('kubeadm') == 'kubeadm' + +- name: Creating the configuration files required for RKE2 service_k8s in nfs share + ansible.builtin.include_tasks: create_rke2_config_nfs.yml + when: + - hostvars['localhost']['service_k8s_support'] + - k8s_distro | default('kubeadm') == 'rke2' + +- name: Creating the configuration files required for compute RKE2 cluster in nfs share + ansible.builtin.include_tasks: create_compute_rke2_config_nfs.yml + when: + - hostvars['localhost']['compute_rke2_support'] | default(false) | bool diff --git a/provision/roles/k8s_config/vars/main.yml b/provision/roles/k8s_config/vars/main.yml index c949564d70..875ea84430 100644 --- a/provision/roles/k8s_config/vars/main.yml +++ b/provision/roles/k8s_config/vars/main.yml @@ -17,6 +17,8 @@ local_repo_access_config_file: "/opt/omnia/provision/local_repo_access.yml" input_project_dir: "{{ hostvars['localhost']['input_project_dir'] }}" # Versioned JSON file: service_k8s_v.json (e.g., service_k8s_v1.35.1.json) k8s_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/service_k8s_v{{ hostvars['localhost']['service_k8s_version'] }}.json" # noqa: yaml[line-length] +rke2_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/service_rke2.json" # noqa: yaml[line-length] +compute_rke2_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/compute_rke2.json" # noqa: yaml[line-length] calico_manifest_yaml_url: "{{ offline_manifest_path }}/{{ calico_package }}/{{ calico_package }}.yml" csi_powerscale_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/csi_driver_powerscale.json" # noqa: yaml[line-length] metallb_manifest_yaml_url: "{{ offline_manifest_path }}/{{ metallb_package }}/{{ metallb_package }}.yml" @@ -87,3 +89,4 @@ print_copy_msg: "Copying {{ item.name }} from {{ item.source_path }} to {{ item. offline_path_x86_64: [] offline_path_aarch64: [] ssh_private_key_path: /root/.ssh/oim_rsa +common_mode: "0755" diff --git a/provision/roles/provision_validations/tasks/include_software_config.yml b/provision/roles/provision_validations/tasks/include_software_config.yml index 2895762f8e..dc53ea2bf4 100644 --- a/provision/roles/provision_validations/tasks/include_software_config.yml +++ b/provision/roles/provision_validations/tasks/include_software_config.yml @@ -53,15 +53,35 @@ - name: Initialise variables ansible.builtin.set_fact: service_k8s_support: false + service_rke2_support: false + compute_rke2_support: false - name: Check if service k8s support is true ansible.builtin.set_fact: service_k8s_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | list | length > 0 }}" +- name: Check if service rke2 support is true + ansible.builtin.set_fact: + service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}" + +- name: Check if compute rke2 support is true + ansible.builtin.set_fact: + compute_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'compute_rke2') | list | length > 0 }}" + +- name: Set service_k8s_support true when rke2 is selected + ansible.builtin.set_fact: + service_k8s_support: true + when: service_rke2_support | bool + - name: Extract k8s version ansible.builtin.set_fact: service_k8s_version: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | map(attribute='version') | first }}" - when: service_k8s_support + when: service_k8s_support and not (service_rke2_support | bool) + +- name: Extract rke2 version + ansible.builtin.set_fact: + service_k8s_version: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | map(attribute='version') | first }}" + when: service_rke2_support | bool - name: Check if csi support is true ansible.builtin.set_fact: