Merge pull request #3642 from abbas1902/release

Add startup script to set OFI env vars for Cloud RDMA workloads

Author: abbas1902

community/examples/hpc-slurm-h4d.yaml

@@ -76,6 +76,11 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       install_cloud_rdma_drivers: true
+      set_ofi_cloud_rdma_tunables: true
+      local_ssd_filesystem:
+        fs_type: ext4
+        mountpoint: /mnt/lssd
+        permissions: "1777"
 
   - id: h4d_nodeset
     source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
@@ -123,8 +128,14 @@ deployment_groups:
       machine_type: n2-standard-4
       enable_login_public_ips: true
 
+  - id: slurm_controller_startup
+    source: modules/scripts/startup-script
+    settings:
+      set_ofi_cloud_rdma_tunables: true
+
   - id: slurm_controller
     source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
     use: [h4d-slurm-net-0, h4d_partition, slurm_login, homefs, appsfs]
     settings:
       enable_controller_public_ips: true
+      controller_startup_script: $(slurm_controller_startup.startup_script)

examples/h4d-vm.yaml

@@ -73,6 +73,12 @@ deployment_groups:
       configure_ssh_host_patterns:
       - $(vars.hostname_prefix)-*
       install_cloud_rdma_drivers: true
+      set_ofi_cloud_rdma_tunables: true
+      local_ssd_filesystem:
+        fs_type: ext4
+        mountpoint: /mnt/lssd
+        permissions: "1777"
+
 
   - id: h4d-vms
     source: modules/compute/vm-instance

modules/scripts/startup-script/README.md

@@ -334,6 +334,7 @@ No modules.
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The region to deploy to | `string` | n/a | yes |
 | <a name="input_runners"></a> [runners](#input\_runners) | List of runners to run on remote VM.<br/>    Runners can be of type ansible-local, shell or data.<br/>    A runner must specify one of 'source' or 'content'.<br/>    All runners must specify 'destination'. If 'destination' does not include a<br/>    path, it will be copied in a temporary folder and deleted after running.<br/>    Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no |
+| <a name="input_set_ofi_cloud_rdma_tunables"></a> [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no |
 
 ## Outputs
 

modules/scripts/startup-script/main.tf

@@ -93,6 +93,20 @@ locals {
     }
   ]
 
+  ofi_runner = var.set_ofi_cloud_rdma_tunables == "" ? [] : [
+    {
+      type        = "data"
+      destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh"
+      content     = <<-EOT
+        #!/bin/bash
+        export FI_PROVIDER="verbs;ofi_rxm"
+        export FI_OFI_RXM_USE_RNDV_WRITE=1
+        export FI_VERBS_INLINE_SIZE=39
+        export I_MPI_FABRICS="shm:rxm"
+        EOT
+    },
+  ]
+
   rdma_runner = !var.install_cloud_rdma_drivers ? [] : [
     {
       type        = "shell"
@@ -153,6 +167,7 @@ locals {
     local.warnings,
     local.hotfix_runner,
     local.proxy_runner,
+    local.ofi_runner,
     local.rdma_runner,
     local.monitoring_agent_installer,
     local.ansible_installer,

modules/scripts/startup-script/variables.tf

@@ -268,3 +268,9 @@ variable "install_cloud_rdma_drivers" {
   type        = bool
   default     = false
 }
+
+variable "set_ofi_cloud_rdma_tunables" {
+  description = "Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads."
+  type        = bool
+  default     = false
+}