Add prgenv-gnu-openmpi with OpenMPI

config.yaml

@@ -307,6 +307,15 @@ uenvs:
         daint: [gh200]
         santis: [gh200]
         eiger: [zen2]
+    "25.11":
+      recipes:
+        zen2: 25.11/mc
+        zen3: 25.11/mc
+        gh200: 25.11/gh200
+      deploy:
+        clariden: [gh200]
+        daint: [gh200]
+        santis: [gh200]
     "25.7":
       recipes:
         mi200: 25.7/amdgpu
@@ -324,6 +333,17 @@ uenvs:
         santis: [gh200]
         bristen: [a100]
         eiger: [zen2]
+  prgenv-gnu-openmpi:
+    "25.11":
+      recipes:
+        # zen2: 25.11/mc
+        # zen3: 25.11/mc
+        gh200: 25.11/gh200
+      deploy:
+        clariden: [gh200]
+        daint: [gh200]
+        santis: [gh200]
+        # eiger: [zen2]
   prgenv-nvfortran:
     "24.11":
       recipes:

recipes/prgenv-gnu-openmpi/25.11/gh200/compilers.yaml

@@ -0,0 +1,2 @@
+gcc:
+  version: "14.2"

recipes/prgenv-gnu-openmpi/25.11/gh200/config.yaml

@@ -0,0 +1,10 @@
+name: prgenv-gnu
+spack:
+  repo: https://github.com/spack/spack.git
+  commit: releases/v1.1
+  packages:
+    repo: https://github.com/spack/spack-packages.git
+    commit: a896fdbe5d01981cbc6f9b5139a5d551ac2fe248 # develop on 2025-11-21
+store: /user-environment
+description: GNU Compiler toolchain with OpenMPI, Python, CMake and other development tools.
+version: 2

recipes/prgenv-gnu-openmpi/25.11/gh200/environments.yaml

@@ -0,0 +1,51 @@
+gcc-env:
+  compiler: [gcc]
+  network:
+      mpi: [email protected]
+      specs:
+      - [email protected] +gdrcopy fabrics=lnx,shm,cxi,xpmem # TODO: Add to/update alps-cluster-config
+  unify: true
+  specs:
+  - boost +chrono +filesystem +iostreams +mpi +python +regex +serialization +shared +system +timer
+  - cmake
+  - fftw
+  - fmt
+  - gmp
+  - gsl
+  - hdf5+cxx+hl+fortran
+  - kokkos +aggressive_vectorization ~alloc_async cuda_arch=90 +cuda_constexpr +cuda_lambda ~cuda_relocatable_device_code ~cuda_uvm cxxstd=17 +openmp +pic +serial +shared +tuning +wrapper
+  - kokkos-kernels +blas +cublas +cusparse +cusolver +execspace_cuda +execspace_openmp +execspace_serial +lapack +memspace_cudaspace +openmp scalars=float,double,complex_float,complex_double +serial +shared +superlu
+  - kokkos-tools +mpi +papi
+  - netlib-scalapack
+  - lua
+  - libtree
+  - lz4
+  - meson
+  - netcdf-c
+  - netcdf-cxx
+  - netcdf-cxx4
+  - netcdf-fortran
+  - ninja
+  - openblas threads=openmp
+  - osu-micro-benchmarks
+  - papi
+  - python
+  - zlib-ng
+  # add GPU-specific packages here, for easier comparison with mc version
+  - nccl
+  - nccl-tests
+  - cuda@12
+  - xcb-util-cursor
+  - aws-ofi-nccl
+  - superlu
+  variants:
+  - +mpi
+  - +cuda
+  - cuda_arch=90a
+  views:
+    default:
+      link: roots
+      uenv:
+        add_compilers: true
+        prefix_paths:
+          LD_LIBRARY_PATH: [lib, lib64]

recipes/prgenv-gnu-openmpi/25.11/gh200/extra/reframe.yaml

@@ -0,0 +1,16 @@
+default:
+  features:
+    - cuda
+    - mpi
+    - openmpi
+    - nccl
+    - nccl-tests
+    - openmp
+    - osu-micro-benchmarks
+    - prgenv
+    - serial
+  cc: mpicc
+  cxx: mpic++
+  ftn: mpifort
+  views:
+    - default

recipes/prgenv-gnu-openmpi/25.11/gh200/modules.yaml

@@ -0,0 +1,23 @@
+modules:
+  # Paths to check when creating modules for all module sets
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    # Where to install modules
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: []
+      projections:
+        all: '{name}/{version}'

recipes/prgenv-gnu-openmpi/25.11/gh200/repo

@@ -0,0 +1 @@
+../repo/

recipes/prgenv-gnu-openmpi/25.11/repo/packages/libfabric/issue-11231-cuda-sync.patch

@@ -0,0 +1,20 @@
+diff --git a/prov/lnx/src/lnx_ops.c b/prov/lnx/src/lnx_ops.c
+index ba3f097..2d6c187 100644
+--- a/prov/lnx/src/lnx_ops.c
++++ b/prov/lnx/src/lnx_ops.c
+@@ -455,6 +455,7 @@ ssize_t lnx_trecv(struct fid_ep *ep, void *buf, size_t len, void *desc,
+ 	struct lnx_ep *lep;
+ 	const struct iovec iov = {.iov_base = buf, .iov_len = len};
+
++	cuda_set_sync_memops(buf);
+ 	lep = container_of(ep, struct lnx_ep, le_ep.ep_fid.fid);
+ 	if (!lep)
+ 		return -FI_ENOSYS;
+@@ -666,6 +667,7 @@ ssize_t lnx_tsenddata(struct fid_ep *ep, const void *buf, size_t len, void *desc
+ 	fi_addr_t core_addr;
+ 	void *core_desc = desc;
+
++	cuda_set_sync_memops(buf);
+ 	lep = container_of(ep, struct lnx_ep, le_ep.ep_fid.fid);
+ 	if (!lep)
+ 		return -FI_ENOSYS;

recipes/prgenv-gnu-openmpi/25.11/repo/packages/libfabric/package.py

@@ -0,0 +1,19 @@
+from spack_repo.builtin.packages.libfabric.package import Libfabric as BuiltinLibfabric
+
+from spack.package import *
+
+class Libfabric(BuiltinLibfabric):
+    # This patches missing synchronization for GPU-GPU transfers in the lnx
+    # provider of libfabric. The patch is from from a comment on the
+    # corresponding issue:
+    # https://github.com/ofiwg/libfabric/issues/11231#issue-3252163450.
+    #
+    # It's unclear if this is a good patch, but it's sufficient for testing of
+    # the lnx provider. If and when the correct fix is published the patch can
+    # be backported on the upstream libfabric package.
+    #
+    # The patch may not apply for all versions (tested with 2.3.1), but there
+    # is no version constraint as the patch is essential. Builds should fail if
+    # the patch doesn't apply.
+    # patch("issue-11231-cuda-sync.patch", when="fabrics=lnx")
+    pass