From cf5ad17185ee2038ff05ea150ff017763f80c129 Mon Sep 17 00:00:00 2001
From: Tom Lin <tom91136@gmail.com>
Date: Tue, 16 Apr 2024 17:17:20 +0100
Subject: [PATCH 1/4] Add support for compiling CUDA with clang++

---
 src/ci-test-compile.sh |  5 +++++
 src/cuda/model.cmake   | 24 +++++++++++++++++-------
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/ci-test-compile.sh b/src/ci-test-compile.sh
index 249bab4f..af301ff0 100755
--- a/src/ci-test-compile.sh
+++ b/src/ci-test-compile.sh
@@ -234,6 +234,11 @@ build_clang() {
   else
     echo "Skipping RAJA models due to CMake version requirement"
   fi
+
+  run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON"
+  run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=MANAGED"
+  run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=PAGEFAULT"
+
   run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${NVHPC_NVCC:?} -DCUDA_ARCH=$NV_ARCH"
   run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${NVHPC_NVCC:?} -DCUDA_ARCH=$NV_ARCH -DMEM=MANAGED"
   run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${NVHPC_NVCC:?} -DCUDA_ARCH=$NV_ARCH -DMEM=PAGEFAULT"
diff --git a/src/cuda/model.cmake b/src/cuda/model.cmake
index 7c1b0d6e..ad8b9bc8 100644
--- a/src/cuda/model.cmake
+++ b/src/cuda/model.cmake
@@ -16,29 +16,39 @@ register_flag_required(CMAKE_CUDA_COMPILER
 register_flag_required(CUDA_ARCH
         "Nvidia architecture, will be passed in via `-arch=` (e.g `sm_70`) for nvcc")
 
+register_flag_optional(CUDA_CLANG_DRIVER
+        "Disable any nvcc-specific flags so that setting CMAKE_CUDA_COMPILER to clang++ can compile successfully"
+        "OFF")
+
 register_flag_optional(CUDA_EXTRA_FLAGS
-        "Additional CUDA flags passed to nvcc, this is appended after `CUDA_ARCH`"
+        "Additional CUDA flags passed to the CUDA compiler, this is appended after `CUDA_ARCH`"
         "")
 
 
 macro(setup)
 
     # XXX CMake 3.18 supports CMAKE_CUDA_ARCHITECTURES/CUDA_ARCHITECTURES but we support older CMakes
-    if(POLICY CMP0104)
+    if (POLICY CMP0104)
         cmake_policy(SET CMP0104 OLD)
-    endif()
+    endif ()
 
-    enable_language(CUDA)
     register_definitions(${MEM})
 
-    # add -forward-unknown-to-host-compiler for compatibility reasons
-    set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} "-forward-unknown-to-host-compiler" "-arch=${CUDA_ARCH}" ${CUDA_EXTRA_FLAGS})
+    if (CUDA_CLANG_DRIVER)
+        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -std=c++17 --cuda-gpu-arch=${CUDA_ARCH} ${CUDA_EXTRA_FLAGS}")
+    else ()
+        # add -forward-unknown-to-host-compiler for compatibility reasons
+        # add -std=c++17 manually as older CMake seems to omit this (source gets treated as C otherwise)
+        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -std=c++17 -forward-unknown-to-host-compiler -arch=${CUDA_ARCH} ${CUDA_EXTRA_FLAGS}")
+    endif ()
     string(REPLACE ";" " " CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}")
 
+    enable_language(CUDA)
+
     # CMake defaults to -O2 for CUDA at Release, let's wipe that and use the global RELEASE_FLAG
     # appended later
     wipe_gcc_style_optimisation_flags(CMAKE_CUDA_FLAGS_${BUILD_TYPE})
 
-    message(STATUS "NVCC flags: ${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${BUILD_TYPE}}")
+    message(STATUS "CUDA compiler flags: ${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${BUILD_TYPE}}")
 endmacro()
 

From fdba8f0266da8bdeb57b12a5fa6db6857e91eed2 Mon Sep 17 00:00:00 2001
From: Tom Lin <tom91136@gmail.com>
Date: Tue, 16 Apr 2024 17:25:38 +0100
Subject: [PATCH 2/4] Validate CMake version for CUDA on clang

---
 src/ci-test-compile.sh | 10 +++++++---
 src/cuda/model.cmake   |  3 +++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/ci-test-compile.sh b/src/ci-test-compile.sh
index af301ff0..dc27a514 100755
--- a/src/ci-test-compile.sh
+++ b/src/ci-test-compile.sh
@@ -235,9 +235,13 @@ build_clang() {
     echo "Skipping RAJA models due to CMake version requirement"
   fi
 
-  run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON"
-  run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=MANAGED"
-  run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=PAGEFAULT"
+  if check_cmake_ver "3.18.0"; then
+    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON"
+    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=MANAGED"
+    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=PAGEFAULT"
+  else
+    echo "Skipping CUDA on clang models due to CMake version requirement"
+  fi
 
   run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${NVHPC_NVCC:?} -DCUDA_ARCH=$NV_ARCH"
   run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${NVHPC_NVCC:?} -DCUDA_ARCH=$NV_ARCH -DMEM=MANAGED"
diff --git a/src/cuda/model.cmake b/src/cuda/model.cmake
index ad8b9bc8..319faa47 100644
--- a/src/cuda/model.cmake
+++ b/src/cuda/model.cmake
@@ -35,6 +35,9 @@ macro(setup)
     register_definitions(${MEM})
 
     if (CUDA_CLANG_DRIVER)
+        if (CMAKE_VERSION VERSION_LESS "3.18.0")
+            message(FATAL_ERROR "Using clang driver for CUDA is only supported for CMake >= 3.18")
+        endif ()
         set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -std=c++17 --cuda-gpu-arch=${CUDA_ARCH} ${CUDA_EXTRA_FLAGS}")
     else ()
         # add -forward-unknown-to-host-compiler for compatibility reasons

From f7cdb7b1fcc6d0f88045a0f39b20badf11e706d8 Mon Sep 17 00:00:00 2001
From: Tom Lin <tom91136@gmail.com>
Date: Tue, 16 Apr 2024 18:29:00 +0100
Subject: [PATCH 3/4] Set CUDAToolkit_ROOT for CUDA on clang

---
 src/ci-test-compile.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ci-test-compile.sh b/src/ci-test-compile.sh
index dc27a514..b4eaf399 100755
--- a/src/ci-test-compile.sh
+++ b/src/ci-test-compile.sh
@@ -236,9 +236,9 @@ build_clang() {
   fi
 
   if check_cmake_ver "3.18.0"; then
-    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON"
-    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=MANAGED"
-    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDA_CLANG_DRIVER=ON -DMEM=PAGEFAULT"
+    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON"
+    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON -DMEM=MANAGED"
+    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON -DMEM=PAGEFAULT"
   else
     echo "Skipping CUDA on clang models due to CMake version requirement"
   fi

From 248c03bb02ccc94400d9d1bc7581178bc8a4a240 Mon Sep 17 00:00:00 2001
From: Tom Lin <tom91136@gmail.com>
Date: Tue, 16 Apr 2024 23:53:04 +0100
Subject: [PATCH 4/4] Let clang detect CUDA correctly

---
 src/ci-test-compile.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/ci-test-compile.sh b/src/ci-test-compile.sh
index b4eaf399..ae841ebe 100755
--- a/src/ci-test-compile.sh
+++ b/src/ci-test-compile.sh
@@ -236,9 +236,14 @@ build_clang() {
   fi
 
   if check_cmake_ver "3.18.0"; then
-    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON"
-    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON -DMEM=MANAGED"
-    run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON -DMEM=PAGEFAULT"
+    # make sure ptxas is on PATH so that clang can detect CUDA
+    (
+      export PATH="${NVHPC_CUDA_DIR}/bin:${PATH:?}"
+      ${CLANG_CXX:?} -v
+      run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON"
+      run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON -DMEM=MANAGED"
+      run_build $name "${CLANG_CXX:?}" cuda "$cxx -DCMAKE_CUDA_COMPILER=${CLANG_CXX:?} -DCUDA_ARCH=$NV_ARCH -DCUDAToolkit_ROOT=${NVHPC_CUDA_DIR:?} -DCUDA_CLANG_DRIVER=ON -DMEM=PAGEFAULT"
+    )
   else
     echo "Skipping CUDA on clang models due to CMake version requirement"
   fi