Merge pull request #169 from UoB-HPC/develop

Merge develop for v5.0

Author: tomdeakin

.github/workflows/main.yaml

@@ -12,12 +12,12 @@ on:
 jobs:
 
   test-rust:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: ./src/rust/rust-stream
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup project
         run: rustup install nightly
       - name: Compile project
@@ -28,25 +28,25 @@ jobs:
         run: ./target/release/rust-stream --arraysize 2048
 
   test-java:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: ./src/java/java-stream
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Test build project
         run: ./mvnw clean package
       - name: Test run
         if: ${{ ! cancelled() }}
         run: java -jar target/java-stream.jar --arraysize 2048
 
   test-julia:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: ./src/julia/JuliaStream.jl
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup project
         run: julia             --project -e 'import Pkg; Pkg.instantiate()'
       - name: Test run PlainStream.jl
@@ -70,14 +70,22 @@ jobs:
 
 
   test-cpp:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@v8
+        with:
+          root-reserve-mb: 8192
+          swap-size-mb: 512
+          remove-android: 'true'
+          remove-codeql: 'true'
+
+      - uses: actions/checkout@v4
 
       - name: Cache compiler
         if: ${{ !env.ACT }}
         id: prepare-compilers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ./compilers
           key: ${{ runner.os }}-${{ hashFiles('./src/ci-prepare-bionic.sh') }}
@@ -90,9 +98,9 @@ jobs:
         run: source ./src/ci-prepare-bionic.sh ./compilers VARS false || true
 
         # Enable tmate debugging of manually-triggered workflows if the input option was provided
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
-        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+      #      - name: Setup tmate session
+      #        uses: mxschmitt/action-tmate@v3
+      #        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
 
       - name: Test compile gcc     @ CMake 3.13
         if: ${{ ! cancelled() }}
@@ -167,4 +175,65 @@ jobs:
         run: ./src/ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_18_BIN }}
       - name: Test compile hipsycl @ CMake 3.18
         if: ${{ ! cancelled() }}
-        run: ./src/ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_18_BIN }}
+        run: ./src/ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_18_BIN }}
+
+      - name: Test compile gcc     @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_20_BIN }}
+      - name: Test compile clang   @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_20_BIN }}
+      - name: Test compile nvhpc   @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_20_BIN }}
+      - name: Test compile aocc    @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_20_BIN }}
+      - name: Test compile aomp    @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_20_BIN }}
+      - name: Test compile hip     @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_20_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_20_BIN }}
+      - name: Test compile hipsycl @ CMake 3.20
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_20_BIN }}
+
+      - name: Test compile gcc     @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_24_BIN }}
+      - name: Test compile clang   @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_24_BIN }}
+      - name: Test compile nvhpc   @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_24_BIN }}
+      - name: Test compile aocc    @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_24_BIN }}
+      - name: Test compile aomp    @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_24_BIN }}
+      - name: Test compile hip     @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_24_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_24_BIN }}
+      - name: Test compile hipsycl @ CMake 3.24
+        if: ${{ ! cancelled() }}
+        run: ./src/ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_24_BIN }}
+
+  test-futhark:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prepare Futhark compiler
+        uses: diku-dk/install-futhark@HEAD
+        with:
+          version: 'latest'
+      - run: cmake -Bbuild -H. -DMODEL=futhark -DFUTHARK_BACKEND=multicore
+      - run: cmake --build build

.gitignore

@@ -10,12 +10,18 @@ sycl-stream
 hip-stream
 tbb-stream
 
+src/fortran/BabelStream
+src/fortran/BabelStream.*
+
 *.o
 *.bc
 *.sycl
 *.tar
 *.gz
 *.a
+*.mod
+*.cub
+*.ptx
 
 KokkosCore_config.*
 

CHANGELOG.md

@@ -2,8 +2,32 @@
 All notable changes to this project will be documented in this file.
 
 ## Unreleased
+### Added
+- Ability to build Kokkos and RAJA versions against existing packages.
+- Thrust managed memory.
+- HIP managed memory.
+- New implementation using SYCL2020 USM (sycl2020-acc) and renamed original `sycl2020` to `sycl2020-acc`.
+- New implementation in Fortran
+- New implementation in [Futhark](https://futhark-lang.org/)
+- Data initialisation and read-back timing for all models, including Java, Scala, Julia, and Rust
+- Add support for the latest Aparapi (3.0.0) and TornadoVM (0.15.x) for Java
+- JuliaStream.jl published to registry (pending #113)
+
 ### Changed
+- Fix std-data/std-indices compatibility with oneDPL, NVHPC, and AdaptiveCpp (a.k.a. hipSYCL).
 - RAJA CUDA CMake build issues resolved.
+- Kokkos build updates (CXX version upgraded to C++17).
+- Fix CUDA memory limit check.
+- Fix CUDA CMake options for `-DMEM` and `-DCMAKE_CUDA_FLAGS`.
+- Use long double for `check_solution` in case of large problem size.
+- OneAPI DPCPP compiler is deprecated in favour of ICPX, so added new build option to SYCL 2020 version.
+- Updates to the HIP kernels and API usage.
+- Number of thread-blocks in CUDA dot kernel implementation changed to 1024.
+- Fix compatibility of `sycl2020` (now `sycl2020-acc`) with AdaptiveCpp. 
+- Bumped Julia compat to 1.9
+- Bumped Scala to 3.3.1
+- Bumped Rust to 1.74.0-nightly (13e6f24b9 2023-09-23)
+- Upgrade CI to Ubuntu 22.04
 
 ## [v4.0] - 2021-12-22
 

CMakeLists.txt

@@ -1,6 +1,10 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 
-project(BabelStream VERSION 4.0 LANGUAGES CXX)
+if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
+    cmake_policy(SET CMP0135 NEW)
+endif ()
+
+project(BabelStream VERSION 5.0 LANGUAGES CXX C)
 
 # uncomment for debugging build issues:
 #set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -27,8 +31,6 @@ endmacro()
 # the final executable name
 set(EXE_NAME babelstream)
 
-# select default build type
-set(CMAKE_BUILD_TYPE "Release")
 # for chrono and some basic CXX features, models can overwrite this if required
 set(CMAKE_CXX_STANDARD 11)
 
@@ -71,6 +73,75 @@ hint_flag(CXX_EXTRA_LINKER_FLAGS "
 # Honor user's CXX_EXTRA_LINK_FLAGS
 set(CXX_EXTRA_LINK_FLAGS ${CXX_EXTRA_FLAGS} ${CXX_EXTRA_LINK_FLAGS})
 
+option(USE_TBB "Enable the oneTBB library for *supported* models. Enabling this on models that
+                don't explicitly link against TBB is a no-op, see description of your selected
+                model on how this is used." OFF)
+
+option(FETCH_TBB "Fetch (download) the oneTBB library for *supported* models. This uses CMake's
+                  FetchContent feature. Specify version by setting FETCH_TBB_VERSION" OFF)
+set(FETCH_TBB_VERSION "v2021.10.0" CACHE STRING "Specify version of oneTBB to use if FETCH_TBB is ON")
+
+if (FETCH_TBB)
+    FetchContent_Declare(
+            TBB
+            GIT_REPOSITORY https://github.com/oneapi-src/oneTBB.git
+            GIT_TAG "${FETCH_TBB_VERSION}"
+    )
+    # Don't fail builds on waring (TBB has -Wall while not being free of warnings from unused symbols...)
+    set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
+    set(TBB_STRICT OFF)
+    # Not using FetchContent_MakeAvailable (CMake>= 3.14) because we need EXCLUDE_FROM_ALL
+    FetchContent_GetProperties(TBB)
+    if (NOT TBB_POPULATED)
+        FetchContent_Populate(TBB)
+        add_subdirectory(${tbb_SOURCE_DIR} ${tbb_BINARY_DIR} EXCLUDE_FROM_ALL)
+    endif ()
+endif ()
+
+option(USE_ONEDPL "Enable the oneDPL library for *supported* models. Enabling this on models that
+                   don't explicitly link against DPL is a no-op, see description of your selected
+                   model on how this is used." OFF)
+
+option(FETCH_ONEDPL "Fetch (download) the oneDPL library for *supported* models. This uses CMake's
+                  FetchContent feature. Specify version by setting FETCH_ONEDPL_VERSION" OFF)
+set(FETCH_ONEDPL_VERSION "oneDPL-2022.2.0-rc1" CACHE STRING "Specify version of oneTBB to use if FETCH_ONEDPL is ON")
+
+if (FETCH_ONEDPL)
+    FetchContent_Declare(
+            oneDPL
+            GIT_REPOSITORY https://github.com/oneapi-src/oneDPL.git
+            GIT_TAG "${FETCH_ONEDPL_VERSION}"
+    )
+    string(TOLOWER ${USE_ONEDPL} ONEDPL_BACKEND)
+    # XXX oneDPL looks for omp instead of openmp, which mismatches(!) with ONEDPL_PAR_BACKEND if using find_package
+    if (ONEDPL_BACKEND STREQUAL "openmp")
+        set(ONEDPL_BACKEND omp)
+    endif ()
+    # Not using FetchContent_MakeAvailable (CMake>= 3.14) because we need EXCLUDE_FROM_ALL
+    FetchContent_GetProperties(oneDPL)
+    if (NOT oneDPL_POPULATED)
+        FetchContent_Populate(oneDPL)
+        if (USE_TBB)
+            macro(find_package NAME)
+                if ("${NAME}" STREQUAL "TBB")
+                    message(STATUS "Discarding oneDPL's call to find_package(${NAME} ${ARGN})")
+                else ()
+                    _find_package(${NAME} ${ARGN})
+                endif ()
+            endmacro()
+        endif ()
+        add_subdirectory(${onedpl_SOURCE_DIR} ${onedpl_BINARY_DIR} EXCLUDE_FROM_ALL)
+
+        # Fixup oneDPL's omission on setting DPCPP definitions.
+        # We do this after the creation of the oneDPL target.
+        if (ONEDPL_BACKEND MATCHES "^(dpcpp|dpcpp_only)$")
+            target_compile_definitions(oneDPL INTERFACE ONEDPL_USE_DPCPP_BACKEND=1)
+        endif ()
+
+    endif ()
+endif ()
+
+
 # include our macros
 include(cmake/register_models.cmake)
 
@@ -84,12 +155,14 @@ register_model(hip HIP HIPStream.cpp)
 register_model(cuda CUDA CUDAStream.cu)
 register_model(kokkos KOKKOS KokkosStream.cpp)
 register_model(sycl SYCL SYCLStream.cpp)
-register_model(sycl2020 SYCL2020 SYCLStream2020.cpp)
+register_model(sycl2020-acc SYCL2020 SYCLStream2020.cpp)
+register_model(sycl2020-usm SYCL2020 SYCLStream2020.cpp)
 register_model(acc ACC ACCStream.cpp)
 # defining RAJA collides with the RAJA namespace so USE_RAJA
 register_model(raja USE_RAJA RAJAStream.cpp)
 register_model(tbb TBB TBBStream.cpp)
 register_model(thrust THRUST ThrustStream.cu) # Thrust uses cu, even for rocThrust
+register_model(futhark FUTHARK FutharkStream.cpp)
 
 
 set(USAGE ON CACHE BOOL "Whether to print all custom flags for the selected model")
@@ -101,6 +174,12 @@ else ()
     message(STATUS "Selected model  :  ${MODEL}")
 endif ()
 
+if (MODEL STREQUAL "sycl2020")
+    message(FATAL_ERROR "
+    Model sycl2020 has been renamed to sycl2020-acc, and a new sycl2020-usm model is now available.
+    Please use sycl2020-acc for SYCL2020 style accessors and sycl2020-usm for USM")
+endif ()
+
 # load the $MODEL.cmake file and setup the correct IMPL_* based on $MODEL
 load_model(${MODEL})
 
@@ -151,6 +230,7 @@ include_directories(src)
 add_executable(${EXE_NAME} ${IMPL_SOURCES} src/main.cpp)
 target_link_libraries(${EXE_NAME} PUBLIC ${LINK_LIBRARIES})
 target_compile_definitions(${EXE_NAME} PUBLIC ${IMPL_DEFINITIONS})
+target_include_directories(${EXE_NAME} PUBLIC ${IMPL_DIRECTORIES})
 
 if (CXX_EXTRA_LIBRARIES)
     target_link_libraries(${EXE_NAME} PUBLIC ${CXX_EXTRA_LIBRARIES})

README.md

@@ -38,9 +38,10 @@ BabelStream is currently implemented in the following parallel programming model
 - C++ Parallel STL
 - Kokkos
 - RAJA
-- SYCL and SYCL 2020
+- SYCL and SYCL2020 (USM and accessors)
 - TBB
 - Thrust (via CUDA or HIP)
+- Futhark
 
 This project also contains implementations in alternative languages with different build systems:
 * Julia - [JuliaStream.jl](./src/julia/JuliaStream.jl)
@@ -101,7 +102,7 @@ The source for each model's implementations are located in `./src/<model>`.
 
 Currently available models are:
 ```
-omp;ocl;std;std20;hip;cuda;kokkos;sycl;sycl2020;acc;raja;tbb;thrust
+omp;ocl;std-data;std-indices;std-ranges;hip;cuda;kokkos;sycl;sycl2020;acc;raja;tbb;thrust;futhark
 ```
 
 #### Overriding default flags
@@ -165,7 +166,7 @@ The `MODEL` variant selects one implementation of BabelStream to build.
 
 Currently available models are:
 ```
-omp;ocl;std;std20;hip;cuda;kokkos;sycl;sycl2020;acc;raja;tbb;thrust
+omp;ocl;std-data;std-indices;std-ranges;hip;cuda;kokkos;sycl;sycl2020-acc;sycl2020-usm;acc;raja;tbb;thrust
 ```
 
 ### GNU Make

src/.gitignore

@@ -16,6 +16,8 @@
 **/*.gz
 **/*.a
 
+**/*.swp
+
 **/KokkosCore_Config_*
 
 **/.DS_Store
@@ -26,4 +28,4 @@ cmake-build-*/
 CMakeFiles/
 .idea/
 .vscode/
-.directory
+.directory

src/acc/ACCStream.cpp

@@ -149,7 +149,7 @@ void ACCStream<T>::nstream()
 template <class T>
 T ACCStream<T>::dot()
 {
-  T sum = 0.0;
+  T sum{};
 
   int array_size = this->array_size;
   T * restrict a = this->a;