diff --git a/.github/actions/install-repo/action.yml b/.github/actions/install-repo/action.yml new file mode 100644 index 0000000..b799586 --- /dev/null +++ b/.github/actions/install-repo/action.yml @@ -0,0 +1,47 @@ +name: Install-Repo + +inputs: + repo-name: + required: true + repo-path: + required: true + repo-ref: + required: true + options: + required: true + cache: + required: true + submodules: + default: '' + +runs: + using: "composite" + steps: + + - name: Check Cache + if: ${{ inputs.cache == 'true'}} + uses: actions/cache@v3 + id: check-cache + with: + key: build-${{ inputs.repo-name }} + path: ${{ runner.temp }}/build-${{ inputs.repo-name }} + + - name: Checkout Repo + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo-path }} + submodules: ${{ inputs.submodules }} + ref: ${{ inputs.repo-ref }} + path: ${{ inputs.repo-name }} + + - name: Configure CMake + if: ${{ !steps.check-cache.outputs.cache-hit }} + shell: bash + run: cmake -S $GITHUB_WORKSPACE/${{ inputs.repo-name }} -B ${{ runner.temp }}/build-${{ inputs.repo-name }} + -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-${{ inputs.repo-name }}/install + ${{ inputs.options }} + + - name: Build Cmake + if: ${{ !steps.check-cache.outputs.cache-hit }} + shell: bash + run: cmake --build ${{ runner.temp }}/build-${{ inputs.repo-name }} -j8 --target install \ No newline at end of file diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 6c9ef06..8cb09ff 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -6,7 +6,7 @@ on: jobs: buildTest: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 defaults: run: @@ -32,212 +32,94 @@ jobs: - name: Install NetCDF-Fortran run: sudo apt-get install libnetcdff-dev - # Build Kokkos + - uses: actions/checkout@v4 - - name: Cache Kokkos Build - uses: actions/cache@v3 - id: build-kokkos + - name: build kokkos + uses: ./.github/actions/install-repo with: - key: build-kokkos - path: ${{ runner.temp }}/build-kokkos - - - name: Kokkos Checkout repo - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 + repo-name: 'kokkos' + repo-path: 'kokkos/kokkos' + repo-ref: '4.5.00' + cache: true + options: '-DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ENABLE_OPENMP=off + -DKokkos_ENABLE_CUDA=off + -DKokkos_ENABLE_CUDA_LAMBDA=off' + + - name: build omega_h + uses: ./.github/actions/install-repo with: - repository: kokkos/kokkos - ref: 4.1.00 - path: kokkos - - - name: Kokkos Create Directory - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-kokkos - - - name: Kokkos Configure CMake - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/kokkos -B ${{ runner.temp }}/build-kokkos - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-kokkos/install - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DKokkos_ENABLE_SERIAL=ON - -DKokkos_ENABLE_OPENMP=off - -DKokkos_ENABLE_CUDA=off - -DKokkos_ENABLE_CUDA_LAMBDA=off - -DKokkos_ENABLE_DEBUG=on - - - name: Kokkos Build - if: ${{ steps.build-kokkos.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-kokkos -j8 --target install - - # Build EnGPar - - - name: Cache Engpar Build - uses: actions/cache@v3 - id: build-engpar - with: - key: build-engpar - path: ${{ runner.temp }}/build-engpar - - - name: EnGPar Checkout repo - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 - with: - repository: SCOREC/EnGPar - path: engpar - - - name: EnGPar Create Directory - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-engpar - - - name: EnGPar Configure CMake - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/engpar -B ${{ runner.temp }}/build-engpar - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-engpar/install - -DCMAKE_C_COMPILER="mpicc" - -DCMAKE_CXX_COMPILER="mpicxx" - -DCMAKE_CXX_FLAGS="-std=c++11" - -DENABLE_PARMETIS=OFF - -DENABLE_PUMI=OFF - -DIS_TESTING=OFF - - - name: EnGPar Build - if: ${{ steps.build-engpar.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-engpar -j8 --target install - - # Build Omega_h - - - name: Cache Omega_h Build - uses: actions/cache@v3 - id: build-omega_h - with: - key: build-omega_h - path: ${{ runner.temp }}/build-omega_h - - - name: Omega_h Checkout repo - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 + repo-name: 'omega_h' + repo-path: 'SCOREC/omega_h' + repo-ref: '' + cache: true + options: '-DCMAKE_BUILD_TYPE=Release + -DBUILD_SHARED_LIBS=OFF + -DOmega_h_USE_Kokkos=ON + -DOmega_h_USE_CUDA=off + -DOmega_h_USE_MPI=on + -DBUILD_TESTING=off + -DCMAKE_C_COMPILER=mpicc + -DCMAKE_CXX_COMPILER=mpicxx + -DKokkos_PREFIX=${{ runner.temp }}/build-kokkos/install/lib/cmake' + + - name: build Engpar + uses: ./.github/actions/install-repo with: - repository: SCOREC/omega_h - ref: scorec-v10.8.0 - path: omega_h - - - name: Omega_h Create Directory - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-omega_h - - - name: Omega_h Configure CMake - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/omega_h -B ${{ runner.temp }}/build-omega_h - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-omega_h/install - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake - -DBUILD_SHARED_LIBS=OFF - -DOmega_h_USE_Kokkos=ON - -DOmega_h_USE_CUDA=off - -DOmega_h_USE_MPI=on - -DCMAKE_BUILD_TYPE=Release - -DBUILD_TESTING=on - -DCMAKE_CXX_COMPILER="mpicxx" - -DCMAKE_C_COMPILER="mpicc" - - - name: Omega_h Build - if: ${{ steps.build-omega_h.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-omega_h -j8 --target install - - # Build Cabana - - - name: Cache Cabana Build - uses: actions/cache@v3 - id: build-cabana + repo-name: 'Engpar' + repo-path: 'SCOREC/EnGPar' + repo-ref: '' + cache: true + options: '-DCMAKE_C_COMPILER=mpicc + -DCMAKE_CXX_COMPILER=mpicxx + -DCMAKE_CXX_FLAGS="-std=c++11" + -DENABLE_PARMETIS=OFF + -DENABLE_PUMI=OFF + -DIS_TESTING=OFF' + + - name: build cabana + uses: ./.github/actions/install-repo with: - key: build-cabana - path: ${{ runner.temp }}/build-cabana - - - name: Cabana Checkout repo - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 + repo-name: 'cabana' + repo-path: 'ECP-copa/cabana' + repo-ref: '0.6.1' + cache: true + options: '-DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON + -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake' + + - name: build pumi-pic + uses: ./.github/actions/install-repo with: - repository: ECP-copa/cabana - ref: 0.6.1 - path: cabana - - - name: Cabana Create Directory - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-cabana - - - name: Cabana Configure CMake - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/cabana -B ${{ runner.temp }}/build-cabana - -DCMAKE_BUILD_TYPE="Release" - -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-cabana/install - - - name: Cabana Build - if: ${{ steps.build-cabana.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-cabana -j8 --target install - - # Build PUMI-PIC - - - name: Cache PUMI-PIC Build - uses: actions/cache@v3 - id: build-pumi-pic - with: - key: build-pumi-pic - path: ${{ runner.temp }}/build-pumi-pic - - - name: PUMI-PIC Checkout repo - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - uses: actions/checkout@v3 - with: - submodules: recursive - repository: SCOREC/pumi-pic - ref: 2.0.3 - path: pumi-pic - - - name: PUMI-PIC Create Directory - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - run: cmake -E make_directory ${{ runner.temp }}/build-pumi-pic - - - name: PUMI-PIC Configure CMake - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - run: cmake -S $GITHUB_WORKSPACE/pumi-pic -B ${{ runner.temp }}/build-pumi-pic - -DCMAKE_CXX_COMPILER=mpicxx - -DIS_TESTING=ON - -DPS_IS_TESTING=ON - -DCMAKE_BUILD_TYPE="Release" - -DTEST_DATA_DIR=$GITHUB_WORKSPACE/pumi-pic/pumipic-data - -DOmega_h_PREFIX=${{ runner.temp }}/build-omega_h/install - -DKokkos_PREFIX=${{ runner.temp }}/build-kokkos/install - -DEnGPar_PREFIX=${{ runner.temp }}/build-engpar/install - -DCabana_PREFIX=${{ runner.temp }}/build-cabana/install - -DENABLE_CABANA=on - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-pumi-pic/install - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake - - - name: PUMI-PIC Build - if: ${{ steps.build-pumi-pic.outputs.cache-hit != 'true' }} - run: cmake --build ${{ runner.temp }}/build-pumi-pic -j8 --target install - - # Build PolyMPO - - - name: PolyMPO Checkout repo - uses: actions/checkout@v3 + repo-name: 'pumi-pic' + repo-path: 'SCOREC/pumi-pic' + submodules: 'recursive' + repo-ref: '' + cache: true + options: '-DCMAKE_CXX_COMPILER=mpicxx + -DTEST_DATA_DIR=$GITHUB_WORKSPACE/pumi-pic/pumipic-data + -DOmega_h_PREFIX=${{ runner.temp }}/build-omega_h/install + -DKokkos_PREFIX=${{ runner.temp }}/build-kokkos/install + -DEnGPar_PREFIX=${{ runner.temp }}/build-Engpar/install + -DCabana_PREFIX=${{ runner.temp }}/build-cabana/install + -DENABLE_CABANA=on + -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-kokkos/install/lib/cmake' + + - name: build polyMPO + uses: ./.github/actions/install-repo with: - submodules: recursive - repository: SCOREC/polyMPO - path: polyMPO - - - name: PolyMPO Create Directory - run: cmake -E make_directory ${{ runner.temp }}/build-polyMPO - - - name: PolyMPO Configure CMake - run: cmake -S $GITHUB_WORKSPACE/polyMPO -B ${{ runner.temp }}/build-polyMPO - -DCMAKE_BUILD_TYPE="Debug" - -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib64/cmake/Kokkos - -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-pumi-pic/install - -DIS_TESTING=on - -DCMAKE_CXX_COMPILER=mpicxx - -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/build-polyMPO/install + repo-name: 'polyMPO' + repo-path: 'SCOREC/polyMPO' + submodules: 'recursive' + repo-ref: '' + cache: false + options: '-DCMAKE_BUILD_TYPE=Debug + -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib64/cmake/Kokkos + -DCMAKE_PREFIX_PATH=${{ runner.temp }}/build-pumi-pic/install + -DIS_TESTING=on + -DCMAKE_CXX_COMPILER=mpicxx' - name: PolyMPO Build run: cmake --build ${{ runner.temp }}/build-polyMPO -j8 --target install diff --git a/.github/workflows/globus-test.yml b/.github/workflows/globus-test.yml new file mode 100644 index 0000000..07b26d7 --- /dev/null +++ b/.github/workflows/globus-test.yml @@ -0,0 +1,13 @@ +name: Globus-Systems +on: + schedule: + # Monday 9:35 UTC or 04:35 EDT + - cron: '35 9 * * 1' + +jobs: + + perlmutter-test: + uses: SCOREC/github-actions/.github/workflows/globus-test.yml@main + secrets: inherit + with: + machine: "perlmutter" \ No newline at end of file diff --git a/.github/workflows/perlmutter/install.sh b/.github/workflows/perlmutter/install.sh new file mode 100755 index 0000000..da6235a --- /dev/null +++ b/.github/workflows/perlmutter/install.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +branch=$1 + +cd $SCRATCH/globus-compute/polyMPO-test + +export root=$PWD +module load cmake/3.24.3 +module load cray-hdf5 +module load cray-netcdf + +export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:\ +$root/build-kokkos/install:\ +$root/build-omegah/install:\ +$root/build-cabana/install:\ +$root/build-engpar/install:\ +$root/build-pumipic/install + +export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper + +# #kokkos +# git clone -b 4.5.00 https://github.com/kokkos/kokkos.git +# cmake -S kokkos -B build-kokkos \ +# -DCMAKE_INSTALL_PREFIX=build-kokkos/install \ +# -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \ +# -DKokkos_ARCH_AMPERE80=ON \ +# -DKokkos_ENABLE_SERIAL=ON \ +# -DKokkos_ENABLE_OPENMP=off \ +# -DKokkos_ENABLE_CUDA=ON \ +# -DKokkos_ENABLE_CUDA_LAMBDA=ON +# cmake --build build-kokkos -j 24 --target install + +# #engpar +# unset MPICH_CXX #don't want nvcc_wrapper for engpar +# git clone https://github.com/SCOREC/EnGPar.git +# cmake -S EnGPar -B build-engpar \ +# -DCMAKE_INSTALL_PREFIX=build-engpar/install \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_C_COMPILER=cc \ +# -DCMAKE_CXX_COMPILER=CC \ +# -DCMAKE_CXX_FLAGS="-std=c++11" \ +# -DENABLE_PARMETIS=OFF \ +# -DENABLE_PUMI=OFF \ +# -DIS_TESTING=OFF +# cmake --build build-engpar -j 24 --target install +# export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper #restore use of nvcc_wrapper + +# #omegah +# git clone -b scorec-v10.8.4 https://github.com/SCOREC/omega_h.git +# cmake -S omega_h -B build-omegah \ +# -DCMAKE_INSTALL_PREFIX=build-omegah/install \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DBUILD_SHARED_LIBS=OFF \ +# -DOmega_h_USE_Kokkos=ON \ +# -DOmega_h_USE_CUDA=on \ +# -DOmega_h_CUDA_ARCH=80 \ +# -DOmega_h_USE_MPI=on \ +# -DBUILD_TESTING=off \ +# -DCMAKE_C_COMPILER=cc \ +# -DCMAKE_CXX_COMPILER=CC +# cmake --build build-omegah -j 24 --target install + +# #cabana +# git clone -b 0.6.1 https://github.com/ECP-copa/Cabana.git cabana +# cmake -S cabana -B build-cabana \ +# -DCMAKE_INSTALL_PREFIX=build-cabana/install \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \ +# -DCabana_ENABLE_TESTING=OFF \ +# -DCabana_ENABLE_EXAMPLES=OFF +# cmake --build build-cabana -j 24 --target install + +# #pumipic +# git clone --recursive https://github.com/SCOREC/pumi-pic.git +# cmake -S pumi-pic -B build-pumipic \ +# -DCMAKE_INSTALL_PREFIX=build-pumipic/install \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_CXX_COMPILER=CC \ +# -DENABLE_CABANA=ON \ +# -DCMAKE_CXX_STANDARD=20 \ +# -DTEST_DATA_DIR=$root/pumi-pic/pumipic-data \ +# -DIS_TESTING=ON \ +# -DPS_IS_TESTING=ON +# cmake --build build-pumipic -j 24 --target install + +# polyMPO +rm polyMPO -rf +git clone -b cws/pumipicDps https://github.com/SCOREC/polyMPO.git +cd polyMPO && git checkout $branch && cd - +rm build-polyMPO -rf +cmake -S polyMPO -B build-polyMPO \ + -DCMAKE_INSTALL_PREFIX=build-polyMPO/install \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \ + -DCMAKE_Fortran_COMPILER=ftn \ + -DIS_TESTING=on +cmake --build build-polyMPO --target install -j4 \ No newline at end of file diff --git a/.github/workflows/perlmutter/run.sh b/.github/workflows/perlmutter/run.sh new file mode 100755 index 0000000..b93262a --- /dev/null +++ b/.github/workflows/perlmutter/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +name=polyMPO +cd $SCRATCH/globus-compute/$name-test + +export root=$PWD +module load cmake +module load cray-hdf5 +module load cray-netcdf +export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper + +cd build-$name +salloc --time 00:10:00 --constrain=gpu --qos=interactive --nodes=1 --ntasks-per-node=40 --cpus-per-task=1 --gpus=1 --account=m4564 ctest +cat $PWD/Testing/Temporary/LastTest.log \ No newline at end of file diff --git a/.github/workflows/updateCmakeVersion.yml b/.github/workflows/updateCmakeVersion.yml new file mode 100644 index 0000000..d7a9da9 --- /dev/null +++ b/.github/workflows/updateCmakeVersion.yml @@ -0,0 +1,31 @@ +name: Update CMAKE Version +on: + pull_request: + +jobs: + checkVersion: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash + + steps: + + - name: Check out master + uses: actions/checkout@v4 + with: + path: master + ref: ${{ github.base_ref }} + + - name: Check out current + uses: actions/checkout@v4 + with: + path: branch + + - name: CMake Version Has Updated + run: | + masterVersion=$(grep -oP 'project\(\s*[\w-]+\s+VERSION\s+\K[0-9]+\.[0-9]+\.[0-9]+' master/CMakeLists.txt) + branchVersion=$(grep -oP 'project\(\s*[\w-]+\s+VERSION\s+\K[0-9]+\.[0-9]+\.[0-9]+' branch/CMakeLists.txt) + [ "$masterVersion" != "$branchVersion" ] + \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 080a418..094d7a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.19) -project(polyMPO VERSION 0.1.0 LANGUAGES CXX Fortran) +project(polyMPO VERSION 0.2.0 LANGUAGES CXX Fortran) include(cmake/bob.cmake) bob_begin_package() diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index 99d2964..70c5263 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -125,7 +125,7 @@ class MaterialPoints { Operating_Mode operating_mode; RebuildHelper rebuildFields; IntFunc getAppID; - MPI_Comm mpi_comm; + MPI_Comm mpi_comm = MPI_COMM_WORLD; public: MaterialPoints() : MPs(nullptr) {};