diff --git a/.github/workflows/build-jedi-container.yaml b/.github/workflows/build-jedi-container.yaml new file mode 100644 index 000000000..71a83cadd --- /dev/null +++ b/.github/workflows/build-jedi-container.yaml @@ -0,0 +1,115 @@ +name: Build and Push JEDI Container + +on: + push: + branches: [feature/build-container] + schedule: + # Runs at 19:00 UTC every Saturday + - cron: '0 19 * * 6' + workflow_dispatch: # Allows manual triggering + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +env: + REGISTRY: ghcr.io + BASE_IMAGE: jcsda/docker-gnu-openmpi-dev:1.9 + INSTALL_PREFIX: /opt/jedi + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo docker image prune --all --force + df -h + + - name: Checkout repository + uses: actions/checkout@v4 + + + - name: Initialize required submodules + run: | + git submodule update --init --recursive \ + sorc/jedicmake \ + sorc/oops \ + sorc/vader \ + sorc/gsibec \ + sorc/saber \ + sorc/ioda \ + sorc/ufo \ + sorc/crtm \ + sorc/bufr-query \ + sorc/jcb + + - name: TEMP Patch oops_src_files for NormGradient.h + run: | + awk '/oops\/interface\/VariableChange.h/ { print; print " oops/interface/NormGradient.h"; next } 1' sorc/oops/src/CMakeLists.txt > sorc/oops/src/CMakeLists.txt.tmp + mv sorc/oops/src/CMakeLists.txt.tmp sorc/oops/src/CMakeLists.txt + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate image metadata + id: meta + run: | + IMAGE_NAME=$(echo "${{ env.REGISTRY }}/${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + DATE_TAG=$(date +'%Y%m%d') + SHA_TAG=$(echo "${{ github.sha }}" | cut -c1-7) + echo "image=${IMAGE_NAME}" >> $GITHUB_OUTPUT + echo "date_tag=${DATE_TAG}" >> $GITHUB_OUTPUT + echo "sha_tag=${SHA_TAG}" >> $GITHUB_OUTPUT + + - name: Build JEDI libraries in container + run: | + docker run --name jedi-build \ + -v ${{ github.workspace }}:/src \ + ${{ env.BASE_IMAGE }} \ + bash -c " + source /opt/spack-environment/activate.sh + export jedi_cmake_ROOT=/opt/view + mkdir -p /build && cd /build + ecbuild \ + -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_PREFIX }} \ + -DBUILD_JEDI=ON \ + -DBUILD_GDAS=OFF \ + -DBUILD_SOCA=OFF \ + -DBUILD_GSIBEC=ON \ + -DBUILD_TESTING=OFF \ + -DCLONE_JCSDADATA=OFF \ + /src/bundle + make -j\$(nproc) install + rm -rf /build + " + + - name: Commit and tag container image + run: | + docker commit \ + --change 'CMD ["/bin/bash"]' \ + --change 'ENV JEDI_INSTALL_PREFIX=${{ env.INSTALL_PREFIX }}' \ + --change 'ENV CMAKE_PREFIX_PATH=${{ env.INSTALL_PREFIX }}:/opt/view' \ + --change 'ENV PATH=${{ env.INSTALL_PREFIX }}/bin:/opt/view/bin:$PATH' \ + --change 'ENV LD_LIBRARY_PATH=${{ env.INSTALL_PREFIX }}/lib:/opt/view/lib:$LD_LIBRARY_PATH' \ + --change 'ENV BASH_ENV=/opt/spack-environment/activate.sh' \ + jedi-build ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }} + docker tag ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }} \ + ${{ steps.meta.outputs.image }}:latest + docker tag ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }} \ + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} + + - name: Push container image + run: | + docker push ${{ steps.meta.outputs.image }}:latest + docker push ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }} + docker push ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} diff --git a/bundle/CMakeLists.txt b/bundle/CMakeLists.txt index 86b54f81e..2f29aa816 100644 --- a/bundle/CMakeLists.txt +++ b/bundle/CMakeLists.txt @@ -33,6 +33,8 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) # Handle user options. option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON) +option(BUILD_JEDI "Build core JEDI libraries (oops, vader, saber, ioda, ufo, crtm, gsibec, bufr-query)" ON) +option(BUILD_GDAS "Build GDAS application code (fv3-jedi, soca, gdas, da-utils, land-jediincr, jcb)" ON) option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF) option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) option(BUILD_REGRID_STATES "Build regrid states land DA utility" ON) @@ -78,67 +80,72 @@ if(BUILD_GDASBUNDLE) # TODO remove the CRTM from here and use it as a library #option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF #ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.2 ) - ecbuild_bundle( PROJECT crtm SOURCE "../sorc/crtm" ) + if(BUILD_JEDI) + ecbuild_bundle( PROJECT crtm SOURCE "../sorc/crtm" ) # Build GSI-B - option(BUILD_GSIBEC "Build GSI-B" OFF) - if(BUILD_GSIBEC) - ecbuild_bundle( PROJECT gsibec SOURCE "../sorc/gsibec" ) - endif() + option(BUILD_GSIBEC "Build GSI-B" OFF) + if(BUILD_GSIBEC) + ecbuild_bundle( PROJECT gsibec SOURCE "../sorc/gsibec" ) + endif() -# Gibbs seawater - if (BUILD_SOCA) - ecbuild_bundle( PROJECT gsw SOURCE "../sorc/gsw" ) - endif() +# Gibbs seawater (optional dependency for vader) + if (BUILD_SOCA) + ecbuild_bundle( PROJECT gsw SOURCE "../sorc/gsw" ) + endif() # EMC BUFR-query library - ecbuild_bundle( PROJECT bufr-query SOURCE "../sorc/bufr-query" ) + ecbuild_bundle( PROJECT bufr-query SOURCE "../sorc/bufr-query" ) # Core JEDI repositories - ecbuild_bundle( PROJECT oops SOURCE "../sorc/oops" ) - ecbuild_bundle( PROJECT vader SOURCE "../sorc/vader" ) - ecbuild_bundle( PROJECT saber SOURCE "../sorc/saber" ) - option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ioda SOURCE "../sorc/ioda" ) - option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo SOURCE "../sorc/ufo" ) - + ecbuild_bundle( PROJECT oops SOURCE "../sorc/oops" ) + ecbuild_bundle( PROJECT vader SOURCE "../sorc/vader" ) + ecbuild_bundle( PROJECT saber SOURCE "../sorc/saber" ) + option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ioda SOURCE "../sorc/ioda" ) + option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ufo SOURCE "../sorc/ufo" ) + endif(BUILD_JEDI) + +# GDAS application code + if(BUILD_GDAS) # fv3-jedi and associated repositories - ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "../sorc/fv3-jedi-lm" ) - option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi SOURCE "../sorc/fv3-jedi" ) + ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "../sorc/fv3-jedi-lm" ) + option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT fv3-jedi SOURCE "../sorc/fv3-jedi" ) # SOCA associated repositories - option(BUILD_SOCA "Build SOCA" ON) - if (BUILD_SOCA) - ecbuild_bundle( PROJECT soca SOURCE "../sorc/soca" ) - endif() + option(BUILD_SOCA "Build SOCA" ON) + if (BUILD_SOCA) + ecbuild_bundle( PROJECT soca SOURCE "../sorc/soca" ) + endif() - # Build JEDI/DA or other peripherals - ecbuild_bundle( PROJECT gdas-utils SOURCE "../utils" ) - ecbuild_bundle( PROJECT da-utils SOURCE "../sorc/da-utils" ) + # Build JEDI/DA or other peripherals + ecbuild_bundle( PROJECT gdas-utils SOURCE "../utils" ) + ecbuild_bundle( PROJECT da-utils SOURCE "../sorc/da-utils" ) # Build IODA converters - if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv SOURCE "../sorc/iodaconv" ) - endif() + if(BUILD_IODA_CONVERTERS) + ecbuild_bundle( PROJECT iodaconv SOURCE "../sorc/iodaconv" ) + endif() # Land associated repositories - ecbuild_bundle( PROJECT land-jediincr SOURCE "../sorc/land-jediincr" ) + ecbuild_bundle( PROJECT land-jediincr SOURCE "../sorc/land-jediincr" ) # GDASApp - ecbuild_bundle( PROJECT gdas SOURCE "../") + ecbuild_bundle( PROJECT gdas SOURCE "../") # Install jcb (JEDI Configuration Builder) Python package - message(STATUS "Installing jcb (JEDI Configuration Builder)") - execute_process( - COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps --no-build-isolation --prefix ${CMAKE_INSTALL_PREFIX} . - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../sorc/jcb - RESULT_VARIABLE jcb_install_result - ) - if(NOT jcb_install_result EQUAL 0) - message(FATAL_ERROR "Failed to install jcb Python package") - endif() + message(STATUS "Installing jcb (JEDI Configuration Builder)") + execute_process( + COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps --no-build-isolation --prefix ${CMAKE_INSTALL_PREFIX} . + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../sorc/jcb + RESULT_VARIABLE jcb_install_result + ) + if(NOT jcb_install_result EQUAL 0) + message(FATAL_ERROR "Failed to install jcb Python package") + endif() + endif(BUILD_GDAS) # ioda, ufo, fv3-jedi, and saber test data #---------------------------------