Build container with JEDI libraries weekly

.github/workflows/build-jedi-container.yaml

@@ -0,0 +1,115 @@
+name: Build and Push JEDI Container
+
+on:
+  push:
+    branches: [feature/build-container]
+  schedule:
+    # Runs at 19:00 UTC every Saturday
+    - cron: '0 19 * * 6'
+  workflow_dispatch: # Allows manual triggering
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+env:
+  REGISTRY: ghcr.io
+  BASE_IMAGE: jcsda/docker-gnu-openmpi-dev:1.9
+  INSTALL_PREFIX: /opt/jedi
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Free disk space
+      run: |
+        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+        sudo docker image prune --all --force
+        df -h
+
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+
+    - name: Initialize required submodules
+      run: |
+        git submodule update --init --recursive \
+          sorc/jedicmake \
+          sorc/oops \
+          sorc/vader \
+          sorc/gsibec \
+          sorc/saber \
+          sorc/ioda \
+          sorc/ufo \
+          sorc/crtm \
+          sorc/bufr-query \
+          sorc/jcb
+
+    - name: TEMP Patch oops_src_files for NormGradient.h
+      run: |
+        awk '/oops\/interface\/VariableChange.h/ { print; print "  oops/interface/NormGradient.h"; next } 1' sorc/oops/src/CMakeLists.txt > sorc/oops/src/CMakeLists.txt.tmp
+        mv sorc/oops/src/CMakeLists.txt.tmp sorc/oops/src/CMakeLists.txt
+
+    - name: Log in to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Generate image metadata
+      id: meta
+      run: |
+        IMAGE_NAME=$(echo "${{ env.REGISTRY }}/${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
+        DATE_TAG=$(date +'%Y%m%d')
+        SHA_TAG=$(echo "${{ github.sha }}" | cut -c1-7)
+        echo "image=${IMAGE_NAME}" >> $GITHUB_OUTPUT
+        echo "date_tag=${DATE_TAG}" >> $GITHUB_OUTPUT
+        echo "sha_tag=${SHA_TAG}" >> $GITHUB_OUTPUT
+
+    - name: Build JEDI libraries in container
+      run: |
+        docker run --name jedi-build \
+          -v ${{ github.workspace }}:/src \
+          ${{ env.BASE_IMAGE }} \
+          bash -c "
+            source /opt/spack-environment/activate.sh
+            export jedi_cmake_ROOT=/opt/view
+            mkdir -p /build && cd /build
+            ecbuild \
+              -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_PREFIX }} \
+              -DBUILD_JEDI=ON \
+              -DBUILD_GDAS=OFF \
+              -DBUILD_SOCA=OFF \
+              -DBUILD_GSIBEC=ON \
+              -DBUILD_TESTING=OFF \
+              -DCLONE_JCSDADATA=OFF \
+              /src/bundle
+            make -j\$(nproc) install
+            rm -rf /build
+          "
+
+    - name: Commit and tag container image
+      run: |
+        docker commit \
+          --change 'CMD ["/bin/bash"]' \
+          --change 'ENV JEDI_INSTALL_PREFIX=${{ env.INSTALL_PREFIX }}' \
+          --change 'ENV CMAKE_PREFIX_PATH=${{ env.INSTALL_PREFIX }}:/opt/view' \
+          --change 'ENV PATH=${{ env.INSTALL_PREFIX }}/bin:/opt/view/bin:$PATH' \
+          --change 'ENV LD_LIBRARY_PATH=${{ env.INSTALL_PREFIX }}/lib:/opt/view/lib:$LD_LIBRARY_PATH' \
+          --change 'ENV BASH_ENV=/opt/spack-environment/activate.sh' \
+          jedi-build ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }}
+        docker tag ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }} \
+          ${{ steps.meta.outputs.image }}:latest
+        docker tag ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }} \
+          ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }}
+
+    - name: Push container image
+      run: |
+        docker push ${{ steps.meta.outputs.image }}:latest
+        docker push ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date_tag }}
+        docker push ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }}

bundle/CMakeLists.txt

@@ -33,6 +33,8 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 # Handle user options.
 option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON)
+option(BUILD_JEDI "Build core JEDI libraries (oops, vader, saber, ioda, ufo, crtm, gsibec, bufr-query)" ON)
+option(BUILD_GDAS "Build GDAS application code (fv3-jedi, soca, gdas, da-utils, land-jediincr, jcb)" ON)
 option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
 option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF)
 option(BUILD_REGRID_STATES "Build regrid states land DA utility" ON)
@@ -78,67 +80,72 @@ if(BUILD_GDASBUNDLE)
   # TODO remove the CRTM from here and use it as a library
   #option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF
   #ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git"   TAG v2.4.1-jedi.2 )
-  ecbuild_bundle( PROJECT crtm SOURCE "../sorc/crtm" )
+  if(BUILD_JEDI)
+    ecbuild_bundle( PROJECT crtm SOURCE "../sorc/crtm" )
 
 # Build GSI-B
-  option(BUILD_GSIBEC "Build GSI-B" OFF)
-  if(BUILD_GSIBEC)
-    ecbuild_bundle( PROJECT gsibec SOURCE "../sorc/gsibec" )
-  endif()
+    option(BUILD_GSIBEC "Build GSI-B" OFF)
+    if(BUILD_GSIBEC)
+      ecbuild_bundle( PROJECT gsibec SOURCE "../sorc/gsibec" )
+    endif()
 
-# Gibbs seawater
-  if (BUILD_SOCA)
-    ecbuild_bundle( PROJECT gsw SOURCE "../sorc/gsw" )
-  endif()
+# Gibbs seawater (optional dependency for vader)
+    if (BUILD_SOCA)
+      ecbuild_bundle( PROJECT gsw SOURCE "../sorc/gsw" )
+    endif()
 
 # EMC BUFR-query library
-  ecbuild_bundle( PROJECT bufr-query SOURCE "../sorc/bufr-query" )
+    ecbuild_bundle( PROJECT bufr-query SOURCE "../sorc/bufr-query" )
 
 # Core JEDI repositories
-  ecbuild_bundle( PROJECT oops SOURCE "../sorc/oops" )
-  ecbuild_bundle( PROJECT vader SOURCE "../sorc/vader" )
-  ecbuild_bundle( PROJECT saber SOURCE "../sorc/saber" )
-  option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT ioda SOURCE "../sorc/ioda" )
-  option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT ufo SOURCE "../sorc/ufo" )
-
+    ecbuild_bundle( PROJECT oops SOURCE "../sorc/oops" )
+    ecbuild_bundle( PROJECT vader SOURCE "../sorc/vader" )
+    ecbuild_bundle( PROJECT saber SOURCE "../sorc/saber" )
+    option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
+    ecbuild_bundle( PROJECT ioda SOURCE "../sorc/ioda" )
+    option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
+    ecbuild_bundle( PROJECT ufo SOURCE "../sorc/ufo" )
+  endif(BUILD_JEDI)
+
+# GDAS application code
+  if(BUILD_GDAS)
 # fv3-jedi and associated repositories
-  ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "../sorc/fv3-jedi-lm" )
-  option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT fv3-jedi SOURCE "../sorc/fv3-jedi" )
+    ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "../sorc/fv3-jedi-lm" )
+    option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
+    ecbuild_bundle( PROJECT fv3-jedi SOURCE "../sorc/fv3-jedi" )
 
 # SOCA associated repositories
-  option(BUILD_SOCA "Build SOCA" ON)
-  if (BUILD_SOCA)
-    ecbuild_bundle( PROJECT soca SOURCE "../sorc/soca" )
-  endif()
+    option(BUILD_SOCA "Build SOCA" ON)
+    if (BUILD_SOCA)
+      ecbuild_bundle( PROJECT soca SOURCE "../sorc/soca" )
+    endif()
 
-  # Build JEDI/DA or other peripherals
-  ecbuild_bundle( PROJECT gdas-utils SOURCE "../utils" )
-  ecbuild_bundle( PROJECT da-utils SOURCE "../sorc/da-utils" )
+    # Build JEDI/DA or other peripherals
+    ecbuild_bundle( PROJECT gdas-utils SOURCE "../utils" )
+    ecbuild_bundle( PROJECT da-utils SOURCE "../sorc/da-utils" )
 
 # Build IODA converters
-  if(BUILD_IODA_CONVERTERS)
-    ecbuild_bundle( PROJECT iodaconv SOURCE "../sorc/iodaconv" )
-  endif()
+    if(BUILD_IODA_CONVERTERS)
+      ecbuild_bundle( PROJECT iodaconv SOURCE "../sorc/iodaconv" )
+    endif()
 
 # Land associated repositories
-  ecbuild_bundle( PROJECT land-jediincr SOURCE "../sorc/land-jediincr" )
+    ecbuild_bundle( PROJECT land-jediincr SOURCE "../sorc/land-jediincr" )
 
 # GDASApp
-  ecbuild_bundle( PROJECT gdas SOURCE "../")
+    ecbuild_bundle( PROJECT gdas SOURCE "../")
 
 # Install jcb (JEDI Configuration Builder) Python package
-  message(STATUS "Installing jcb (JEDI Configuration Builder)")
-  execute_process(
-    COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps --no-build-isolation --prefix ${CMAKE_INSTALL_PREFIX} .
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../sorc/jcb
-    RESULT_VARIABLE jcb_install_result
-  )
-  if(NOT jcb_install_result EQUAL 0)
-    message(FATAL_ERROR "Failed to install jcb Python package")
-  endif()
+    message(STATUS "Installing jcb (JEDI Configuration Builder)")
+    execute_process(
+      COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps --no-build-isolation --prefix ${CMAKE_INSTALL_PREFIX} .
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../sorc/jcb
+      RESULT_VARIABLE jcb_install_result
+    )
+    if(NOT jcb_install_result EQUAL 0)
+      message(FATAL_ERROR "Failed to install jcb Python package")
+    endif()
+  endif(BUILD_GDAS)
 
 # ioda, ufo, fv3-jedi, and saber test data
 #---------------------------------