From 185ca0f7dcb58e97d1a993a4cd96b7abcfabb804 Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 17:40:40 -0400 Subject: [PATCH 01/11] bug fixed in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index a2e2c40..6068723 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -80,10 +80,7 @@ jobs: uses: actions/cache@v3 with: path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: - ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ - env.MD5_CHECKSUM }}-${{ - env.CACHE_NUMBER }} + key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{env.MD5_CHECKSUM }}-${{env.CACHE_NUMBER }} env: # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed CACHE_NUMBER: 0 From 74450142eb137f42bac2f54b1fd3b326d9335e68 Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 17:49:17 -0400 Subject: [PATCH 02/11] removed unwated single quotes in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index 6068723..d3c2101 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -37,13 +37,13 @@ jobs: export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - export CONDA_ENV_NAME='${{ inputs.conda_env_name }}' + export CONDA_ENV_NAME=${{ inputs.conda_env_name }} echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - export ZENODO_ID='${{ inputs.zenodo_id }}' + export ZENODO_ID=${{ inputs.zenodo_id }} echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - export MD5_CHECKSUM='${{ inputs.md5_checksum }}' + export MD5_CHECKSUM=${{ inputs.md5_checksum }} echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV # Workaround for unset GDAL env variables used by newer conda envs From 88d590aa4d162f93c787ebe974176c997568ea9f Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 17:55:31 -0400 Subject: [PATCH 03/11] bug fixed in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index d3c2101..90abe96 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -80,7 +80,7 @@ jobs: uses: actions/cache@v3 with: path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{env.MD5_CHECKSUM }}-${{env.CACHE_NUMBER }} + key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ runner.arch }}-${{env.MD5_CHECKSUM }}-${{env.CACHE_NUMBER }} env: # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed CACHE_NUMBER: 0 From 6c2cec8908f114be8c74f0cd942fb8e8efcaab03 Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 18:03:14 -0400 Subject: [PATCH 04/11] bug fixed in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index 90abe96..12cf7c5 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -80,7 +80,7 @@ jobs: uses: actions/cache@v3 with: path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ runner.arch }}-${{env.MD5_CHECKSUM }}-${{env.CACHE_NUMBER }} + key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ runner.arch }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} env: # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed CACHE_NUMBER: 0 From d96614da78603a16ab752818c0aab3407dbefbbe Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 18:15:11 -0400 Subject: [PATCH 05/11] bug fixed in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index 12cf7c5..bfd77b6 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -80,7 +80,7 @@ jobs: uses: actions/cache@v3 with: path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ runner.arch }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} + key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ runner.arch }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} env: # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed CACHE_NUMBER: 0 From 757e24cfd8d57ae71e7744807f72b6cdfa7bc54f Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 18:19:58 -0400 Subject: [PATCH 06/11] attempting to fix bug in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index bfd77b6..b1bd3de 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -80,7 +80,7 @@ jobs: uses: actions/cache@v3 with: path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ runner.arch }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} + key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} env: # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed CACHE_NUMBER: 0 From 9347dbc9aa4e2e3f5f2568cb1af42acdffd21559 Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 21:22:43 -0400 Subject: [PATCH 07/11] conda-env-tests.yml modified --- .github/workflows/conda-env-tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index aa0673a..d5a40b7 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -1,29 +1,29 @@ -name: Unit Tests in NSLS-II Conda Environments +name: Run tests in NSLS-II Conda Environments -on: [push, pull_request, workflow_dispatch] - -env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} +on: + push: + branches: [main] + pull_request: jobs: - + # First prepare the matrix matrix_prep: - uses: ./.github/workflows/_matrix_prep.yml with: config_prefix: recent + # Then run tests using that matrix test-in-conda-env: - needs: matrix_prep + strategy: + max-parallel: 4 + fail-fast: false + matrix: + include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + uses: ./.github/workflows/_test-in-conda-env.yml with: conda_env_name: ${{ matrix.conda_env_name }} zenodo_id: ${{ matrix.zenodo_id }} md5_checksum: ${{ matrix.md5_checksum }} python-version: ${{ matrix.python-version }} - strategy: - max-parallel: 4 - matrix: - include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} - fail-fast: false From c7a0557c34251aa20cfca90eff2b6b5ab48293c3 Mon Sep 17 00:00:00 2001 From: nisar Date: Tue, 26 Aug 2025 21:27:44 -0400 Subject: [PATCH 08/11] modified conda-env-tests.yml --- .github/workflows/_test-in-conda-env.yml | 219 +++++++++++------------ 1 file changed, 108 insertions(+), 111 deletions(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index b1bd3de..2191e50 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -1,6 +1,6 @@ name: Unit Test in NSLS-II Conda Environment -on: +on: workflow_call: inputs: conda_env_name: @@ -21,120 +21,117 @@ on: type: string jobs: - test-in-conda-env: - runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: "bash -l {0}" steps: - - - name: Set env vars - run: | - export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo - echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - - export CONDA_ENV_NAME=${{ inputs.conda_env_name }} - echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - - export ZENODO_ID=${{ inputs.zenodo_id }} - echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - - export MD5_CHECKSUM=${{ inputs.md5_checksum }} - echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV - - # Workaround for unset GDAL env variables used by newer conda envs - export GDAL_DATA="${GDAL_DATA-''}" - echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV - export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" - echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV - export GEOTIFF_CSV="${GEOTIFF_CSV-''}" - echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV - - # Workaround for unset MKL env variables used by newer conda envs - export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" - echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV - - - name: Check out the code repo - uses: actions/checkout@v4 - - - name: Workaround: Fix .condarc MultipleKeysError - run: | - sed -i '/auto_activate_base/d' /home/runner/.condarc || true - sed -i '/auto_activate:/d' /home/runner/.condarc || true - - - name: Set up Python ${{ inputs.python-version }} with conda - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }} - auto-update-conda: true - miniconda-version: "latest" - python-version: ${{ inputs.python-version }} - mamba-version: "*" - channels: conda-forge - - - name: Cache the archived conda environment - uses: actions/cache@v3 - with: - path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} - env: - # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed - CACHE_NUMBER: 0 - id: cache-conda-archive - - - name: Download and install the archived conda environment - if: steps.cache-conda-archive.outputs.cache-hit != 'true' - run: | - set -vxeuo pipefail - conda activate "${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }}" - url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" - wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz - status=$? - if [ $status -gt 0 ]; then - echo "Cannot download from ${url}. Exit code: ${status}" - exit $status - fi - echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt - md5sum --check checksum.txt - mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} - tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} - - - name: Activate and inspect the archived conda environment - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - conda unpack - - conda info -a - conda env list - conda list - pip list - - - name: Install the package and its dependencies - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - echo "Using conda environment at ${CONDA_PREFIX}" - pip install --upgrade pip setuptools wheel - pip install -r requirements-dev.txt - pip install -r requirements-extras.txt - pip install -e . - pip list - - - name: Test with pytest - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - echo "Using conda environment at ${CONDA_PREFIX}" - coverage run --source=csxtools run_tests.py - coverage xml - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - file: ./coverage.xml - flags: conda-env-unittests + - name: Set env vars + run: | + export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo + echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV + + export CONDA_ENV_NAME=${{ inputs.conda_env_name }} + echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + + export ZENODO_ID=${{ inputs.zenodo_id }} + echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV + + export MD5_CHECKSUM=${{ inputs.md5_checksum }} + echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + + # Workaround for unset GDAL env variables used by newer conda envs + export GDAL_DATA="${GDAL_DATA-''}" + echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV + export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" + echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV + export GEOTIFF_CSV="${GEOTIFF_CSV-''}" + echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV + + # Workaround for unset MKL env variables used by newer conda envs + export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" + echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV + + - name: Check out the code repo + uses: actions/checkout@v4 + + - name: "Workaround: Fix .condarc MultipleKeysError" + run: | + sed -i '/auto_activate_base/d' /home/runner/.condarc || true + sed -i '/auto_activate:/d' /home/runner/.condarc || true + + - name: Set up Python ${{ inputs.python-version }} with conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }} + auto-update-conda: true + miniconda-version: "latest" + python-version: ${{ inputs.python-version }} + mamba-version: "*" + channels: conda-forge + + - name: Cache the archived conda environment + uses: actions/cache@v3 + with: + path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} + key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed + CACHE_NUMBER: 0 + id: cache-conda-archive + + - name: Download and install the archived conda environment + if: steps.cache-conda-archive.outputs.cache-hit != 'true' + run: | + set -vxeuo pipefail + conda activate "${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }}" + url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" + wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz + status=$? + if [ $status -gt 0 ]; then + echo "Cannot download from ${url}. Exit code: ${status}" + exit $status + fi + echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt + md5sum --check checksum.txt + mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} + tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} + + - name: Activate and inspect the archived conda environment + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + conda unpack + + conda info -a + conda env list + conda list + pip list + + - name: Install the package and its dependencies + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + pip install --upgrade pip setuptools wheel + pip install -r requirements-dev.txt + pip install -r requirements-extras.txt + pip install -e . + pip list + + - name: Test with pytest + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + coverage run --source=csxtools run_tests.py + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + file: ./coverage.xml + flags: conda-env-unittests From 35600dd77b6a9bf5a4ec42f572346b8f62aa9a57 Mon Sep 17 00:00:00 2001 From: nisar Date: Wed, 27 Aug 2025 11:36:24 -0400 Subject: [PATCH 09/11] Replace pytest-pep8 with pytest-flake8 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cc9e74d..dabff92 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,6 @@ codecov coveralls flake8 pytest -pytest-pep8 +pytest-flake8 sphinx sphinx-bootstrap-theme From b2c24c5a81b7034ced739f217f2c3896d5ed6a7f Mon Sep 17 00:00:00 2001 From: nisar Date: Wed, 27 Aug 2025 12:11:29 -0400 Subject: [PATCH 10/11] Some fixes in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index 2191e50..cce24d1 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -117,7 +117,12 @@ jobs: conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} echo "Using conda environment at ${CONDA_PREFIX}" pip install --upgrade pip setuptools wheel - pip install -r requirements-dev.txt + + # Fix backports issue + pip uninstall -y backports || true + pip install backports.tarfile + + pip install -r requirements-dev.txt pip install -r requirements-extras.txt pip install -e . pip list @@ -135,3 +140,4 @@ jobs: with: file: ./coverage.xml flags: conda-env-unittests + From 94cd022d3b8a02c0112c7f0e0e0377dbcb9cf145 Mon Sep 17 00:00:00 2001 From: nisar Date: Wed, 27 Aug 2025 12:19:42 -0400 Subject: [PATCH 11/11] Some fixes in _test-in-conda-env.yml --- .github/workflows/_test-in-conda-env.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index cce24d1..d1cf0af 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -117,12 +117,9 @@ jobs: conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} echo "Using conda environment at ${CONDA_PREFIX}" pip install --upgrade pip setuptools wheel - - # Fix backports issue pip uninstall -y backports || true pip install backports.tarfile - - pip install -r requirements-dev.txt + pip install -r requirements-dev.txt pip install -r requirements-extras.txt pip install -e . pip list