diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index a2e2c40..d1cf0af 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -1,6 +1,6 @@ name: Unit Test in NSLS-II Conda Environment -on: +on: workflow_call: inputs: conda_env_name: @@ -21,123 +21,120 @@ on: type: string jobs: - test-in-conda-env: - runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: "bash -l {0}" steps: + - name: Set env vars + run: | + export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo + echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV + + export CONDA_ENV_NAME=${{ inputs.conda_env_name }} + echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + + export ZENODO_ID=${{ inputs.zenodo_id }} + echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV + + export MD5_CHECKSUM=${{ inputs.md5_checksum }} + echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + + # Workaround for unset GDAL env variables used by newer conda envs + export GDAL_DATA="${GDAL_DATA-''}" + echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV + export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" + echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV + export GEOTIFF_CSV="${GEOTIFF_CSV-''}" + echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV + + # Workaround for unset MKL env variables used by newer conda envs + export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" + echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV + + - name: Check out the code repo + uses: actions/checkout@v4 + + - name: "Workaround: Fix .condarc MultipleKeysError" + run: | + sed -i '/auto_activate_base/d' /home/runner/.condarc || true + sed -i '/auto_activate:/d' /home/runner/.condarc || true + + - name: Set up Python ${{ inputs.python-version }} with conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }} + auto-update-conda: true + miniconda-version: "latest" + python-version: ${{ inputs.python-version }} + mamba-version: "*" + channels: conda-forge + + - name: Cache the archived conda environment + uses: actions/cache@v3 + with: + path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} + key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}-${{ env.MD5_CHECKSUM }}-${{ env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed + CACHE_NUMBER: 0 + id: cache-conda-archive + + - name: Download and install the archived conda environment + if: steps.cache-conda-archive.outputs.cache-hit != 'true' + run: | + set -vxeuo pipefail + conda activate "${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }}" + url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" + wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz + status=$? + if [ $status -gt 0 ]; then + echo "Cannot download from ${url}. Exit code: ${status}" + exit $status + fi + echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt + md5sum --check checksum.txt + mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} + tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} + + - name: Activate and inspect the archived conda environment + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + conda unpack + + conda info -a + conda env list + conda list + pip list + + - name: Install the package and its dependencies + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + pip install --upgrade pip setuptools wheel + pip uninstall -y backports || true + pip install backports.tarfile + pip install -r requirements-dev.txt + pip install -r requirements-extras.txt + pip install -e . + pip list + + - name: Test with pytest + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + coverage run --source=csxtools run_tests.py + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + file: ./coverage.xml + flags: conda-env-unittests - - name: Set env vars - run: | - export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo - echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - - export CONDA_ENV_NAME='${{ inputs.conda_env_name }}' - echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - - export ZENODO_ID='${{ inputs.zenodo_id }}' - echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - - export MD5_CHECKSUM='${{ inputs.md5_checksum }}' - echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV - - # Workaround for unset GDAL env variables used by newer conda envs - export GDAL_DATA="${GDAL_DATA-''}" - echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV - export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" - echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV - export GEOTIFF_CSV="${GEOTIFF_CSV-''}" - echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV - - # Workaround for unset MKL env variables used by newer conda envs - export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" - echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV - - - name: Check out the code repo - uses: actions/checkout@v4 - - - name: Workaround: Fix .condarc MultipleKeysError - run: | - sed -i '/auto_activate_base/d' /home/runner/.condarc || true - sed -i '/auto_activate:/d' /home/runner/.condarc || true - - - name: Set up Python ${{ inputs.python-version }} with conda - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }} - auto-update-conda: true - miniconda-version: "latest" - python-version: ${{ inputs.python-version }} - mamba-version: "*" - channels: conda-forge - - - name: Cache the archived conda environment - uses: actions/cache@v3 - with: - path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: - ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ - env.MD5_CHECKSUM }}-${{ - env.CACHE_NUMBER }} - env: - # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed - CACHE_NUMBER: 0 - id: cache-conda-archive - - - name: Download and install the archived conda environment - if: steps.cache-conda-archive.outputs.cache-hit != 'true' - run: | - set -vxeuo pipefail - conda activate "${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }}" - url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" - wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz - status=$? - if [ $status -gt 0 ]; then - echo "Cannot download from ${url}. Exit code: ${status}" - exit $status - fi - echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt - md5sum --check checksum.txt - mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} - tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} - - - name: Activate and inspect the archived conda environment - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - conda unpack - - conda info -a - conda env list - conda list - pip list - - - name: Install the package and its dependencies - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - echo "Using conda environment at ${CONDA_PREFIX}" - pip install --upgrade pip setuptools wheel - pip install -r requirements-dev.txt - pip install -r requirements-extras.txt - pip install -e . - pip list - - - name: Test with pytest - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - echo "Using conda environment at ${CONDA_PREFIX}" - coverage run --source=csxtools run_tests.py - coverage xml - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - file: ./coverage.xml - flags: conda-env-unittests diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index aa0673a..d5a40b7 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -1,29 +1,29 @@ -name: Unit Tests in NSLS-II Conda Environments +name: Run tests in NSLS-II Conda Environments -on: [push, pull_request, workflow_dispatch] - -env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} +on: + push: + branches: [main] + pull_request: jobs: - + # First prepare the matrix matrix_prep: - uses: ./.github/workflows/_matrix_prep.yml with: config_prefix: recent + # Then run tests using that matrix test-in-conda-env: - needs: matrix_prep + strategy: + max-parallel: 4 + fail-fast: false + matrix: + include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + uses: ./.github/workflows/_test-in-conda-env.yml with: conda_env_name: ${{ matrix.conda_env_name }} zenodo_id: ${{ matrix.zenodo_id }} md5_checksum: ${{ matrix.md5_checksum }} python-version: ${{ matrix.python-version }} - strategy: - max-parallel: 4 - matrix: - include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} - fail-fast: false diff --git a/requirements-dev.txt b/requirements-dev.txt index cc9e74d..dabff92 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,6 @@ codecov coveralls flake8 pytest -pytest-pep8 +pytest-flake8 sphinx sphinx-bootstrap-theme