Skip to content

Commit 0a8164f

Browse files
tomkinsctomkinsc
authored
Merge branch 'master' into fn_update_docker
2 parents 2faa74e + 79337d9 commit 0a8164f

16 files changed

+295
-137
lines changed

‎.github/workflows/build.yml‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
name: "viral-classify CI"
2+
3+
on:
4+
push:
5+
pull_request:
6+
branches:
7+
- master
8+
release:
9+
types:
10+
- created
11+
schedule:
12+
- cron: '0 18 * * 1' # weekly at 18:00 on Mondays
13+
14+
env:
15+
HOME: "${{ github.workspace }}"
16+
CACHE_DIR: "${{ github.workspace }}/misc_cache"
17+
MINICONDA_DIR: "${{ github.workspace }}/miniconda"
18+
GATK_PATH: "${{ github.workspace }}/misc_cache"
19+
PYTHONIOENCODING: UTF8
20+
21+
DOCKER_REGISTRY: "quay.io"
22+
DOCKER_REPO_PROD: "quay.io/broadinstitute/viral-classify"
23+
DOCKER_REPO_DEV: "quay.io/broadinstitute/viral-classify"
24+
25+
#BOTO_CONFIG: /dev/null # bogus value to override config on travis
26+
_JAVA_OPTIONS: "-Xmx3g" # overrides jvm opts set elsewhere; see: https://stackoverflow.com/questions/28327620/difference-between-java-options-java-tool-options-and-java-opts
27+
28+
# TravisCI variables described here:
29+
# https://docs.travis-ci.com/user/environment-variables/
30+
# GitHub Actions environment variables and context described here:
31+
# https://docs.github.com/en/actions/reference/environment-variables
32+
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#env-context
33+
GITHUB_ACTIONS_COMMIT: ${{ github.sha }}
34+
GITHUB_ACTIONS_BUILD_DIR: ${{ github.workspace }}
35+
#GITHUB_ACTIONS_BRANCH: ${{ github.base_ref }}
36+
GITHUB_ACTIONS_BRANCH: ${{ github.ref }}
37+
GITHUB_ACTIONS_PULL_REQUEST: ${{ github.event.number }}
38+
GITHUB_ACTIONS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
39+
GITHUB_ACTIONS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
40+
GITHUB_ACTIONS_BASE_REF: ${{ github.base_ref }}
41+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
42+
43+
jobs:
44+
build_docker:
45+
runs-on: ubuntu-20.04
46+
steps:
47+
- name: checkout repository
48+
uses: actions/checkout@v3
49+
# fetch git tags (tagged releases) because
50+
# actions/checkout@v3 does either a full checkout or a shallow checkout without tags
51+
- name: fetch tags
52+
run: git fetch --prune --unshallow --tags
53+
- name: Programmatic environment setup
54+
run: |
55+
set -e -x
56+
# $GITHUB_ENV is available for subsequent steps
57+
GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '')
58+
echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV
59+
#
60+
# Set GITHUB_ACTIONS_BRANCH
61+
# TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ )
62+
# for push builds, or builds not triggered by a pull request, this is the name of the branch.
63+
# for builds triggered by a pull request this is the name of the branch targeted by the pull request.
64+
# for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG).
65+
# if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build
66+
if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then
67+
GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/}
68+
# if event_name=="release", this is a tagged build
69+
elif [[ "${{ github.event_name }}" == "release" ]]; then
70+
GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG
71+
else
72+
GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
73+
fi
74+
echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
75+
echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
76+
77+
- name: Login to docker registry
78+
uses: docker/login-action@v2
79+
with:
80+
registry: ${{ env.DOCKER_REGISTRY }}
81+
username: ${{ secrets.QUAY_USERNAME }}
82+
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
83+
# cache; see: https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
84+
- name: setup cache
85+
id: docker_cache
86+
uses: actions/cache@v3
87+
env:
88+
cache-name: old-docker-tag
89+
with:
90+
path: "$CACHE_DIR"
91+
key: ${{ runner.os }}-${{ env.cache-name }}
92+
# - name: Pull old docker image from cache
93+
# if: steps.docker_cache.outputs.cache-hit != 'true'
94+
# run: |
95+
- name: Attempt to pull older image from cache
96+
if: steps.docker_cache.outputs.cache-hit == 'true'
97+
shell: bash
98+
run: |
99+
set -e
100+
# restore old tag from cache if present
101+
if [ -f "$CACHE_DIR/old-docker-tag.txt" ]; then
102+
OLD_DOCKER_TAG=$(cat $CACHE_DIR/old-docker-tag.txt)
103+
else
104+
OLD_DOCKER_TAG=$DOCKER_REPO_PROD
105+
fi
106+
echo "old docker tag = $OLD_DOCKER_TAG"
107+
108+
# attempt to pull tag from cache
109+
if docker pull $OLD_DOCKER_TAG; then
110+
echo _CACHE_FROM="--cache-from $OLD_DOCKER_TAG" >> $GITHUB_ENV
111+
else
112+
echo "_CACHE_FROM=" >> $GITHUB_ENV
113+
fi
114+
- name: Build docker image
115+
shell: bash
116+
run: |
117+
set -e -x
118+
git describe --tags --always | tee VERSION
119+
120+
if [ -n "$GITHUB_ACTIONS_TAG" ]; then
121+
echo "Release $GITHUB_ACTIONS_TAG"
122+
elif [ -n "$GITHUB_ACTIONS_PULL_REQUEST_BRANCH" ]; then
123+
echo "LABEL quay.expires-after=10w" >> Dockerfile
124+
elif [[ "$GITHUB_ACTIONS_BRANCH" != "master" ]]; then
125+
echo "LABEL quay.expires-after=10w" >> Dockerfile
126+
fi
127+
128+
echo "Building with cache from: $_CACHE_FROM"
129+
docker build -t local/build-container:build $_CACHE_FROM .
130+
- name: Deploy docker image
131+
run: |
132+
github_actions_ci/deploy-docker.sh
133+
- name: store tag ID in cache
134+
run: |
135+
mkdir -p $CACHE_DIR
136+
github_actions_ci/list-docker-tags.sh | tail -1 | tee $CACHE_DIR/old-docker-tag.txt
137+
138+
test_py36_in_docker:
139+
needs: build_docker
140+
runs-on: ubuntu-20.04
141+
env:
142+
GITHUB_ACTIONS_PYTHON_VERSION: 3.8
143+
PYTEST_ADDOPTS: "-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov read_utils --cov reports --cov tools --cov util --cov file_utils"
144+
steps:
145+
- name: checkout repository
146+
uses: actions/checkout@v3
147+
# fetch git tags (tagged releases) because
148+
# actions/checkout@v3 does either a full checkout or a shallow checkout without tags
149+
- name: fetch tags
150+
run: git fetch --prune --unshallow --tags
151+
- name: Programmatic environment setup
152+
run: |
153+
set -e -x
154+
# $GITHUB_ENV is available for subsequent steps
155+
GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '')
156+
echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV
157+
#
158+
# Set GITHUB_ACTIONS_BRANCH
159+
# TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ )
160+
# for push builds, or builds not triggered by a pull request, this is the name of the branch.
161+
# for builds triggered by a pull request this is the name of the branch targeted by the pull request.
162+
# for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG).
163+
# if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build
164+
if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then
165+
GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/}
166+
# if event_name=="release", this is a tagged build
167+
elif [[ "${{ github.event_name }}" == "release" ]]; then
168+
GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG
169+
else
170+
GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
171+
fi
172+
echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
173+
echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
174+
- name: install python
175+
uses: actions/setup-python@v4
176+
with:
177+
python-version: "${{ env.GITHUB_ACTIONS_PYTHON_VERSION }}"
178+
- name: Set up Docker Buildx
179+
uses: docker/setup-buildx-action@v2
180+
- name: pull docker image
181+
run: |
182+
set -e -x
183+
DOCKER_TAG=$(github_actions_ci/list-docker-tags.sh | tail -1)
184+
echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
185+
echo "pulling $DOCKER_TAG"
186+
docker pull $DOCKER_TAG
187+
mkdir coverage
188+
- name: test with docker
189+
run: |
190+
docker run -e _JAVA_OPTIONS -e PYTEST_ADDOPTS -v `pwd`/coverage:/coverage -v `pwd`/test:/opt/viral-ngs/source/test --entrypoint /bin/bash $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/source; pytest test/unit; cp .coverage /coverage'
191+
- name: run coveralls
192+
run: |
193+
mv coverage/.coverage .
194+
pip install coveralls>=1.3.0
195+
coveralls --service=github
196+
197+
## note: this test_docs job does not actually produce the output on readthedocs
198+
## readthedocs does its own build trigger. this job exists simply to alert us
199+
## of build failures of the docs because otherwise we would never know.
200+
test_docs:
201+
needs: build_docker
202+
runs-on: ubuntu-20.04
203+
steps:
204+
- name: checkout repository
205+
uses: actions/checkout@v3
206+
# fetch git tags (tagged releases) because
207+
# actions/checkout@v3 does either a full checkout or a shallow checkout without tags
208+
- name: fetch tags
209+
run: git fetch --prune --unshallow --tags
210+
- name: Programmatic environment setup
211+
run: |
212+
set -e -x
213+
# $GITHUB_ENV is available for subsequent steps
214+
GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '')
215+
echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV
216+
#
217+
# Set GITHUB_ACTIONS_BRANCH
218+
# TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ )
219+
# for push builds, or builds not triggered by a pull request, this is the name of the branch.
220+
# for builds triggered by a pull request this is the name of the branch targeted by the pull request.
221+
# for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG).
222+
# if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build
223+
if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then
224+
GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/}
225+
# if event_name=="release", this is a tagged build
226+
elif [[ "${{ github.event_name }}" == "release" ]]; then
227+
GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG
228+
else
229+
GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
230+
fi
231+
echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
232+
echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
233+
- name: Set up Docker Buildx
234+
uses: docker/setup-buildx-action@v2
235+
- name: pull docker image
236+
run: |
237+
set -e -x
238+
DOCKER_TAG=$(github_actions_ci/list-docker-tags.sh | tail -1)
239+
echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
240+
echo "pulling $DOCKER_TAG"
241+
docker pull $DOCKER_TAG
242+
- name: test building docs
243+
run: |
244+
set -e -x
245+
docker run --entrypoint /bin/bash -v `pwd`:/opt/viral-ngs/source $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/source; github_actions_ci/install-pip-docs.sh; cd docs; make html'

‎.readthedocs.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ version: 2
77

88
# Build documentation in the docs/ directory with Sphinx
99
sphinx:
10+
builder: html
1011
configuration: docs/conf.py
1112

1213
# Build documentation with MkDocs
@@ -18,6 +19,6 @@ formats: all
1819

1920
# Optionally set the version of Python and requirements required to build your docs
2021
python:
21-
version: 3.6
22+
version: 3.8
2223
install:
2324
- requirements: docs/requirements.txt

‎.travis.yml‎

Lines changed: 0 additions & 84 deletions
This file was deleted.

‎Dockerfile‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
1-
FROM quay.io/broadinstitute/viral-core:2.1.33
1+
FROM quay.io/broadinstitute/viral-core:2.2.1
22

33
LABEL maintainer "[email protected]"
44

55
ENV VIRAL_CLASSIFY_PATH=$INSTALL_PATH/viral-classify \
66
PATH="$PATH:$MINICONDA_PATH/envs/env2/bin"
77

8-
COPY requirements-conda.txt requirements-conda-env2.txt $VIRAL_CLASSIFY_
8+
COPY requirements-conda.txt requirements-conda-env2.txt $VIRAL_CLASSIFY_PATH/
9+
# install most dependencies to the main environment
910
RUN $VIRAL_NGS_PATH/docker/install-conda-dependencies.sh $VIRAL_CLASSIFY_PATH/requirements-conda.txt
10-
RUN CONDA_PREFIX="$MINICONDA_PATH/envs/env2"; conda config --set channel_priority strict; conda create -q -y -n env2; $VIRAL_NGS_PATH/docker/install-conda-dependencies.sh $VIRAL_CLASSIFY_PATH/requirements-conda-env2.txt
11+
12+
# install packages with dependency incompatibilities to the second environment
13+
RUN CONDA_PREFIX="$MINICONDA_PATH/envs/env2"; \
14+
conda config --set channel_priority strict; \
15+
conda create -q -y -n env2; \
16+
$VIRAL_NGS_PATH/docker/install-conda-dependencies.sh $VIRAL_CLASSIFY_PATH/requirements-conda-env2.txt
1117

1218
# Copy all source code into the base repo
1319
# (this probably changes all the time, so all downstream build

‎docs/conf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def _get_viral_core():
7676

7777
# General information about the project.
7878
project = 'viral-ngs'
79-
copyright = '2015, Broad Institute Viral Genomics'
79+
copyright = '2023, Broad Institute Viral Genomics'
8080

8181
# The version info for the project you're documenting, acts as replacement for
8282
# |version| and |release|, also used in various other places throughout the

‎docs/requirements.txt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
Sphinx>=5.3.0
2-
sphinx-jinja2-compat
3-
sphinx-argparse>=0.1.15
4-
sphinx_rtd_theme>=0.1.9
5-
matplotlib>=2.2.4
6-
mock>=2.0.0
1+
jinja2==3.1.2 # https://github.com/readthedocs/readthedocs.org/issues/9037#issuecomment-1077818554
2+
Sphinx==5.3.0 #override sphinx pinning done by RTD: https://docs.readthedocs.io/en/stable/build-default-versions.html#external-dependencies
3+
sphinx-argparse
4+
sphinx-rtd-theme==1.1.1
5+
matplotlib==2.2.4
6+
PyYAML==6.0
7+
mock==5.0.1
8+
recommonmark

0 commit comments

Comments
 (0)