Skip to content

Commit 74e89d3

Browse files
oliverholworthymarcromeyn
oliverholworthy
and
marcromeyn
authored
Add actionlint job to lint check in GitHub Workflows (#1119)
* Add actionlint job to check GitHub Workflows * Update formatting of GitHub Workflow files * correct indentation in cpu-horovod --------- Co-authored-by: Marc Romeyn <[email protected]>
1 parent efaa5c0 commit 74e89d3

17 files changed

+176
-172
lines changed

‎.github/actionlint-matcher.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"problemMatcher": [
3+
{
4+
"owner": "actionlint",
5+
"pattern": [
6+
{
7+
"regexp": "^(?:\\x1b\\[\\d+m)?(.+?)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*: (?:\\x1b\\[\\d+m)*(.+?)(?:\\x1b\\[\\d+m)* \\[(.+?)\\]$",
8+
"file": 1,
9+
"line": 2,
10+
"column": 3,
11+
"message": 4,
12+
"code": 5
13+
}
14+
]
15+
}
16+
]
17+
}

‎.github/actionlint.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
self-hosted-runner:
2+
# Labels of self-hosted runner in array of string
3+
labels:
4+
- 1GPU
5+
- 2GPU

‎.github/workflows/cpu-horovod.yml‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ jobs:
2727
- name: Get changed backends
2828
id: backend_check
2929
run: |
30-
echo "changed=$(python ci/get_changed_backends.py --backend tensorflow --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
30+
echo "changed=$(python ci/get_changed_backends.py --backend tensorflow --branch ${{github.base_ref}})" >> "$GITHUB_OUTPUT"
3131
outputs:
3232
needs_testing: ${{ steps.backend_check.outputs.changed }}
33-
33+
3434
tests:
3535
needs: check-changes
3636
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
@@ -56,24 +56,20 @@ jobs:
5656
wget https://repo.anaconda.com/miniconda/Miniconda3-py38_22.11.1-1-Linux-x86_64.sh -O /tmp/miniconda/miniconda.sh
5757
bash /tmp/miniconda/miniconda.sh -b -u -p /tmp/miniconda
5858
/tmp/miniconda/bin/conda init bash
59-
echo "/tmp/miniconda/bin" >> $GITHUB_PATH
59+
echo "/tmp/miniconda/bin" >> "$GITHUB_PATH"
6060
- name: Install and upgrade python packages
6161
run: |
6262
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
6363
- name: Install tox-conda
6464
run: |
6565
python -m pip install tox-conda
66+
- name: Get Merlin Branch name
67+
id: get-branch-name
68+
uses: NVIDIA-Merlin/.github/actions/branch-name@main
6669
- name: Run tests
6770
run: |
68-
ref_type=${{ github.ref_type }}
69-
branch=main
70-
pr_flag="false"
71-
if [[ $ref_type == "tag"* ]]
72-
then
73-
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
74-
branch=$(git branch -r --contains ${{ github.ref_name }} --list '*release*' --format "%(refname:short)" | sed -e 's/^origin\///')
75-
fi
71+
merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
7672
if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
7773
extra_pytest_markers="and changed"
7874
fi
79-
cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-horovod-cpu
75+
EXTRA_PYTEST_MARKERS="$extra_pytest_markers" MERLIN_BRANCH="$merlin_branch" COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-horovod-cpu

‎.github/workflows/cpu-nvtabular.yml‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ jobs:
3030
- name: Get changed backends
3131
id: backend_check
3232
run: |
33-
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
33+
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> "$GITHUB_OUTPUT"
3434
outputs:
3535
needs_testing: ${{ steps.backend_check.outputs.changed }}
36-
36+
3737
tests:
3838
needs: check-changes
3939
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
@@ -58,13 +58,10 @@ jobs:
5858
- name: Install and upgrade python packages
5959
run: |
6060
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
61+
- name: Get Merlin Branch name
62+
id: get-branch-name
63+
uses: NVIDIA-Merlin/.github/actions/branch-name@main
6164
- name: Run tests
6265
run: |
63-
ref_type=${{ github.ref_type }}
64-
branch=main
65-
if [[ $ref_type == "tag"* ]]
66-
then
67-
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
68-
branch=$(git branch -r --contains ${{ github.ref_name }} --list '*release*' --format "%(refname:short)" | sed -e 's/^origin\///')
69-
fi
70-
GIT_COMMIT=`git rev-parse HEAD` tox -e py38-nvtabular-cpu -- $branch
66+
merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
67+
MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e py38-nvtabular-cpu

‎.github/workflows/cpu-systems.yml‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ jobs:
3030
- name: Get changed backends
3131
id: backend_check
3232
run: |
33-
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
33+
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> "$GITHUB_OUTPUT"
3434
outputs:
3535
needs_testing: ${{ steps.backend_check.outputs.changed }}
36-
36+
3737
tests:
3838
needs: check-changes
3939
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
@@ -58,13 +58,10 @@ jobs:
5858
- name: Install and upgrade python packages
5959
run: |
6060
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
61+
- name: Get Merlin Branch name
62+
id: get-branch-name
63+
uses: NVIDIA-Merlin/.github/actions/branch-name@main
6164
- name: Run tests
6265
run: |
63-
ref_type=${{ github.ref_type }}
64-
branch=main
65-
if [[ $ref_type == "tag"* ]]
66-
then
67-
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
68-
branch=$(git branch -r --contains ${{ github.ref_name }} --list '*release*' --format "%(refname:short)" | sed -e 's/^origin\///')
69-
fi
70-
GIT_COMMIT=`git rev-parse HEAD` tox -e py38-systems-cpu -- $branch
66+
merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
67+
MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e py38-systems-cpu

‎.github/workflows/cpu-t4r.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ jobs:
2626
- name: Get changed backends
2727
id: backend_check
2828
run: |
29-
echo "changed=$(python ci/get_changed_backends.py --backend 'datasets|torch' --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
29+
echo "changed=$(python ci/get_changed_backends.py --backend 'datasets|torch' --branch ${{github.base_ref}})" >> "$GITHUB_OUTPUT"
3030
outputs:
3131
needs_testing: ${{ steps.backend_check.outputs.changed }}
32-
32+
3333
tests:
3434
needs: check-changes
3535
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
@@ -54,10 +54,10 @@ jobs:
5454
- name: Install and upgrade python packages
5555
run: |
5656
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
57-
- name: Get Branch name
57+
- name: Get Merlin Branch name
5858
id: get-branch-name
5959
uses: NVIDIA-Merlin/.github/actions/branch-name@main
6060
- name: Run tests
6161
run: |
62-
branch="${{ steps.get-branch-name.outputs.branch }}"
63-
GIT_COMMIT=`git rev-parse HEAD` tox -e py38-transformers4rec-cpu -- $branch
62+
merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
63+
MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e py38-transformers4rec-cpu

‎.github/workflows/datasets.yml‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ jobs:
2727
- name: Get changed backends
2828
id: backend_check
2929
run: |
30-
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
30+
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> "$GITHUB_OUTPUT"
3131
outputs:
3232
needs_testing: ${{ steps.backend_check.outputs.changed }}
33-
33+
3434
tests:
3535
needs: check-changes
3636
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
@@ -52,22 +52,19 @@ jobs:
5252
run: |
5353
sudo apt-get update -y
5454
sudo apt-get install -y protobuf-compiler
55+
- name: Get Merlin Branch name
56+
id: get-branch-name
57+
uses: NVIDIA-Merlin/.github/actions/branch-name@main
5558
- name: Install Merlin dependencies
5659
run: |
57-
ref_type=${{ github.ref_type }}
58-
branch=main
59-
if [[ $ref_type == "tag"* ]]
60-
then
61-
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
62-
branch=$(git branch -r --contains ${{ github.ref_name }} --list '*release*' --format "%(refname:short)" | sed -e 's/^origin\///')
63-
fi
60+
merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
6461
pip install "pandas>=1.2.0,<1.4.0dev0"
65-
pip install "NVTabular@git+https://github.com/NVIDIA-Merlin/NVTabular.git@$branch"
66-
pip install "merlin-core@git+https://github.com/NVIDIA-Merlin/core.git@$branch"
62+
pip install "NVTabular@git+https://github.com/NVIDIA-Merlin/NVTabular.git@$merlin_branch"
63+
pip install "merlin-core@git+https://github.com/NVIDIA-Merlin/core.git@$merlin_branch"
6764
- name: Install dependencies
6865
run: |
6966
python -m pip install --upgrade pip
70-
python -m pip install .[dev]
67+
python -m pip install ".[dev]"
7168
- name: Build
7269
run: |
7370
python setup.py develop
@@ -77,4 +74,4 @@ jobs:
7774
make tests-datasets
7875
else
7976
make tests-datasets-changed
80-
fi
77+
fi

‎.github/workflows/docs-sched-rebuild.yaml‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
- name: Copy HTML directories
6161
run: |
6262
ls -asl
63-
for i in `ls -d *`
63+
for i in */
6464
do
6565
echo "Git adding ${i}"
6666
git add "${i}"
@@ -85,25 +85,26 @@ jobs:
8585
# If any of these commands fail, fail the build.
8686
def_branch=$(gh api "repos/${GITHUB_REPOSITORY}" --jq ".default_branch")
8787
html_url=$(gh api "repos/${GITHUB_REPOSITORY}/pages" --jq ".html_url")
88-
# Beware ugly quotation mark avoidance in the foll lines.
89-
echo '<!DOCTYPE html>' > index.html
90-
echo '<html>' >> index.html
91-
echo ' <head>' >> index.html
92-
echo ' <title>Redirect to documentation</title>' >> index.html
93-
echo ' <meta charset="utf-8">' >> index.html
94-
echo ' <meta http=equiv="refresh" content="3; URL='${html_url}${def_branch}'/index.html">' >> index.html
95-
echo ' <link rel="canonical" href="'${html_url}${def_branch}'/index.html">' >> index.html
96-
echo ' <script language="javascript">' >> index.html
97-
echo ' function redirect() {' >> index.html
98-
echo ' window.location.assign("'${html_url}${def_branch}'/index.html")' >> index.html
99-
echo ' }' >> index.html
100-
echo ' </script>' >> index.html
101-
echo ' </head>' >> index.html
102-
echo ' <body onload="redirect()">' >> index.html
103-
echo ' <p>Please follow the link to the <a href="'${html_url}${def_branch}'/index.html">' >> index.html
104-
echo ${def_branch}'</a> branch documentation.</p>' >> index.html
105-
echo ' </body>' >> index.html
106-
echo '</html>' >> index.html
88+
cat > index.html << EOF
89+
<!DOCTYPE html>
90+
<html>
91+
<head>
92+
<title>Redirect to documentation</title>
93+
<meta charset="utf-8">
94+
<meta http=equiv="refresh" content="3; URL="${html_url}${def_branch}/index.html"
95+
<link rel="canonical" href="'${html_url}${def_branch}/index.html">
96+
<script language="javascript">
97+
function redirect() {
98+
window.location.assign("${html_url}${def_branch}/index.html")
99+
}
100+
</script>
101+
</head>
102+
<body onload="redirect()">
103+
<p>Please follow the link to the <a href="${html_url}${def_branch}/index.html">
104+
${def_branch}'</a> branch documentation.</p>
105+
</body>
106+
</html>
107+
EOF
107108
git add index.html
108109
- name: Commit changes to the GitHub Pages branch
109110
run: |

‎.github/workflows/gpu-multi.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ jobs:
3131
- name: Get changed backends
3232
id: backend_check
3333
run: |
34-
echo "changed=$(python ci/get_changed_backends.py --backend tensorflow --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
34+
echo "changed=$(python ci/get_changed_backends.py --backend tensorflow --branch ${{github.base_ref}})" >> "$GITHUB_OUTPUT"
3535
outputs:
3636
needs_testing: ${{ steps.backend_check.outputs.changed }}
37-
37+
3838
tensorflow:
3939
needs: check-changes-tf
4040
if: ${{needs.check-changes-tf.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
@@ -48,7 +48,6 @@ jobs:
4848
run: |
4949
ref_type=${{ github.ref_type }}
5050
branch=main
51-
pr_flag="false"
5251
if [[ $ref_type == "tag"* ]]
5352
then
5453
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
@@ -57,4 +56,4 @@ jobs:
5756
if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
5857
extra_pytest_markers="and changed"
5958
fi
60-
cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-multi-gpu
59+
cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-multi-gpu

‎.github/workflows/gpu.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ jobs:
2626
run: |
2727
ref_type=${{ github.ref_type }}
2828
branch=main
29-
pr_flag="false"
3029
if [[ $ref_type == "tag"* ]]
3130
then
3231
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*

0 commit comments

Comments
 (0)