Skip to content

Commit e210c33

Browse files
lanluo-nvidialanluo-nvidia
committed
add build test workflow
1 parent b986d16 commit e210c33

File tree

4 files changed

+257
-0
lines changed

4 files changed

+257
-0
lines changed

‎.github/scripts/generate-matrix.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import copy
5+
import json
6+
import sys
7+
8+
9+
import requests # type: ignore[import-untyped]
10+
11+
# please update the cuda version/python version/tensorRT version you want to test
12+
# TODO: add cu130 support
13+
# # build and upload the ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda13.0-ubuntu-llvm17
14+
CUDA_VERSIONS_DICT = {
15+
"nightly": ["cu129"],
16+
"test": ["cu129"],
17+
"release": ["cu129"],
18+
}
19+
20+
TRT_VERSIONS_DICT = {
21+
"nightly": ["10.12", "10.13"],
22+
"test": ["10.12"],
23+
"release": ["10.12", "10.13"],
24+
}
25+
26+
DOCKER_IMAGE_DICT = {
27+
"nightly": {
28+
"cu129": "ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.9-ubuntu-llvm17",
29+
},
30+
"test": {
31+
"cu129": "ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.9-ubuntu-llvm17",
32+
},
33+
"release": {
34+
"cu129": "ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.9-ubuntu-llvm17",
35+
},
36+
}
37+
38+
def main(args: list[str]) -> None:
39+
parser = argparse.ArgumentParser()
40+
parser.add_argument(
41+
"--channel",
42+
help="channel",
43+
type=str,
44+
default="test",
45+
)
46+
47+
options = parser.parse_args(args)
48+
if not options.channel in ("nightly", "test", "release"):
49+
raise Exception("--channel is invalid, please provide nightly, test or release")
50+
channel = options.channel
51+
52+
cuda_versions = CUDA_VERSIONS_DICT[channel]
53+
trt_versions = TRT_VERSIONS_DICT[channel]
54+
docker_images = DOCKER_IMAGE_DICT[channel]
55+
56+
matrix_dict = {"include": []}
57+
for cuda_version in cuda_versions:
58+
for trt_version in trt_versions:
59+
matrix_dict["include"].append({
60+
"cuda": cuda_version,
61+
"trt": trt_version,
62+
"docker_image": docker_images[channel][cuda_version],
63+
})
64+
print(json.dumps(matrix_dict, indent=4))
65+
66+
67+
if __name__ == "__main__":
68+
main(sys.argv[1:])

‎.github/workflows/mlir-tensorrt-build-test.yml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
name: MLIR-TensorRT Build and Test
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
push:
7+
branches:
8+
- main
9+
- nightly
10+
- release/*
11+
paths: ["mlir-tensorrt/**"]
12+
tags:
13+
# release tag example: v0.4.2
14+
# release candidate tag example: v0.4.2-rc1
15+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
16+
17+
jobs:
18+
generate-matrix:
19+
runs-on: ubuntu-latest
20+
outputs:
21+
matrix: ${{ steps.generate.outputs.matrix }}
22+
steps:
23+
- name: checkout TensorRT-Incubator
24+
uses: actions/checkout@v4
25+
- name: python format and clang check
26+
run: |
27+
set -eou pipefail
28+
set -x
29+
python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/compiler/
30+
python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/integrations/python/
31+
git clang-format HEAD~1 --diff
32+
if [ $? -ne 0 ]; then
33+
echo "Python format and clang check failed"
34+
exit 1
35+
fi
36+
echo "Python format and clang check passed"
37+
- name: Generate matrix
38+
id: generate
39+
run: |
40+
set -eou pipefail
41+
set -x
42+
MATRIX_BLOB="$(python3 .github/scripts/generate-matrix.py --channel test)"
43+
echo "${MATRIX_BLOB}"
44+
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
45+
46+
mlir-tensorrt-build-test:
47+
needs: generate-matrix
48+
strategy:
49+
matrix:
50+
${{ fromJson(needs.generate-matrix.outputs.matrix) }}
51+
runs-on: [linux-amd64-gpu-h100-latest-1]
52+
container:
53+
image: ${{ matrix.docker_image }}
54+
options: '--gpus all'
55+
steps:
56+
- name: checkout TensorRT-Incubator
57+
uses: actions/checkout@v4
58+
- name: create cache folder
59+
run: |
60+
set -eou pipefail
61+
set -x
62+
mkdir -p ${{ github.workspace }}/ccache
63+
mkdir -p ${{ github.workspace }}/.cache.cpm
64+
- name: restore cache
65+
uses: actions/cache/restore@v4
66+
with:
67+
key: ${{ runner.os }}-mlir-tensorrt-cache-${{ hashFiles('mlir-tensorrt/**/*.cpp', 'mlir-tensorrt/**/*.h', 'mlir-tensorrt/build_tools/**/*') }}
68+
restore-keys: |
69+
${{ runner.os }}-mlir-tensorrt-cache-
70+
path: |
71+
${{ github.workspace }}/ccache
72+
${{ github.workspace }}/.cache.cpm/*
73+
!${{ github.workspace }}/.cache.cpm/tensorrt
74+
- name: build_${{ matrix.cuda }}_trt${{ matrix.trt }}
75+
run: |
76+
set -eou pipefail
77+
set -x
78+
cd mlir-tensorrt
79+
# build and testings
80+
DOWNLOAD_TENSORRT_VERSION=${{ matrix.trt }} ./build_tools/scripts/cicd_build.sh --build_only
81+
- name: save cache
82+
uses: actions/cache/save@v4
83+
with:
84+
key: ${{ runner.os }}-mlir-tensorrt-cache-${{ hashFiles('mlir-tensorrt/**/*.cpp', 'mlir-tensorrt/**/*.h', 'mlir-tensorrt/build_tools/**/*') }}
85+
path: |
86+
${{ github.workspace }}/ccache
87+
${{ github.workspace }}/.cache.cpm/*
88+
!${{ github.workspace }}/.cache.cpm/tensorrt
89+
- name: test_${{ matrix.cuda }}_trt${{ matrix.trt }}
90+
run: |
91+
set -eou pipefail
92+
set -x
93+
cd mlir-tensorrt
94+
DOWNLOAD_TENSORRT_VERSION=${{ matrix.trt }} ./build_tools/scripts/cicd_build.sh
95+
- name: ASAN_test_${{ matrix.cuda }}_trt${{ matrix.trt }}
96+
run: |
97+
set -eou pipefail
98+
set -x
99+
cd mlir-tensorrt
100+
DOWNLOAD_TENSORRT_VERSION=${{ matrix.trt }} ENABLE_ASAN=ON ./build_tools/scripts/cicd_build.sh
101+
- name: build_wheel_${{ matrix.cuda }}_trt${{ matrix.trt }}
102+
run: |
103+
set -eou pipefail
104+
set -x
105+
cd mlir-tensorrt
106+
DOWNLOAD_TENSORRT_VERSION=${{ matrix.trt }} ./build_tools/scripts/cicd_build_wheels.sh --build_wheels
107+
- name: smoke_test_wheel_${{ matrix.cuda }}_trt${{ matrix.trt }}
108+
run: |
109+
set -eou pipefail
110+
set -x
111+
cd mlir-tensorrt
112+
python3 -m pip install .private.wheels/mlir_tensorrt-*.whl
113+
# TODO: add smoke test for the wheel
114+

‎.github/workflows/mlir-tensorrt-release.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: MLIR-TensorRT Build and Test
2+
3+
on:
4+
push:
5+
tags:
6+
# release tag example: v0.4.2
7+
# release candidate tag example: v0.4.2-rc1
8+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
9+
10+
jobs:
11+
generate-matrix:
12+
runs-on: ubuntu-latest
13+
outputs:
14+
matrix: ${{ steps.generate.outputs.matrix }}
15+
steps:
16+
- name: checkout TensorRT-Incubator
17+
uses: actions/checkout@v4
18+
- name: python format and clang check
19+
run: |
20+
set -eou pipefail
21+
set -x
22+
python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/compiler/
23+
python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/integrations/python/
24+
git clang-format HEAD~1 --diff
25+
if [ $? -ne 0 ]; then
26+
echo "Python format and clang check failed"
27+
exit 1
28+
fi
29+
echo "Python format and clang check passed"
30+
- name: Generate matrix
31+
id: generate
32+
run: |
33+
set -eou pipefail
34+
set -x
35+
MATRIX_BLOB="$(python3 .github/scripts/generate-matrix.py --channel release)"
36+
echo "${MATRIX_BLOB}"
37+
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
38+
39+
mlir-tensorrt-build-test:
40+
needs: generate-matrix
41+
strategy:
42+
matrix:
43+
${{ fromJson(needs.generate-matrix.outputs.matrix) }}
44+
runs-on: [linux-amd64-gpu-h100-latest-1]
45+
container:
46+
image: ${{ matrix.docker_image }}
47+
options: '--gpus all'
48+
steps:
49+
- name: checkout TensorRT-Incubator
50+
uses: actions/checkout@v4
51+
- name: create cache folder
52+
run: |
53+
set -eou pipefail
54+
set -x
55+
mkdir -p ${{ github.workspace }}/ccache
56+
mkdir -p ${{ github.workspace }}/.cache.cpm
57+
- name: restore cache
58+
uses: actions/cache/restore@v4
59+
with:
60+
key: ${{ runner.os }}-mlir-tensorrt-cache-${{ hashFiles('mlir-tensorrt/**/*.cpp', 'mlir-tensorrt/**/*.h', 'mlir-tensorrt/build_tools/**/*') }}
61+
restore-keys: |
62+
${{ runner.os }}-mlir-tensorrt-cache-
63+
path: |
64+
${{ github.workspace }}/ccache
65+
${{ github.workspace }}/.cache.cpm/*
66+
!${{ github.workspace }}/.cache.cpm/tensorrt
67+
- name: build_test_${{ matrix.cuda }}_py${{ matrix.python }}_trt${{ matrix.trt }}
68+
run: |
69+
set -eou pipefail
70+
set -x
71+
cd mlir-tensorrt
72+
./build_tools/scripts/cicd_build.sh
73+

‎mlir-tensorrt/build_tools/scripts/release_wheels.sh‎ renamed to ‎mlir-tensorrt/build_tools/scripts/cicd_build_wheels.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,5 @@ rm -rf ${BUILD_DIR} || true
2121

2222
cmake -B ${BUILD_DIR} --preset python-wheel-build
2323

24+
ninja -C ${BUILD_DIR} mlir-tensorrt-all-wheels
25+
rsync -za build/wheels/ .private.wheels/

0 commit comments

Comments
 (0)