Skip to content

Commit dcc47a5

Browse files
alisonshaoalisonshao
authored
Implement nightly test workflow naming conventions (#13170)
1 parent 6448b4c commit dcc47a5

File tree

5 files changed

+402
-12
lines changed

5 files changed

+402
-12
lines changed

‎.github/workflows/nightly-test-intel.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Nightly Test (Intel)
2+
3+
on:
4+
schedule:
5+
- cron: '0 0 * * *'
6+
push:
7+
branches:
8+
- main
9+
paths:
10+
- "python/sglang/version.py"
11+
workflow_dispatch:
12+
13+
concurrency:
14+
group: nightly-test-intel-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
# Placeholder for Intel GPU tests
19+
# Add Intel-specific nightly test workflows here when available
20+
21+
placeholder:
22+
if: github.repository == 'sgl-project/sglang'
23+
runs-on: ubuntu-latest
24+
steps:
25+
- name: Placeholder
26+
run: echo "Intel nightly tests will be added here"

‎.github/workflows/nightly-test-nvidia.yml‎

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
name: Nightly Test (Nvidia)
2+
3+
on:
4+
schedule:
5+
- cron: '0 0 * * *'
6+
push:
7+
branches:
8+
- main
9+
paths:
10+
- "python/sglang/version.py"
11+
workflow_dispatch:
12+
13+
concurrency:
14+
group: nightly-test-nvidia-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
# General tests - 1 GPU
19+
nightly-test-general-1-gpu-runner:
20+
if: github.repository == 'sgl-project/sglang'
21+
runs-on: 1-gpu-runner
22+
steps:
23+
- name: Checkout code
24+
uses: actions/checkout@v4
25+
26+
- name: Install dependencies
27+
run: |
28+
bash scripts/ci/ci_install_dependency.sh
29+
30+
- name: Run test
31+
timeout-minutes: 60
32+
run: |
33+
cd test/srt
34+
python3 run_suite.py --suite nightly-1-gpu --continue-on-error
35+
36+
# General tests - 4 GPU H100
37+
nightly-test-general-4-gpu-h100:
38+
if: github.repository == 'sgl-project/sglang'
39+
runs-on: 4-gpu-h100
40+
steps:
41+
- name: Checkout code
42+
uses: actions/checkout@v4
43+
44+
- name: Install dependencies
45+
run: |
46+
bash scripts/ci/ci_install_dependency.sh
47+
48+
- name: Run test
49+
timeout-minutes: 30
50+
run: |
51+
cd test/srt
52+
python3 run_suite.py --suite nightly-4-gpu --continue-on-error
53+
54+
# General tests - 8 GPU H200
55+
nightly-test-general-8-gpu-h200:
56+
if: github.repository == 'sgl-project/sglang'
57+
runs-on: 8-gpu-h200
58+
env:
59+
RUNNER_LABELS: 8-gpu-h200
60+
steps:
61+
- name: Checkout code
62+
uses: actions/checkout@v4
63+
64+
- name: Install dependencies
65+
run: |
66+
bash scripts/ci/ci_install_dependency.sh
67+
68+
- name: Run test
69+
timeout-minutes: 30
70+
env:
71+
GPU_CONFIG: "8-gpu-h200"
72+
run: |
73+
cd test/srt
74+
python3 run_suite.py --suite nightly-8-gpu-h200 --continue-on-error
75+
76+
# General tests - 8 GPU H20
77+
nightly-test-general-8-gpu-h20:
78+
if: github.repository == 'sgl-project/sglang'
79+
runs-on: 8-gpu-h20
80+
env:
81+
SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
82+
steps:
83+
- name: Checkout code
84+
uses: actions/checkout@v4
85+
86+
- name: Install dependencies
87+
run: |
88+
bash scripts/ci/ci_install_dependency.sh
89+
90+
- name: Run test
91+
timeout-minutes: 30
92+
env:
93+
GPU_CONFIG: "8-gpu-h20"
94+
run: |
95+
cd test/srt
96+
python3 run_suite.py --suite nightly-8-gpu-h20 --continue-on-error
97+
98+
# Text model accuracy tests
99+
nightly-test-text-accuracy-2-gpu-runner:
100+
if: github.repository == 'sgl-project/sglang'
101+
runs-on: 2-gpu-runner
102+
steps:
103+
- name: Checkout code
104+
uses: actions/checkout@v4
105+
106+
- name: Install dependencies
107+
run: |
108+
bash scripts/ci/ci_install_dependency.sh
109+
110+
- name: Run eval test for text models
111+
timeout-minutes: 120
112+
run: |
113+
cd test/srt
114+
python3 nightly/test_text_models_gsm8k_eval.py
115+
116+
# Text model performance tests
117+
nightly-test-text-perf-2-gpu-runner:
118+
if: github.repository == 'sgl-project/sglang'
119+
runs-on: 2-gpu-runner
120+
steps:
121+
- name: Checkout code
122+
uses: actions/checkout@v4
123+
124+
- name: Install dependencies
125+
run: |
126+
bash scripts/ci/ci_install_dependency.sh
127+
128+
- name: Run performance test for text models
129+
timeout-minutes: 180
130+
env:
131+
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
132+
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
133+
GPU_CONFIG: "2-gpu-runner"
134+
run: |
135+
cd test/srt
136+
rm -rf performance_profiles_text_models/
137+
python3 nightly/test_text_models_perf.py
138+
139+
- name: Publish traces to storage repo
140+
env:
141+
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
142+
GITHUB_RUN_ID: ${{ github.run_id }}
143+
GITHUB_RUN_NUMBER: ${{ github.run_number }}
144+
run: |
145+
python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_text_models
146+
147+
# VLM accuracy tests
148+
nightly-test-vlm-accuracy-2-gpu-runner:
149+
if: github.repository == 'sgl-project/sglang'
150+
runs-on: 2-gpu-runner
151+
steps:
152+
- name: Checkout code
153+
uses: actions/checkout@v4
154+
155+
- name: Install dependencies
156+
run: |
157+
bash scripts/ci/ci_install_dependency.sh
158+
159+
- name: Run eval test for VLM models (fixed MMMU-100)
160+
timeout-minutes: 240
161+
run: |
162+
cd test/srt
163+
python3 nightly/test_vlms_mmmu_eval.py
164+
165+
# VLM performance tests
166+
nightly-test-vlm-perf-2-gpu-runner:
167+
if: github.repository == 'sgl-project/sglang'
168+
runs-on: 2-gpu-runner
169+
steps:
170+
- name: Checkout code
171+
uses: actions/checkout@v4
172+
173+
- name: Install dependencies
174+
run: |
175+
bash scripts/ci/ci_install_dependency.sh
176+
177+
- name: Run perf test for VLM models (MMMU)
178+
timeout-minutes: 240
179+
env:
180+
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
181+
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
182+
GPU_CONFIG: "2-gpu-runner"
183+
run: |
184+
cd test/srt
185+
rm -rf performance_profiles_vlms/
186+
python3 nightly/test_vlms_perf.py
187+
188+
- name: Publish traces to storage repo
189+
env:
190+
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
191+
GITHUB_RUN_ID: ${{ github.run_id }}
192+
GITHUB_RUN_NUMBER: ${{ github.run_number }}
193+
run: |
194+
python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_vlms
195+
196+
# B200 Performance tests - 4 GPU
197+
nightly-test-perf-4-gpu-b200:
198+
if: github.repository == 'sgl-project/sglang'
199+
runs-on: 4-gpu-b200
200+
steps:
201+
- name: Checkout code
202+
uses: actions/checkout@v4
203+
204+
- name: Install dependencies
205+
run: |
206+
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
207+
208+
- name: Run test
209+
timeout-minutes: 60
210+
run: |
211+
cd test/srt
212+
python3 run_suite.py --suite nightly-4-gpu-b200 --continue-on-error
213+
214+
# B200 Performance tests - 8 GPU
215+
nightly-test-perf-8-gpu-b200:
216+
if: github.repository == 'sgl-project/sglang'
217+
runs-on: 8-gpu-b200
218+
env:
219+
RUNNER_LABELS: 8-gpu-b200
220+
steps:
221+
- name: Checkout code
222+
uses: actions/checkout@v4
223+
224+
- name: Install dependencies
225+
run: |
226+
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
227+
228+
- name: Run DeepSeek v3.1 nightly performance test
229+
timeout-minutes: 180
230+
env:
231+
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
232+
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
233+
GPU_CONFIG: "8-gpu-b200"
234+
run: |
235+
rm -rf test/srt/performance_profiles_deepseek_v31/
236+
cd test/srt
237+
IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py
238+
239+
- name: Publish DeepSeek v3.1 traces to storage repo
240+
env:
241+
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
242+
GITHUB_RUN_ID: ${{ github.run_id }}
243+
GITHUB_RUN_NUMBER: ${{ github.run_number }}
244+
run: |
245+
python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_deepseek_v31
246+
247+
- name: Run DeepSeek v3.2 nightly performance test
248+
timeout-minutes: 180
249+
env:
250+
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
251+
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
252+
GPU_CONFIG: "8-gpu-b200"
253+
run: |
254+
rm -rf test/srt/performance_profiles_deepseek_v32/
255+
cd test/srt
256+
IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py
257+
258+
- name: Publish DeepSeek v3.2 traces to storage repo
259+
env:
260+
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
261+
GITHUB_RUN_ID: ${{ github.run_id }}
262+
GITHUB_RUN_NUMBER: ${{ github.run_number }}
263+
run: |
264+
python3 scripts/ci/publish_traces.py --traces-dir test/srt/performance_profiles_deepseek_v32
265+
266+
# Final check job
267+
check-all-jobs:
268+
if: github.repository == 'sgl-project/sglang' && always()
269+
needs:
270+
- nightly-test-general-1-gpu-runner
271+
- nightly-test-general-4-gpu-h100
272+
- nightly-test-general-8-gpu-h200
273+
- nightly-test-general-8-gpu-h20
274+
- nightly-test-text-accuracy-2-gpu-runner
275+
- nightly-test-text-perf-2-gpu-runner
276+
- nightly-test-vlm-accuracy-2-gpu-runner
277+
- nightly-test-vlm-perf-2-gpu-runner
278+
- nightly-test-perf-4-gpu-b200
279+
- nightly-test-perf-8-gpu-b200
280+
runs-on: ubuntu-latest
281+
steps:
282+
- name: Check if any job failed
283+
run: |
284+
if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
285+
echo "One or more nightly test jobs failed"
286+
exit 1
287+
fi
288+
if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
289+
echo "One or more nightly test jobs were cancelled"
290+
exit 1
291+
fi
292+
echo "All nightly test jobs passed"

‎python/sglang/bench_one_batch_server.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,17 @@ def generate_markdown_report(trace_dir, results: List["BenchmarkResult"]) -> str
130130
"""Generate a markdown report from a list of BenchmarkResult object from a single run."""
131131
import os
132132

133-
summary = f"### {results[0].model_path}\n"
133+
# Build model header with run_name if it's not "default"
134+
model_header = results[0].model_path
135+
if results[0].run_name and results[0].run_name != "default":
136+
model_header += f" ({results[0].run_name})"
137+
138+
# Include GPU config in model header if available
139+
gpu_config = os.getenv("GPU_CONFIG", "")
140+
if gpu_config:
141+
model_header += f" [{gpu_config}]"
142+
143+
summary = f"### {model_header}\n"
134144

135145
# summary += (
136146
# f"Input lens: {result.input_len}. Output lens: {result.output_len}.\n"

‎test/srt/nightly/nightly_utils.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,18 +29,27 @@ def __init__(
2929
profile_dir: str,
3030
test_name: str,
3131
base_url: str,
32+
gpu_config: str = None,
3233
):
3334
"""Initialize the benchmark runner.
3435
3536
Args:
3637
profile_dir: Directory to store performance profiles
3738
test_name: Name of the test (used for reporting)
3839
base_url: Base URL for the server
40+
gpu_config: Optional GPU configuration string (e.g., "2-gpu-h100", "8-gpu-b200")
3941
"""
4042
self.profile_dir = profile_dir
4143
self.test_name = test_name
4244
self.base_url = base_url
43-
self.full_report = f"## {test_name}\n" + BenchmarkResult.help_str()
45+
self.gpu_config = gpu_config or os.environ.get("GPU_CONFIG", "")
46+
47+
# Include GPU config in report header if available
48+
header = f"## {test_name}"
49+
if self.gpu_config:
50+
header += f" ({self.gpu_config})"
51+
header += "\n"
52+
self.full_report = header + BenchmarkResult.help_str()
4453

4554
def setup_profile_directory(self) -> None:
4655
"""Create the profile directory if it doesn't exist."""
@@ -241,14 +250,19 @@ def run_benchmark_for_model(
241250
)
242251

243252
# Build and run benchmark command
253+
# Prepare extra args with run_name if variant is specified
254+
bench_args = list(extra_bench_args) if extra_bench_args else []
255+
if variant:
256+
bench_args.extend(["--run-name", variant])
257+
244258
command = self.build_benchmark_command(
245259
model_path,
246260
batch_sizes,
247261
input_lens,
248262
output_lens,
249263
profile_path_prefix,
250264
json_output_file,
251-
extra_args=extra_bench_args,
265+
extra_args=bench_args,
252266
)
253267

254268
result, cmd_success = self.run_benchmark_command(command, model_description)

0 commit comments

Comments
 (0)