Microbenchmarks-Perf-Nightly #55
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Microbenchmarks-Perf-Nightly | |
| # Dashboard: https://hud.pytorch.org/benchmark/llms?repoName=pytorch%2Fao&benchmarkName=micro-benchmark+api | |
| on: | |
| push: | |
| tags: | |
| - ciflow/benchmark/* | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 3 * * *' # Run daily at 7 AM UTC | |
| jobs: | |
| benchmark: | |
| runs-on: linux.aws.h100 | |
| strategy: | |
| matrix: | |
| torch-spec: | |
| - '--pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu126' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup miniconda | |
| uses: pytorch/test-infra/.github/actions/setup-miniconda@main | |
| with: | |
| python-version: "3.9" | |
| - name: Run benchmark | |
| shell: bash | |
| run: | | |
| set -eux | |
| # Upgrade pip | |
| ${CONDA_RUN} python -m pip install --upgrade pip | |
| ${CONDA_RUN} ls | |
| ${CONDA_RUN} bash -c 'pwd' | |
| ${CONDA_RUN} bash -c 'echo $PYTHONPATH' | |
| # Install dependencies | |
| ${CONDA_RUN} pip install ${{ matrix.torch-spec }} | |
| ${CONDA_RUN} pip install -r dev-requirements.txt | |
| ${CONDA_RUN} pip install . | |
| ${CONDA_RUN} ls | |
| ${CONDA_RUN} bash -c 'pwd' | |
| ${CONDA_RUN} bash -c 'echo $PYTHONPATH' | |
| # Set PYTHONPATH to current directory (.) if not set, and include the benchmarks directory | |
| ${CONDA_RUN} export PYTHONPATH="${PYTHONPATH:-$(pwd)}:$(pwd)/benchmarks" | |
| # Create benchmark results directory | |
| mkdir -p ${{ runner.temp }}/benchmark-results | |
| # Run microbenchmarks for dashboard | |
| ${CONDA_RUN} bash -c ' | |
| export PYTHONPATH="${PYTHONPATH:-$(pwd)}:$(pwd)/benchmarks" | |
| echo "PYTHONPATH is: $PYTHONPATH" | |
| echo "Current directory is: $(pwd)" | |
| python benchmarks/dashboard/ci_microbenchmark_runner.py \ | |
| --config benchmarks/dashboard/microbenchmark_quantization_config.yml \ | |
| --output "$RUNNER_TEMP/benchmark-results/microbenchmark-results.json"' | |
| - name: Upload the benchmark results to OSS benchmark database for the dashboard | |
| uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main | |
| with: | |
| benchmark-results-dir: ${{ runner.temp }}/benchmark-results | |
| dry-run: false | |
| schema-version: v3 | |
| github-token: ${{ secrets.GITHUB_TOKEN }} |