From f0e1d4819cd249ed427642b3943d142548a913c0 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 20 Aug 2024 16:29:14 +0200 Subject: [PATCH 01/12] try codspeed for KA --- .github/workflows/benchmark_pr.yml | 91 +++++++++++++++++------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index 1af037fd6..bc166883c 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -1,7 +1,14 @@ name: Benchmark a pull request on: + push: + branches: + - "main" + # Run on pull requests pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: permissions: pull-requests: write @@ -32,45 +39,51 @@ jobs: run: | echo "$HOME/.julia/bin" >> $GITHUB_PATH - name: Run benchmarks - run: | - echo $PATH - ls -l ~/.julia/bin - mkdir results - benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune - - name: Create plots from benchmarks - run: | - mkdir -p plots - benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/ - - name: Upload plot as artifact - uses: actions/upload-artifact@v2 + uses: CodSpeedHQ/action@v3 with: - name: plots - path: plots - - name: Create markdown table from benchmarks - run: | - benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md - echo '### Benchmark Results' > body.md - echo '' >> body.md - echo '' >> body.md - cat table.md >> body.md - echo '' >> body.md - echo '' >> body.md - echo '### Benchmark Plots' >> body.md - echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md - echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md + token: ${{ secrets.CODSPEED_TOKEN }} + run: "" - - name: Find Comment - uses: peter-evans/find-comment@v2 - id: fcbenchmark - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Benchmark Results + # - name: Run benchmarks + # run: | + # echo $PATH + # ls -l ~/.julia/bin + # mkdir results + # benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune + # - name: Create plots from benchmarks + # run: | + # mkdir -p plots + # benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/ + # - name: Upload plot as artifact + # uses: actions/upload-artifact@v2 + # with: + # name: plots + # path: plots + # - name: Create markdown table from benchmarks + # run: | + # benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md + # echo '### Benchmark Results' > body.md + # echo '' >> body.md + # echo '' >> body.md + # cat table.md >> body.md + # echo '' >> body.md + # echo '' >> body.md + # echo '### Benchmark Plots' >> body.md + # echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md + # echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md - - name: Comment on PR - uses: peter-evans/create-or-update-comment@v3 - with: - comment-id: ${{ steps.fcbenchmark.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: body.md - edit-mode: replace + # - name: Find Comment + # uses: peter-evans/find-comment@v2 + # id: fcbenchmark + # with: + # issue-number: ${{ github.event.pull_request.number }} + # comment-author: 'github-actions[bot]' + # body-includes: Benchmark Results + + # - name: Comment on PR + # uses: peter-evans/create-or-update-comment@v3 + # with: + # comment-id: ${{ steps.fcbenchmark.outputs.comment-id }} + # issue-number: ${{ github.event.pull_request.number }} + # body-path: body.md + # edit-mode: replace From a4fcbc6185351527d71612f93029bc0072077024 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 10:18:36 +0200 Subject: [PATCH 02/12] Try ValgrindBenchmarkTools --- .github/workflows/benchmark_pr.yml | 15 ++++++++++++--- benchmark/benchmarks.jl | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index bc166883c..cf98392ad 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -31,18 +31,27 @@ jobs: - name: Build AirspeedVelocity env: JULIA_NUM_THREADS: 2 + JULIA_PKG_PRECOMPILE_AUTO: false run: | # Lightweight build step, as sometimes the runner runs out of memory: - julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add("AirspeedVelocity")' - julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")' + julia -e 'import Pkg; Pkg.add("AirspeedVelocity")' + julia -e 'import Pkg; Pkg.build("AirspeedVelocity")' + - name: Install ValgrindBenchmarkTools + env: + JULIA_PKG_PRECOMPILE_AUTO: false + run: | + julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt")' + julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt", subdir="lib/ValgrindBenchmarkTools")' - name: Add ~/.julia/bin to PATH run: | echo "$HOME/.julia/bin" >> $GITHUB_PATH - name: Run benchmarks uses: CodSpeedHQ/action@v3 + env: + ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: "" + run: 'benchpkg --rev="dirty"' # - name: Run benchmarks # run: | diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index b153f60ee..361ec7014 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -3,7 +3,7 @@ # result = benchmarkpkg(KernelAbstractions, BenchmarkConfig(env=Dict("KA_BACKEND"=>"CPU", "JULIA_NUM_THREADS"=>"auto"))) # export_markdown("perf.md", result) -using BenchmarkTools +using ValgrindBenchmarkTools using KernelAbstractions using Random From c1271ea09467c5b5b16927ac8f5e5bcba09065a8 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 10:27:09 +0200 Subject: [PATCH 03/12] Precompile --- .github/workflows/benchmark_pr.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index cf98392ad..d7aa4dea7 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -42,6 +42,12 @@ jobs: run: | julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt")' julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt", subdir="lib/ValgrindBenchmarkTools")' + - name: Precompile + env: + JULIA_CPU_TARGET: "haswell" + run: | + julia -e 'import Pkg; Pkg.precompile()' + julia --project=. -e 'import Pkg; Pkg.precompile()' - name: Add ~/.julia/bin to PATH run: | echo "$HOME/.julia/bin" >> $GITHUB_PATH From 17c37beba2727859cbe0bbdfc5adaf3c83bfe925 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 10:47:05 +0200 Subject: [PATCH 04/12] use PkgBenchmark instead of ASV --- .github/workflows/benchmark_pr.yml | 62 ++---------------------------- 1 file changed, 3 insertions(+), 59 deletions(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index d7aa4dea7..b321d8654 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -23,28 +23,16 @@ jobs: with: version: "1" - uses: julia-actions/cache@v1 - - name: Extract Package Name from Project.toml - id: extract-package-name - run: | - PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/') - echo "::set-output name=package_name::$PACKAGE_NAME" - - name: Build AirspeedVelocity - env: - JULIA_NUM_THREADS: 2 - JULIA_PKG_PRECOMPILE_AUTO: false - run: | - # Lightweight build step, as sometimes the runner runs out of memory: - julia -e 'import Pkg; Pkg.add("AirspeedVelocity")' - julia -e 'import Pkg; Pkg.build("AirspeedVelocity")' - name: Install ValgrindBenchmarkTools env: JULIA_PKG_PRECOMPILE_AUTO: false run: | + julia -e 'import Pkg; Pkg.add("PkgBenchmark")' julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt")' julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt", subdir="lib/ValgrindBenchmarkTools")' - name: Precompile env: - JULIA_CPU_TARGET: "haswell" + JULIA_CPU_TARGET: "haswell" # CPU emulated by Valgrind run: | julia -e 'import Pkg; Pkg.precompile()' julia --project=. -e 'import Pkg; Pkg.precompile()' @@ -57,48 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'benchpkg --rev="dirty"' - - # - name: Run benchmarks - # run: | - # echo $PATH - # ls -l ~/.julia/bin - # mkdir results - # benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune - # - name: Create plots from benchmarks - # run: | - # mkdir -p plots - # benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/ - # - name: Upload plot as artifact - # uses: actions/upload-artifact@v2 - # with: - # name: plots - # path: plots - # - name: Create markdown table from benchmarks - # run: | - # benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md - # echo '### Benchmark Results' > body.md - # echo '' >> body.md - # echo '' >> body.md - # cat table.md >> body.md - # echo '' >> body.md - # echo '' >> body.md - # echo '### Benchmark Plots' >> body.md - # echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md - # echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md - - # - name: Find Comment - # uses: peter-evans/find-comment@v2 - # id: fcbenchmark - # with: - # issue-number: ${{ github.event.pull_request.number }} - # comment-author: 'github-actions[bot]' - # body-includes: Benchmark Results - - # - name: Comment on PR - # uses: peter-evans/create-or-update-comment@v3 - # with: - # comment-id: ${{ steps.fcbenchmark.outputs.comment-id }} - # issue-number: ${{ github.event.pull_request.number }} - # body-path: body.md - # edit-mode: replace + run: 'julia --project=. -e "import BenchmarkPkg, KernelAbstractions; benchmarkpkg(KernelAbstractions)' From b8fdea68f7a2539713bc973585faef1ddbe3f7d3 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 10:50:54 +0200 Subject: [PATCH 05/12] fixup! use PkgBenchmark instead of ASV --- .github/workflows/benchmark_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index b321d8654..70360fd50 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -45,4 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'julia --project=. -e "import BenchmarkPkg, KernelAbstractions; benchmarkpkg(KernelAbstractions)' + run: 'julia --project=. -e "import BenchmarkPkg, KernelAbstractions; benchmarkpkg(KernelAbstractions)"' From e625744a3c8a86842760fb94dd6c69a79561b94d Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 10:54:39 +0200 Subject: [PATCH 06/12] fixup! fixup! use PkgBenchmark instead of ASV --- .github/workflows/benchmark_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index 70360fd50..f6caf398c 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -45,4 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'julia --project=. -e "import BenchmarkPkg, KernelAbstractions; benchmarkpkg(KernelAbstractions)"' + run: 'julia --project=. -e "import PkgBenchmark, KernelAbstractions; benchmarkpkg(KernelAbstractions)"' From 3aadec39c7713af0dfaf9df80d44ffb002e1944d Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 11:12:43 +0200 Subject: [PATCH 07/12] fixup! fixup! fixup! use PkgBenchmark instead of ASV --- .github/workflows/benchmark_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index f6caf398c..f00281bb2 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -45,4 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'julia --project=. -e "import PkgBenchmark, KernelAbstractions; benchmarkpkg(KernelAbstractions)"' + run: 'julia --project=. -e "using PkgBenchmark, KernelAbstractions; benchmarkpkg(KernelAbstractions)"' From 01abd366a8ce69ec290a2adde0364e9c33367db5 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 11:25:57 +0200 Subject: [PATCH 08/12] add development version of BT --- .github/workflows/benchmark_pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index f00281bb2..05b83c9ed 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -27,6 +27,7 @@ jobs: env: JULIA_PKG_PRECOMPILE_AUTO: false run: | + julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaCI/BenchmarkTools.jl", rev="vc/extendable")' julia -e 'import Pkg; Pkg.add("PkgBenchmark")' julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt")' julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt", subdir="lib/ValgrindBenchmarkTools")' From d9bde627976dd6677745b419a6077a8b314913f2 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 11:31:37 +0200 Subject: [PATCH 09/12] just use BT directly --- .github/workflows/benchmark_pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index 05b83c9ed..a31a0897c 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -28,7 +28,6 @@ jobs: JULIA_PKG_PRECOMPILE_AUTO: false run: | julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaCI/BenchmarkTools.jl", rev="vc/extendable")' - julia -e 'import Pkg; Pkg.add("PkgBenchmark")' julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt")' julia -e 'import Pkg; Pkg.add(url="https://github.com/JuliaPerf/Valgrind.jl", rev="vc/bt", subdir="lib/ValgrindBenchmarkTools")' - name: Precompile @@ -46,4 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'julia --project=. -e "using PkgBenchmark, KernelAbstractions; benchmarkpkg(KernelAbstractions)"' + run: 'julia --project=. -L benchmarks/benchmark.jl -e "run(SUITE)"' From 6c4f7f9c891051d2fc166fd62ebea7e839561bb1 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 11:38:23 +0200 Subject: [PATCH 10/12] fixup! just use BT directly --- .github/workflows/benchmark_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index a31a0897c..98e405879 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -45,4 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'julia --project=. -L benchmarks/benchmark.jl -e "run(SUITE)"' + run: 'julia --project=. -L benchmark/benchmarks.jl -e "run(SUITE)"' From 9388efdb06e5c4092d5e852426eed6f06467f17b Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 11:48:42 +0200 Subject: [PATCH 11/12] fixup! fixup! just use BT directly --- .github/workflows/benchmark_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index 98e405879..ed8eee01f 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -45,4 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'julia --project=. -L benchmark/benchmarks.jl -e "run(SUITE)"' + run: 'julia --project=. -L benchmark/benchmarks.jl -e "run(SUITE, "")"' From ff28432141a50c32bf6b4d947f3c7c0fe4b5dd23 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 23 Aug 2024 11:48:53 +0200 Subject: [PATCH 12/12] fixup! fixup! fixup! just use BT directly --- .github/workflows/benchmark_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml index ed8eee01f..b76bccf9c 100644 --- a/.github/workflows/benchmark_pr.yml +++ b/.github/workflows/benchmark_pr.yml @@ -45,4 +45,4 @@ jobs: ENABLE_JITPROFILING: 1 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: 'julia --project=. -L benchmark/benchmarks.jl -e "run(SUITE, "")"' + run: 'julia --project=. -L benchmark/benchmarks.jl -e "run(SUITE, \"\")"'