From 08258955213c40b57169e5f445e8bfe7cde21713 Mon Sep 17 00:00:00 2001 From: James Cor Date: Fri, 7 Aug 2026 14:12:49 -0700 Subject: [PATCH 1/5] update mini sysbench to use kubernetes --- .github/workflows/mini-sysbench.yml | 128 ++++++++-------------------- 1 file changed, 37 insertions(+), 91 deletions(-) diff --git a/.github/workflows/mini-sysbench.yml b/.github/workflows/mini-sysbench.yml index 7be7339401..82a0a191ea 100644 --- a/.github/workflows/mini-sysbench.yml +++ b/.github/workflows/mini-sysbench.yml @@ -10,103 +10,44 @@ permissions: jobs: mini-sysbench: - runs-on: ubuntu-latest - + runs-on: ubuntu-22.04 + if: ${{ github.event.client_payload.issue_number != -1 }} steps: - name: Checkout DoltgreSQL uses: actions/checkout@v6 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup Git User - uses: fregante/setup-git-user@v2 - - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Install Sysbench - run: | - curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | sudo bash - sudo apt -y install sysbench - - - name: Test PR branch - id: test_doltgresql_pr - continue-on-error: true - run: | - ./postgres/parser/build.sh - ./scripts/quick_sysbench.sh - mv ./scripts/mini_sysbench/results.log ./scripts/mini_sysbench/results1.log - cat ./scripts/mini_sysbench/results1.log - - - name: Test main branch - id: test_doltgresql_main - continue-on-error: true - run: | - git reset --hard - git fetch --all --unshallow - git checkout origin/main - ./postgres/parser/build.sh - ./scripts/quick_sysbench.sh - mv ./scripts/mini_sysbench/results.log ./scripts/mini_sysbench/results2.log - cat ./scripts/mini_sysbench/results2.log - - - name: Check Sysbench Logs - id: check_logs - run: | - cd scripts/mini_sysbench - if [[ -f "results1.log" && -f "results2.log" ]]; then - echo "logs_exist=true" >> $GITHUB_OUTPUT - echo "logs exist" - else - echo "logs_exist=false" >> $GITHUB_OUTPUT - echo "One of the branches could not successfully run the benchmarks." - echo "Please review them for errors, which should be fixed." - exit 1 - fi - - - name: Build Sysbench Results Comment - id: build_results - if: steps.check_logs.outputs.logs_exist == 'true' - run: | - cd testing/go/benchmark - output=$(go run .) - echo "program_output<> $GITHUB_OUTPUT - echo "$output" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - echo "$output" - - - name: Is PR From Fork - id: from_fork - run: | - if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then - echo "This is running from a fork, skipping commenting" - echo "fork=true" >> $GITHUB_OUTPUT - else - echo "This is not running from a fork" - echo "fork=false" >> $GITHUB_OUTPUT - fi - - - name: Post Comment - if: steps.from_fork.outputs.fork == 'false' && steps.build_results.outputs.program_output - uses: actions/github-script@v6 + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Get benchmark results + id: get-results + run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" results.log env: - PROGRAM_OUTPUT: ${{ steps.build_results.outputs.program_output }} + KEY: ${{ github.event.client_payload.key }} + BUCKET: ${{ github.event.client_payload.bucket }} + - name: Post results to PR + uses: actions/github-script@v7 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }} script: | - const commentMarker = '' - const output = process.env.PROGRAM_OUTPUT - const body = `${commentMarker}\n${output}` - + const { ACTOR, FORMAT, ISSUE_NUMBER, JOB_TYPE, GITHUB_WORKSPACE } = process.env; + const { owner, repo } = context.repo; + const issue_number = parseInt(ISSUE_NUMBER, 10); + + const fs = require('fs').promises; + const resData = await fs.readFile(`${GITHUB_WORKSPACE}/results.log`, 'utf8'); + + const commentMarker = ''; + // List comments on the PR const { data: comments } = await github.rest.issues.listComments({ - issue_number: context.issue.number, + issue_number: issue_number, owner: context.repo.owner, - repo: context.repo.repo, - }) - + repo: context.repo.repo + }); + // Check if a comment already exists const comment = comments.find(comment => comment.body.includes(commentMarker)) @@ -116,14 +57,19 @@ jobs: comment_id: comment.id, owner: context.repo.owner, repo: context.repo.repo, - body: body + body: `${commentMarker}\n@${ACTOR} ${FORMAT}\n${resData}` }) } else { // Create a new comment await github.rest.issues.createComment({ - issue_number: context.issue.number, + issue_number: issue_number, owner: context.repo.owner, repo: context.repo.repo, - body: body - }) + body: `${commentMarker}\n@${ACTOR} ${FORMAT}\n${resData}` + }); } + env: + ACTOR: ${{ github.event.client_payload.actor }} + FORMAT: ${{ github.event.client_payload.noms_bin_format }} + ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} + JOB_TYPE: ${{ github.event.client_payload.job_type }} \ No newline at end of file From 598850cb6fc8493b595ded3d5cb1da981bb65fbf Mon Sep 17 00:00:00 2001 From: James Cor Date: Fri, 7 Aug 2026 15:08:34 -0700 Subject: [PATCH 2/5] updating allowed perf people --- .github/scripts/performance-benchmarking/validate-commentor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/performance-benchmarking/validate-commentor.sh b/.github/scripts/performance-benchmarking/validate-commentor.sh index 654a4de73f..7acf8ca1fd 100755 --- a/.github/scripts/performance-benchmarking/validate-commentor.sh +++ b/.github/scripts/performance-benchmarking/validate-commentor.sh @@ -7,7 +7,7 @@ if [[ $# -ne 1 ]]; then exit 1 fi -validcommentors="coffeegoddd bheni jennifersp Hydrocharged reltuk tbantle22 timsehn zachmu max-hoffman nicktobey fulghum" +validcommentors="coffeegoddd bheni Hydrocharged reltuk tbantle22 timsehn zachmu nicktobey fulghum macneale4 jycor angelamayxie elianddb jennifersp eric-richardson1" contains() { [[ $1 =~ (^|[[:space:]])$2($|[[:space:]]) ]] && echo "valid=true" >> $GITHUB_OUTPUT || exit 0 From 01d12b11b823ff57e1c9be1629383b7e2b1342b4 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Aug 2026 15:30:49 -0700 Subject: [PATCH 3/5] copy sysbench from dolt --- .github/workflows/mini-sysbench.yml | 75 --------------------- .github/workflows/sysbench-performance.yaml | 55 +++++++++++++++ 2 files changed, 55 insertions(+), 75 deletions(-) delete mode 100644 .github/workflows/mini-sysbench.yml create mode 100644 .github/workflows/sysbench-performance.yaml diff --git a/.github/workflows/mini-sysbench.yml b/.github/workflows/mini-sysbench.yml deleted file mode 100644 index 82a0a191ea..0000000000 --- a/.github/workflows/mini-sysbench.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Mini Sysbench - -on: - pull_request: - types: [opened, synchronize, reopened] - -permissions: - contents: read - pull-requests: write - -jobs: - mini-sysbench: - runs-on: ubuntu-22.04 - if: ${{ github.event.client_payload.issue_number != -1 }} - steps: - - name: Checkout DoltgreSQL - uses: actions/checkout@v6 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Get benchmark results - id: get-results - run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" results.log - env: - KEY: ${{ github.event.client_payload.key }} - BUCKET: ${{ github.event.client_payload.bucket }} - - name: Post results to PR - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { ACTOR, FORMAT, ISSUE_NUMBER, JOB_TYPE, GITHUB_WORKSPACE } = process.env; - const { owner, repo } = context.repo; - const issue_number = parseInt(ISSUE_NUMBER, 10); - - const fs = require('fs').promises; - const resData = await fs.readFile(`${GITHUB_WORKSPACE}/results.log`, 'utf8'); - - const commentMarker = ''; - - // List comments on the PR - const { data: comments } = await github.rest.issues.listComments({ - issue_number: issue_number, - owner: context.repo.owner, - repo: context.repo.repo - }); - - // Check if a comment already exists - const comment = comments.find(comment => comment.body.includes(commentMarker)) - - if (comment) { - // Update the existing comment - await github.rest.issues.updateComment({ - comment_id: comment.id, - owner: context.repo.owner, - repo: context.repo.repo, - body: `${commentMarker}\n@${ACTOR} ${FORMAT}\n${resData}` - }) - } else { - // Create a new comment - await github.rest.issues.createComment({ - issue_number: issue_number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `${commentMarker}\n@${ACTOR} ${FORMAT}\n${resData}` - }); - } - env: - ACTOR: ${{ github.event.client_payload.actor }} - FORMAT: ${{ github.event.client_payload.noms_bin_format }} - ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} - JOB_TYPE: ${{ github.event.client_payload.job_type }} \ No newline at end of file diff --git a/.github/workflows/sysbench-performance.yaml b/.github/workflows/sysbench-performance.yaml new file mode 100644 index 0000000000..5db240bac9 --- /dev/null +++ b/.github/workflows/sysbench-performance.yaml @@ -0,0 +1,55 @@ +name: Mini Sysbench + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +jobs: + mini-sysbench: + name: Trigger Mini Benchmark Latency K8s Workflow + if: github.actor != 'dependabot[bot]' + runs-on: ubuntu-22.04 + steps: + - name: Build benchmark payload (main HEAD + PR HEAD) + uses: actions/github-script@v7 + id: build_payload + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + // For PR benchmarks, use the PR base commit (the exact target commit this PR was opened/updated against), + // not the moving HEAD of `main`. + const fromSha = context.payload.pull_request.base.sha; + + // The latest commit on the PR branch. + const toSha = context.payload.pull_request.head.sha; + + const payload = { + from_server: "doltgres", + from_version: fromSha, + to_server: "doltgres", + to_version: toSha, + mode: "pullRequest", + issue_number: String(context.issue.number), + init_big_repo: "true", + actor: context.actor, + sysbench_test_time: "20", + template_script: "./.github/scripts/performance-benchmarking/get-doltgres-doltgres-job-json.sh", + }; + + core.setOutput("from_sha", fromSha); + core.setOutput("to_sha", toSha); + core.setOutput("payload", JSON.stringify(payload)); + - name: Log benchmark versions + shell: bash + run: | + echo "from_version_sha=${{ steps.build_payload.outputs.from_sha }}" + echo "to_version_sha=${{ steps.build_payload.outputs.to_sha }}" + - uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.REPO_ACCESS_TOKEN }} + event-type: benchmark-latency + client-payload: ${{ steps.build_payload.outputs.payload }} From 2d8df07f8b61f86fad853b3b0dcc1621473a9c6b Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Aug 2026 15:36:26 -0700 Subject: [PATCH 4/5] try this? --- .github/workflows/pull-report.yaml | 41 +++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pull-report.yaml b/.github/workflows/pull-report.yaml index 53a7e95601..e0f7cb76b9 100644 --- a/.github/workflows/pull-report.yaml +++ b/.github/workflows/pull-report.yaml @@ -34,20 +34,37 @@ jobs: github-token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }} script: | const { ACTOR, FORMAT, ISSUE_NUMBER, GITHUB_WORKSPACE } = process.env; - const issue_number = parseInt(ISSUE_NUMBER, 10); const { owner, repo } = context.repo; - fs = require('fs'); - fs.readFile(`${GITHUB_WORKSPACE}/results.log`, 'utf8', function (err,data) { - if (err) { - return console.log(err); - } - return github.rest.issues.createComment({ - issue_number, - owner, - repo, - body: `@${ACTOR} ${FORMAT}\n ${data}` - }); + const issue_number = parseInt(ISSUE_NUMBER, 10); + + const fs = require('fs').promises; + const resData = await fs.readFile(`${GITHUB_WORKSPACE}/results.log`, 'utf8'); + + const commentMarker = ''; + + // List comments on the PR + const { data: comments } = await github.rest.issues.listComments({ + issue_number: issue_number, + owner: context.repo.owner, + repo: context.repo.repo }); + + // Check if a comment already exists + const comment = comments.find(comment => comment.body.includes(commentMarker)) + comment_id: comment.id, + owner: context.repo.owner, + repo: context.repo.repo, + body: `${commentMarker}\n@${ACTOR} ${FORMAT}\n${resData}` + }) + } else { + // Create a new comment + await github.rest.issues.createComment({ + issue_number: issue_number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `${commentMarker}\n@${ACTOR} ${FORMAT}\n${resData}` + }); + } env: ACTOR: ${{ github.event.client_payload.actor }} ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} From ffc2a0af927fb15d5f486306df649bdb7cca1708 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Aug 2026 15:56:32 -0700 Subject: [PATCH 5/5] add sysbench test time to script --- .../get-doltgres-doltgres-job-json.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/scripts/performance-benchmarking/get-doltgres-doltgres-job-json.sh b/.github/scripts/performance-benchmarking/get-doltgres-doltgres-job-json.sh index 4f0357d86a..5681e4139a 100755 --- a/.github/scripts/performance-benchmarking/get-doltgres-doltgres-job-json.sh +++ b/.github/scripts/performance-benchmarking/get-doltgres-doltgres-job-json.sh @@ -2,8 +2,8 @@ set -e -if [ "$#" -lt 9 ]; then - echo "Usage: ./get-job-json.sh " +if [ "$#" -lt 10 ]; then + echo "Usage: ./get-job-json.sh " exit 1 fi @@ -18,7 +18,8 @@ format="$8" issueNumber="$9" initBigRepo="${10}" nomsBinFormat="${11}" -withTpcc="${12}" +sysbenchTestTime="${12}" +withTpcc="${13}" tpccRegex="tpcc%" if [ -n "$initBigRepo" ]; then @@ -29,6 +30,10 @@ if [ -n "$nomsBinFormat" ]; then nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\"," fi +if [ -n "$sysbenchTestTime" ]; then + sysbenchTestTime="\"--sysbench-test-time=$sysbenchTestTime\"," +fi + if [ -n "$withTpcc" ]; then withTpcc="\"--withTpcc=$withTpcc\"," fi @@ -96,6 +101,7 @@ echo ' "--issue-number='$issueNumber'", "--results-dir='$timePrefix'", "--results-prefix='$actorPrefix'", + '"$sysbenchTestTime"' '"$withTpcc"' '"$initBigRepo"' '"$nomsBinFormat"'