Merge upstream v3.120.0 (Streaming Job Dispatch)

.buildkite/Dockerfile-compile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.8@sha256:273d4e65baa782dbe293c9192d600b72b17d415c1429e16bed99efcc5e61efb8
+FROM public.ecr.aws/docker/library/golang:1.25.8@sha256:779b230b2508037a8095c9e2d223a6405f8426e12233b694dbae50197b9f6d04
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest

.buildkite/Dockerfile-e2e

@@ -0,0 +1,9 @@
+FROM public.ecr.aws/docker/library/golang:1.25.8@sha256:779b230b2508037a8095c9e2d223a6405f8426e12233b694dbae50197b9f6d04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    unzip \
+    curl \
+    jq \
+    && curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install

.buildkite/Dockerfile-lint

@@ -1 +1 @@
-FROM golangci/golangci-lint:v2.5-alpine@sha256:ac072ef3a8a6aa52c04630c68a7514e06be6f634d09d5975be60f2d53b484106
+FROM golangci/golangci-lint:v2.9-alpine@sha256:efea7fae4d772680c2c2dc3a067bde22c8c0344dde7e800d110589aaee6ce977

.buildkite/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
   lint:
     build:
@@ -32,6 +30,22 @@ services:
       - GOMODCACHE=/gomodcache
       - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN
 
+  e2e:
+    build:
+      context: .
+      dockerfile: Dockerfile-e2e
+    volumes:
+      - ../:/work:cached
+    working_dir: /work
+    environment:
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_BUILD_CREATOR_EMAIL
+      - BUILDKITE_BUILD_CREATOR
+      - BUILDKITE_TRIGGERED_FROM_BUILD_ID
+      - CI_E2E_TESTS_BUILDKITE_API_TOKEN
+      - CI_E2E_TESTS_AGENT_TOKEN
+
   ruby:
     image: ruby:2.7.6
     volumes:

.buildkite/pipeline.e2e.yml

@@ -0,0 +1,21 @@
+agents:
+  queue: agent-runners-linux-amd64
+
+steps:
+  - label: ":go::sleuth_or_spy: Run agent end-to-end tests"
+    command: .buildkite/steps/e2e-tests.sh
+    secrets:
+      - CI_E2E_TESTS_BUILDKITE_API_TOKEN
+      - CI_E2E_TESTS_AGENT_TOKEN
+      - CI_E2E_TESTS_ANALYTICS_TOKEN
+    plugins:
+      - docker-compose#v4.14.0:
+          config: .buildkite/docker-compose.yml
+          cli-version: 2
+          mount-buildkite-agent: true
+          run: e2e
+      - test-collector#v1.11.0:
+          files: "junit.xml"
+          format: "junit"
+          api-token-env-name: CI_E2E_TESTS_ANALYTICS_TOKEN
+

.buildkite/pipeline.release-unstable.yml

@@ -17,6 +17,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -40,6 +41,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -64,6 +66,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - build_branch
       - docker#v5.8.0:
           environment:
             - "AWS_ACCESS_KEY_ID"
@@ -125,6 +128,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -190,6 +194,7 @@ steps:
                 - organization_slug
                 - organization_id
                 - pipeline_slug
+                - build_branch
           - ecr#v2.7.0:
               login: true
               account-ids: "445615400570"
@@ -217,6 +222,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"

.buildkite/pipeline.yml

@@ -13,24 +13,44 @@ steps:
   - name: ":go::robot_face: Lint"
     key: check-code-committed
     command: .buildkite/steps/check-code-committed.sh
-    if_changed: "{go.mod,go.sum,**.go,.buildkite/steps/check-code-committed.sh}"
+    if_changed:
+      - go.{mod,sum}
+      - "**.go"
+      - .buildkite/steps/check-code-committed.sh
     plugins:
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
           cli-version: 2
           mount-buildkite-agent: true
           run: lint
 
+  - name: ":go::robot_face: Check protobuf generation"
+    key: check-protobuf-genreation
+    command: .buildkite/steps/check-protobuf-generation.sh
+    if_changed:
+      - api/proto/**
+      - .buildkite/steps/check-protobuf-generation.sh
+    plugins:
+    - docker-compose#v4.14.0:
+        config: .buildkite/docker-compose.yml
+        cli-version: 2
+        mount-buildkite-agent: true
+        run: lint
+
   - group: ":go::scientist: Tests and Coverage"
-    if_changed: "{go.mod,go.sum,**.go,**/fixtures/**,.buildkite/steps/{tests,test-coverage-report}.sh}"
+    if_changed:
+      - go.{mod,sum}
+      - "**.go"
+      - "**/fixtures/**"
+      - .buildkite/steps/{tests,test-coverage-report}.sh
     steps:
     - name: ":linux: Linux AMD64 Tests"
       key: test-linux-amd64
       command: ".buildkite/steps/tests.sh"
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       plugins:
         - docker-compose#v4.14.0:
             config: .buildkite/docker-compose.yml
@@ -44,14 +64,14 @@ steps:
               - "os=linux"
               - "arch=amd64"
               - "race=false"
-  
+
     - name: ":linux: Linux ARM64 Tests"
       key: test-linux-arm64
       command: ".buildkite/steps/tests.sh"
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
       plugins:
@@ -67,14 +87,14 @@ steps:
               - "os=linux"
               - "arch=arm64"
               - "race=false"
-              
+
     - name: ":windows: Windows AMD64 Tests"
       key: test-windows
       command: "bash .buildkite\\steps\\tests.sh"
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_WINDOWS_QUEUE
       plugins:
@@ -85,15 +105,15 @@ steps:
               - "os=windows"
               - "arch=arm64"
               - "race=false"
-  
+
     - name: ":satellite: Detect Data Races"
       key: test-race-linux-arm64
       command: ".buildkite/steps/tests.sh -race"
       # Extra parallelism because this data race test is slow
       parallelism: 3
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
       plugins:
@@ -109,10 +129,10 @@ steps:
               - "os=linux"
               - "arch=arm64"
               - "race=true"
-              
+
     - name: ":coverage: Test coverage report Linux ARM64"
       key: test-coverage-linux-arm64
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-arm64"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -124,12 +144,11 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-linux-arm64
-
+            download: "coverage-linux-arm64/**"
+
     - name: ":coverage: Test coverage report Linux AMD64"
       key: test-coverage-linux-amd64
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-amd64"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -141,12 +160,11 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-linux-amd64
-
+            download: "coverage-linux-amd64/**"
+
     - name: ":coverage: Test coverage report Linux ARM64 Race"
       key: test-coverage-linux-arm64-race
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-arm64-race"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -158,9 +176,8 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-race-linux-arm64
-
+            download: "coverage-linux-arm64-race/**"
+
     - label: ":writing_hand: Annotate with Test Failures"
       depends_on:
         - test-linux-amd64
@@ -171,7 +188,7 @@ steps:
       plugins:
         - junit-annotate#v1.6.0:
             artifacts: junit-*.xml
-            
+
   # --- end Tests and Coverage ---
 
   - group: ":hammer_and_wrench: Binary builds"
@@ -225,9 +242,23 @@ steps:
 
             - with: { os: openbsd, arch: arm64 }
               skip: "arm64 OpenBSD is not currently supported"
-              
+
   # --- end Binary builds ---
 
+  - label: ":end::two::end: E2E Testing"
+    key: e2e-tests
+    depends_on: build-binary
+    trigger: agent-e2e-tests
+    async: false
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+
   - label: ":bathtub: Check version string is clean"
     key: check-version-string
     depends_on: build-binary
@@ -340,7 +371,7 @@ steps:
               - ubuntu-22.04
               - ubuntu-24.04
               - sidecar
-              
+
   # --- end Docker Image tests ---
 
   - name: ":debian: Debian package build"

.buildkite/steps/check-code-committed.sh

@@ -12,12 +12,14 @@ if ! git diff --no-ext-diff --exit-code; then
   exit 1
 fi
 
-echo --- :go: Checking go formatting
-gofmt -w .
-if ! git diff --no-ext-diff --exit-code; then
+echo +++ :go: Checking go formatting
+
+fumpt_out=$(go tool gofumpt -extra -l .)
+if ! [ -z "${fumpt_out}" ]; then
   echo ^^^ +++
-  echo "Files have not been formatted with gofmt."
-  echo "Fix this by running \`go fmt ./...\` locally, and committing the result."
+  echo "Files have not been formatted with gofumpt:"
+  echo "${fumpt_out}"
+  echo "Fix this by running \`gofumpt -extra -w .\` locally, and committing the result."
 
   exit 1
 fi
@@ -35,7 +37,7 @@ if ! git diff --no-ext-diff --exit-code; then
 fi
 
 echo +++ :go: Running golangci-lint...
-if ! lint_out="$(golangci-lint run --color=always)" ; then 
+if ! lint_out="$(golangci-lint run --color=always)" ; then
   echo ^^^ +++
   echo "golangci-lint found the following issues:"
   echo ""

.buildkite/steps/check-protobuf-generation.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+
+set -euf
+
+cd api/proto
+
+echo --- :buf: Installing buf...
+go install github.com/bufbuild/buf/cmd/buf@v1.61.0
+go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.10
+go install connectrpc.com/connect/cmd/protoc-gen-connect-go@v1.19.1
+
+echo --- :connectrpc: Checking protobuf file generation...
+buf generate
+if ! git diff --no-ext-diff --exit-code; then
+  echo ^^^ +++
+  echo "Generated protobuf files are out of sync with the source code"
+  echo "Please run \`buf generate\` in the internal/proto directory locally, and commit the result."
+  exit 1
+fi

.buildkite/steps/e2e-tests.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ -z "${BUILDKITE_TRIGGERED_FROM_BUILD_ID:-}" ]] ; then
+	echo "Running e2e tests on the agent that's currently running"
+	# For now, e2e test the agent that's currently running
+	CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
+	export CI_E2E_TESTS_AGENT_PATH
+else
+	echo "Running e2e tests on the agent from the triggering build"
+	# Download the artifact from the triggering build
+	ARTIFACT="pkg/buildkite-agent-$(go env GOOS)-$(go env GOARCH)"
+	buildkite-agent artifact download "${ARTIFACT}" . --build "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}"
+	chmod +x "${ARTIFACT}"
+	export CI_E2E_TESTS_AGENT_PATH="${PWD}/${ARTIFACT}"
+fi
+
+go tool gotestsum --junitfile junit.xml -- -tags e2e ./internal/e2e/...

.buildkite/steps/test-coverage-report.sh

@@ -2,5 +2,5 @@
 set -euo pipefail
 
 echo 'Producing coverage report'
-go tool covdata textfmt -i "coverage" -o cover.out
+go tool covdata textfmt -i "$1" -o cover.out
 go tool cover -html cover.out -o cover.html