From 7b0cb1c43c86fe5e89bce771a28caa0c3d97e585 Mon Sep 17 00:00:00 2001
From: Ellie <4158750+esenmarti@users.noreply.github.com>
Date: Tue, 29 Oct 2024 10:23:29 +0100
Subject: [PATCH 001/242] Support version pinning in Windows (#2)

---
 install.ps1 | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/install.ps1 b/install.ps1
index 0501bea7c3..0491dd54ce 100644
--- a/install.ps1
+++ b/install.ps1
@@ -41,20 +41,24 @@ if ([string]::IsNullOrEmpty($url)) {
     if($null -eq $version){
       $version = "latest"
     }
-    $releaseInfoUrl = "https://buildkite.com/agent/releases/$($version)?platform=windows&arch=$arch"
-    if($beta) {
-        $releaseInfoUrl = $releaseInfoUrl + "&prerelease=true"
+    if ($version -eq "latest") {
+      $releaseInfoUrl = "https://buildkite.com/agent/releases/$($version)?platform=windows&arch=$arch"
+      if($beta) {
+          $releaseInfoUrl = $releaseInfoUrl + "&prerelease=true"
+      }
+      Write-Host "Finding latest release"
+  
+      $resp = Invoke-WebRequest -Uri "$releaseInfoUrl" -UseBasicParsing -Method GET
+  
+      $releaseInfo = @{}
+      foreach ($line in $resp.Content.Split("`n")) {
+          $info = $line -split "="
+          $releaseInfo.add($info[0],$info[1])
+      }
+      $url = $releaseInfo.url
+    } else {
+      $url = "https://github.com/buildkite/agent/releases/download/v$($version)/buildkite-agent-windows-$arch-$($version).zip"
     }
-    Write-Host "Finding latest release"
-
-    $resp = Invoke-WebRequest -Uri "$releaseInfoUrl" -UseBasicParsing -Method GET
-
-    $releaseInfo = @{}
-    foreach ($line in $resp.Content.Split("`n")) {
-        $info = $line -split "="
-        $releaseInfo.add($info[0],$info[1])
-    }
-    $url = $releaseInfo.url
 }
 
 # Github requires TLS1.2

From 6908f5c62c6a384b8a17521c63d129590ec762ed Mon Sep 17 00:00:00 2001
From: Ellie <4158750+esenmarti@users.noreply.github.com>
Date: Wed, 30 Oct 2024 11:18:20 +0100
Subject: [PATCH 002/242] Support version pinning in Linux (#3)

---
 install.sh | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/install.sh b/install.sh
index 61170c02f6..12dd72f5b9 100755
--- a/install.sh
+++ b/install.sh
@@ -18,8 +18,6 @@ echo -e "\033[33m
                                                 __/ |
                                                |___/\033[0m"
 
-echo -e "Finding latest release..."
-
 SYSTEM="$(uname -s | awk '{print tolower($0)}')"
 MACHINE="$(uname -m | awk '{print tolower($0)}')"
 
@@ -73,11 +71,6 @@ else
   esac
 fi
 
-RELEASE_INFO_URL="https://buildkite.com/agent/releases/latest?platform=${PLATFORM}&arch=${ARCH}&system=${SYSTEM}&machine=${MACHINE}"
-if [[ "${BETA:-}" == "true" ]]; then
-  RELEASE_INFO_URL="${RELEASE_INFO_URL}&prerelease=true"
-fi
-
 if command -v curl >/dev/null 2>&1 ; then
   HTTP_GET="curl -LsS"
 elif command -v wget >/dev/null 2>&1 ; then
@@ -88,11 +81,24 @@ else
   exit 1
 fi
 
-LATEST_RELEASE="$(eval "${HTTP_GET} '${RELEASE_INFO_URL}'")"
+if [[ "${BUILDKITE_AGENT_VERSION:-"latest"}" == "latest" ]]; then
+    echo -e "Finding latest release..."
+
+    RELEASE_INFO_URL="https://buildkite.com/agent/releases/latest?platform=${PLATFORM}&arch=${ARCH}&system=${SYSTEM}&machine=${MACHINE}"
+    if [[ "${BETA:-}" == "true" ]]; then
+      RELEASE_INFO_URL="${RELEASE_INFO_URL}&prerelease=true"
+    fi
+
+    LATEST_RELEASE="$(eval "${HTTP_GET} '${RELEASE_INFO_URL}'")"
 
-VERSION="$(          echo "${LATEST_RELEASE}" | awk -F= '/version=/  { print $2 }')"
-DOWNLOAD_FILENAME="$(echo "${LATEST_RELEASE}" | awk -F= '/filename=/ { print $2 }')"
-DOWNLOAD_URL="$(     echo "${LATEST_RELEASE}" | awk -F= '/url=/      { print $2 }')"
+    VERSION="$(          echo "${LATEST_RELEASE}" | awk -F= '/version=/  { print $2 }')"
+    DOWNLOAD_FILENAME="$(echo "${LATEST_RELEASE}" | awk -F= '/filename=/ { print $2 }')"
+    DOWNLOAD_URL="$(     echo "${LATEST_RELEASE}" | awk -F= '/url=/      { print $2 }')"
+else
+    VERSION=$BUILDKITE_AGENT_VERSION
+    DOWNLOAD_FILENAME="buildkite-agent-${PLATFORM}-${ARCH}-${VERSION}.tar.gz"
+    DOWNLOAD_URL="https://github.com/buildkite/agent/releases/download/v${VERSION}/${DOWNLOAD_FILENAME}"
+fi
 
 if [[ "${DISABLE_CHECKSUM_VERIFICATION:-}" != "true" ]]; then
   if command -v openssl >/dev/null 2>&1 ; then

From 968897e0b335ff487bb6c9578b5230c122ec0de7 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 5 Mar 2025 17:37:06 +1100
Subject: [PATCH 003/242] Catch all 'ignored' vars

---
 agent/job_runner.go | 140 +++++++++++++++++++++++---------------------
 1 file changed, 73 insertions(+), 67 deletions(-)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index 35fb1984a2..6a84efbe59 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -423,6 +423,16 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
 		}
 	}
 
+	// Wrap setting values in env, so that when any that were already present in
+	// supplied Job env are overwritten, they can be added to ignoredEnv.
+	var ignoredEnv []string
+	setEnv := func(name, value string) {
+		if _, exists := env[name]; exists {
+			ignoredEnv = append(ignoredEnv, name)
+		}
+		env[name] = value
+	}
+
 	// Write out the job environment to file:
 	// - envShellFile: in k="v" format, with newlines escaped. If the
 	//   propagate-agent-vars experiment is enabled, the names of several agent
@@ -488,29 +498,15 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	// Now that the env files have been written, we can add their corresponding
 	// paths to the job env.
 	if r.envShellFile != nil {
-		env["BUILDKITE_ENV_FILE"] = r.envShellFile.Name()
+		setEnv("BUILDKITE_ENV_FILE", r.envShellFile.Name())
 	}
 	if r.envJSONFile != nil {
-		env["BUILDKITE_ENV_JSON_FILE"] = r.envJSONFile.Name()
-	}
-
-	var ignoredEnv []string
-
-	// Check if the user has defined any protected env
-	for k := range envutil.ProtectedEnv {
-		if _, exists := r.conf.Job.Env[k]; exists {
-			ignoredEnv = append(ignoredEnv, k)
-		}
+		setEnv("BUILDKITE_ENV_JSON_FILE", r.envJSONFile.Name())
 	}
 
 	cache := r.conf.Job.Step.Cache
 	if cache != nil && len(cache.Paths) > 0 {
-		env["BUILDKITE_AGENT_CACHE_PATHS"] = strings.Join(cache.Paths, ",")
-	}
-
-	// Set BUILDKITE_IGNORED_ENV so the bootstrap can show warnings
-	if len(ignoredEnv) > 0 {
-		env["BUILDKITE_IGNORED_ENV"] = strings.Join(ignoredEnv, ",")
+		setEnv("BUILDKITE_AGENT_CACHE_PATHS", strings.Join(cache.Paths, ","))
 	}
 
 	// Set BUILDKITE_SECRETS_CONFIG so bootstrap can access secrets configuration
@@ -521,14 +517,14 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 			return nil, err
 		}
 
-		env["BUILDKITE_SECRETS_CONFIG"] = string(secretsJSON)
+		setEnv("BUILDKITE_SECRETS_CONFIG", string(secretsJSON))
 	}
 
 	// Add the API configuration
 	apiConfig := r.apiClient.Config()
-	env["BUILDKITE_AGENT_ENDPOINT"] = apiConfig.Endpoint
-	env["BUILDKITE_AGENT_ACCESS_TOKEN"] = apiConfig.Token
-	env["BUILDKITE_NO_HTTP2"] = fmt.Sprint(apiConfig.DisableHTTP2)
+	setEnv("BUILDKITE_AGENT_ENDPOINT", apiConfig.Endpoint)
+	setEnv("BUILDKITE_AGENT_ACCESS_TOKEN", apiConfig.Token)
+	setEnv("BUILDKITE_NO_HTTP2", fmt.Sprint(apiConfig.DisableHTTP2))
 
 	// ... including any server-specified request headers, so that sub-processes such as
 	// buildkite-agent annotate etc can respect them.
@@ -543,9 +539,9 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	}
 
 	// Add agent environment variables
-	env["BUILDKITE_AGENT_DEBUG"] = fmt.Sprint(r.conf.Debug)
-	env["BUILDKITE_AGENT_DEBUG_HTTP"] = fmt.Sprint(r.conf.DebugHTTP)
-	env["BUILDKITE_AGENT_PID"] = strconv.Itoa(os.Getpid())
+	setEnv("BUILDKITE_AGENT_DEBUG", fmt.Sprint(r.conf.Debug))
+	setEnv("BUILDKITE_AGENT_DEBUG_HTTP", fmt.Sprint(r.conf.DebugHTTP))
+	setEnv("BUILDKITE_AGENT_PID", strconv.Itoa(os.Getpid()))
 
 	// We know the BUILDKITE_BIN_PATH dir, because it's the path to the
 	// currently running file (there is only 1 binary)
@@ -554,77 +550,80 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	if err != nil {
 		return nil, err
 	}
-	env["BUILDKITE_BIN_PATH"] = filepath.Dir(exePath)
+
+	setEnv("BUILDKITE_BIN_PATH", filepath.Dir(exePath))
 
 	// Add options from the agent configuration
-	env["BUILDKITE_CONFIG_PATH"] = r.conf.AgentConfiguration.ConfigPath
-	env["BUILDKITE_BUILD_PATH"] = r.conf.AgentConfiguration.BuildPath
-	env["BUILDKITE_SOCKETS_PATH"] = r.conf.AgentConfiguration.SocketsPath
-	env["BUILDKITE_GIT_MIRRORS_PATH"] = r.conf.AgentConfiguration.GitMirrorsPath
-	env["BUILDKITE_GIT_MIRRORS_SKIP_UPDATE"] = fmt.Sprint(r.conf.AgentConfiguration.GitMirrorsSkipUpdate)
-	env["BUILDKITE_HOOKS_PATH"] = r.conf.AgentConfiguration.HooksPath
-	env["BUILDKITE_ADDITIONAL_HOOKS_PATHS"] = strings.Join(r.conf.AgentConfiguration.AdditionalHooksPaths, ",")
-	env["BUILDKITE_PLUGINS_PATH"] = r.conf.AgentConfiguration.PluginsPath
-	env["BUILDKITE_SSH_KEYSCAN"] = fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan)
-	env["BUILDKITE_GIT_SUBMODULES"] = fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules)
-	env["BUILDKITE_COMMAND_EVAL"] = fmt.Sprint(r.conf.AgentConfiguration.CommandEval)
-	env["BUILDKITE_PLUGINS_ENABLED"] = fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled)
-	env["BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH"] = fmt.Sprint(r.conf.AgentConfiguration.PluginsAlwaysCloneFresh)
-	env["BUILDKITE_LOCAL_HOOKS_ENABLED"] = fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled)
-	env["BUILDKITE_GIT_CHECKOUT_FLAGS"] = r.conf.AgentConfiguration.GitCheckoutFlags
-	env["BUILDKITE_GIT_CLONE_FLAGS"] = r.conf.AgentConfiguration.GitCloneFlags
-	env["BUILDKITE_GIT_FETCH_FLAGS"] = r.conf.AgentConfiguration.GitFetchFlags
-	env["BUILDKITE_GIT_CLONE_MIRROR_FLAGS"] = r.conf.AgentConfiguration.GitCloneMirrorFlags
-	env["BUILDKITE_GIT_CLEAN_FLAGS"] = r.conf.AgentConfiguration.GitCleanFlags
-	env["BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT"] = strconv.Itoa(r.conf.AgentConfiguration.GitMirrorsLockTimeout)
-	env["BUILDKITE_SHELL"] = r.conf.AgentConfiguration.Shell
-	env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(ctx), ",")
-	env["BUILDKITE_REDACTED_VARS"] = strings.Join(r.conf.AgentConfiguration.RedactedVars, ",")
-	env["BUILDKITE_STRICT_SINGLE_HOOKS"] = fmt.Sprint(r.conf.AgentConfiguration.StrictSingleHooks)
-	env["BUILDKITE_CANCEL_GRACE_PERIOD"] = strconv.Itoa(r.conf.AgentConfiguration.CancelGracePeriod)
-	env["BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS"] = strconv.Itoa(int(r.conf.AgentConfiguration.SignalGracePeriod / time.Second))
-	env["BUILDKITE_TRACE_CONTEXT_ENCODING"] = r.conf.AgentConfiguration.TraceContextEncoding
+	setEnv("BUILDKITE_CONFIG_PATH", r.conf.AgentConfiguration.ConfigPath)
+	setEnv("BUILDKITE_BUILD_PATH", r.conf.AgentConfiguration.BuildPath)
+	setEnv("BUILDKITE_SOCKETS_PATH", r.conf.AgentConfiguration.SocketsPath)
+	setEnv("BUILDKITE_GIT_MIRRORS_PATH", r.conf.AgentConfiguration.GitMirrorsPath)
+	setEnv("BUILDKITE_GIT_MIRRORS_SKIP_UPDATE", fmt.Sprint(r.conf.AgentConfiguration.GitMirrorsSkipUpdate))
+	setEnv("BUILDKITE_HOOKS_PATH", r.conf.AgentConfiguration.HooksPath)
+	setEnv("BUILDKITE_ADDITIONAL_HOOKS_PATHS", strings.Join(r.conf.AgentConfiguration.AdditionalHooksPaths, ","))
+	setEnv("BUILDKITE_PLUGINS_PATH", r.conf.AgentConfiguration.PluginsPath)
+	setEnv("BUILDKITE_SSH_KEYSCAN", fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan))
+	setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
+	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
+	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
+	setEnv("BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", fmt.Sprint(r.conf.AgentConfiguration.PluginsAlwaysCloneFresh))
+	setEnv("BUILDKITE_LOCAL_HOOKS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled))
+
+	setEnv("BUILDKITE_GIT_CHECKOUT_FLAGS", r.conf.AgentConfiguration.GitCheckoutFlags)
+	setEnv("BUILDKITE_GIT_CLONE_FLAGS", r.conf.AgentConfiguration.GitCloneFlags)
+	setEnv("BUILDKITE_GIT_FETCH_FLAGS", r.conf.AgentConfiguration.GitFetchFlags)
+	setEnv("BUILDKITE_GIT_CLONE_MIRROR_FLAGS", r.conf.AgentConfiguration.GitCloneMirrorFlags)
+	setEnv("BUILDKITE_GIT_CLEAN_FLAGS", r.conf.AgentConfiguration.GitCleanFlags)
+	setEnv("BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", strconv.Itoa(r.conf.AgentConfiguration.GitMirrorsLockTimeout))
+
+	setEnv("BUILDKITE_SHELL", r.conf.AgentConfiguration.Shell)
+	setEnv("BUILDKITE_AGENT_EXPERIMENT", strings.Join(experiments.Enabled(ctx), ","))
+	setEnv("BUILDKITE_REDACTED_VARS", strings.Join(r.conf.AgentConfiguration.RedactedVars, ","))
+	setEnv("BUILDKITE_STRICT_SINGLE_HOOKS", fmt.Sprint(r.conf.AgentConfiguration.StrictSingleHooks))
+	setEnv("BUILDKITE_CANCEL_GRACE_PERIOD", strconv.Itoa(r.conf.AgentConfiguration.CancelGracePeriod))
+	setEnv("BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS", strconv.Itoa(int(r.conf.AgentConfiguration.SignalGracePeriod/time.Second)))
+	setEnv("BUILDKITE_TRACE_CONTEXT_ENCODING", r.conf.AgentConfiguration.TraceContextEncoding)
 
 	if r.conf.KubernetesExec {
-		env["BUILDKITE_KUBERNETES_EXEC"] = "true"
+		setEnv("BUILDKITE_KUBERNETES_EXEC", "true")
 	}
 
 	if !r.conf.AgentConfiguration.AllowMultipartArtifactUpload {
-		env["BUILDKITE_NO_MULTIPART_ARTIFACT_UPLOAD"] = "true"
+		setEnv("BUILDKITE_NO_MULTIPART_ARTIFACT_UPLOAD", "true")
 	}
 
 	// propagate CancelSignal to bootstrap, unless it's the default SIGTERM
 	if r.conf.CancelSignal != process.SIGTERM {
-		env["BUILDKITE_CANCEL_SIGNAL"] = r.conf.CancelSignal.String()
+		setEnv("BUILDKITE_CANCEL_SIGNAL", r.conf.CancelSignal.String())
 	}
 
 	// Whether to enable profiling in the bootstrap
 	if r.conf.AgentConfiguration.Profile != "" {
-		env["BUILDKITE_AGENT_PROFILE"] = r.conf.AgentConfiguration.Profile
+		setEnv("BUILDKITE_AGENT_PROFILE", r.conf.AgentConfiguration.Profile)
 	}
 
 	// PTY-mode is enabled by default in `start` and `bootstrap`, so we only need
 	// to propagate it if it's explicitly disabled.
 	if !r.conf.AgentConfiguration.RunInPty {
-		env["BUILDKITE_PTY"] = "false"
+		setEnv("BUILDKITE_PTY", "false")
 	}
 
 	// pass through the KMS key ID for signing
 	if r.conf.AgentConfiguration.SigningAWSKMSKey != "" {
-		env["BUILDKITE_AGENT_AWS_KMS_KEY"] = r.conf.AgentConfiguration.SigningAWSKMSKey
+		setEnv("BUILDKITE_AGENT_AWS_KMS_KEY", r.conf.AgentConfiguration.SigningAWSKMSKey)
 	}
 
 	// Pass signing details through to the executor - any pipelines uploaded by this agent will be signed
 	if r.conf.AgentConfiguration.SigningJWKSFile != "" {
-		env["BUILDKITE_AGENT_JWKS_FILE"] = r.conf.AgentConfiguration.SigningJWKSFile
+		setEnv("BUILDKITE_AGENT_JWKS_FILE", r.conf.AgentConfiguration.SigningJWKSFile)
 	}
 
 	if r.conf.AgentConfiguration.SigningJWKSKeyID != "" {
-		env["BUILDKITE_AGENT_JWKS_KEY_ID"] = r.conf.AgentConfiguration.SigningJWKSKeyID
+		setEnv("BUILDKITE_AGENT_JWKS_KEY_ID", r.conf.AgentConfiguration.SigningJWKSKeyID)
 	}
 
 	if r.conf.AgentConfiguration.DebugSigning {
-		env["BUILDKITE_AGENT_DEBUG_SIGNING"] = "true"
+		setEnv("BUILDKITE_AGENT_DEBUG_SIGNING", "true")
 	}
 
 	enablePluginValidation := r.conf.AgentConfiguration.PluginValidation
@@ -633,28 +632,30 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	if pluginValidation, ok := env["BUILDKITE_PLUGIN_VALIDATION"]; ok {
 		switch pluginValidation {
 		case "true", "1", "on":
+			// Skip ignoredEnv by pretending it wasn't set by the job.
+			delete(env, "BUILDKITE_PLUGIN_VALIDATION")
 			enablePluginValidation = true
 		}
 	}
-	env["BUILDKITE_PLUGIN_VALIDATION"] = fmt.Sprint(enablePluginValidation)
+	setEnv("BUILDKITE_PLUGIN_VALIDATION", fmt.Sprint(enablePluginValidation))
 
 	if r.conf.AgentConfiguration.TracingBackend != "" {
-		env["BUILDKITE_TRACING_BACKEND"] = r.conf.AgentConfiguration.TracingBackend
-		env["BUILDKITE_TRACING_SERVICE_NAME"] = r.conf.AgentConfiguration.TracingServiceName
+		setEnv("BUILDKITE_TRACING_BACKEND", r.conf.AgentConfiguration.TracingBackend)
+		setEnv("BUILDKITE_TRACING_SERVICE_NAME", r.conf.AgentConfiguration.TracingServiceName)
 
 		// Buildkite backend can provide a traceparent property on the job
 		// which can be propagated to the job tracing if OpenTelemetry is used
 		//
 		// https://www.w3.org/TR/trace-context/#traceparent-header
 		if r.conf.Job.TraceParent != "" {
-			env["BUILDKITE_TRACING_TRACEPARENT"] = r.conf.Job.TraceParent
+			setEnv("BUILDKITE_TRACING_TRACEPARENT", r.conf.Job.TraceParent)
 		}
 		if r.conf.AgentConfiguration.TracingPropagateTraceparent {
-			env["BUILDKITE_TRACING_PROPAGATE_TRACEPARENT"] = "true"
+			setEnv("BUILDKITE_TRACING_PROPAGATE_TRACEPARENT", "true")
 		}
 	}
 
-	env["BUILDKITE_AGENT_DISABLE_WARNINGS_FOR"] = strings.Join(r.conf.AgentConfiguration.DisableWarningsFor, ",")
+	setEnv("BUILDKITE_AGENT_DISABLE_WARNINGS_FOR", strings.Join(r.conf.AgentConfiguration.DisableWarningsFor, ","))
 
 	// see documentation for BuildkiteMessageMax
 	if err := truncateEnv(r.agentLogger, env, BuildkiteMessageName, BuildkiteMessageMax); err != nil {
@@ -662,6 +663,11 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 		// attempt to continue anyway
 	}
 
+	// Finally, set BUILDKITE_IGNORED_ENV so the bootstrap can show warnings.
+	if len(ignoredEnv) > 0 {
+		env["BUILDKITE_IGNORED_ENV"] = strings.Join(ignoredEnv, ",")
+	}
+
 	// Convert the env map into a slice (which is what the script gear
 	// needs)
 	envSlice := []string{}

From 3569db553c5aaa2d09874dee991bb20df8f807f8 Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Wed, 15 Oct 2025 14:00:48 +0100
Subject: [PATCH 004/242] docs: Add examples for step update commands for
 priority and notify attributes

---
 clicommand/step_update.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/clicommand/step_update.go b/clicommand/step_update.go
index 53b5f4ad53..9924a656d7 100644
--- a/clicommand/step_update.go
+++ b/clicommand/step_update.go
@@ -37,7 +37,10 @@ Example:
     $ buildkite-agent step update "label" "New Label"
     $ buildkite-agent step update "label" " (add to end of label)" --append
     $ buildkite-agent step update "label" < ./tmp/some-new-label
-    $ ./script/label-generator | buildkite-agent step update "label"`
+    $ ./script/label-generator | buildkite-agent step update "label"
+	$ buildkite-agent step update "priority" 10 --step "my-step-key"
+	$ buildkite-agent step update "notify" '[{"github_commit_status": {"context": "my-context"}}]' --append
+`
 
 type StepUpdateConfig struct {
 	GlobalConfig

From d09c9ad5e279ec02b153af61890bf8234817da63 Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Wed, 15 Oct 2025 15:38:45 +0100
Subject: [PATCH 005/242] docs: Add Slack example for step update commands for
 notify attribute

---
 clicommand/step_update.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clicommand/step_update.go b/clicommand/step_update.go
index 9924a656d7..205a674dd5 100644
--- a/clicommand/step_update.go
+++ b/clicommand/step_update.go
@@ -40,6 +40,7 @@ Example:
     $ ./script/label-generator | buildkite-agent step update "label"
 	$ buildkite-agent step update "priority" 10 --step "my-step-key"
 	$ buildkite-agent step update "notify" '[{"github_commit_status": {"context": "my-context"}}]' --append
+	$ buildkite-agent step update "notify" '[{"slack": "my-slack-workspace#my-channel"}]' --append
 `
 
 type StepUpdateConfig struct {

From 24c536ae43a5ab5cb0e3862bcca23568ecdd4d34 Mon Sep 17 00:00:00 2001
From: Pete Tomasik <senate6268@gmail.com>
Date: Thu, 16 Oct 2025 14:59:42 -0400
Subject: [PATCH 006/242] Update URLs in agent cfg comments

---
 .../root/usr/share/buildkite-agent/buildkite-agent.cfg     | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg
index 3bf54c117c..98f4e35cef 100644
--- a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg
+++ b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg
@@ -67,14 +67,15 @@ plugins-path="/etc/buildkite-agent/plugins"
 # no-color=true
 
 # The next two options are relevant to the Datadog integration, available 3.7.0 and on
-# See https://forum.buildkite.community/t/about-our-datadog-integration/216 for details
+# See https://buildkite.com/docs/agent/v3/configuration#metrics-datadog
 # Send metrics to DogStatsD running on metrics-datadog-host
 # metrics-datadog=true
 
 # Host to collect Buildkite metrics
-# datadog-agent will need to run DogStatsD, presumed on port 8125. 
+# datadog-agent will need to run DogStatsD, presumed on port 8125.
+# See https://buildkite.com/docs/agent/v3/configuration#metrics-datadog-host
 # Specify port below like my-host:8126 if not using 8125
 # metrics-datadog-host=127.0.0.1
 
-# If set and valid, the given tracing backend will be enabled. Eg: datadog
+# If set and valid, the given tracing backend will be enabled. Eg: datadog, opentelemetry
 # tracing-backend=""

From 867b5c443fca0374cbae1d620ebe3cacf9d99089 Mon Sep 17 00:00:00 2001
From: Danny Fallon <danny@intercom.io>
Date: Fri, 17 Oct 2025 13:21:01 +0100
Subject: [PATCH 007/242] Upgrade Datadog-go to v5.8.1 to work around mod
 checksum issues

See https://github.com/DataDog/datadog-go/issues/340 for more details
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index eb497952c6..bb5aa01605 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	drjosh.dev/zzglob v0.4.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2
-	github.com/DataDog/datadog-go/v5 v5.8.0
+	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go v1.55.8
 	github.com/aws/aws-sdk-go-v2 v1.39.2
diff --git a/go.sum b/go.sum
index 535c8d2a19..978ad90948 100644
--- a/go.sum
+++ b/go.sum
@@ -41,8 +41,8 @@ github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0 h1:aIWF85OKxXGo7rVyqJ
 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0/go.mod h1:Lfap5FuM4b/Pw9IrTuAvWBWZEmXOvZhCya3dYv4G8O0=
 github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyLGxp8E6j2f2Mh5IPbYmQ=
 github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8=
-github.com/DataDog/datadog-go/v5 v5.8.0 h1:pKZtux5CfqkqGYGvKCM3wV5i8sYAzcddK7nkrChUtxo=
-github.com/DataDog/datadog-go/v5 v5.8.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/DataDog/datadog-go/v5 v5.8.1 h1:+GOES5W9zpKlhwHptZVW2C0NLVf7ilr7pHkDcbNvpIc=
+github.com/DataDog/datadog-go/v5 v5.8.1/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/DataDog/dd-trace-go/v2 v2.2.3 h1:6RvVdY9suR/rYYYZHjx4txrtSYcRZ5u5Cs2sXMsIBf4=
 github.com/DataDog/dd-trace-go/v2 v2.2.3/go.mod h1:1LcqWELgQwgk6x7sO0MXUgsvxcAVjxSA423cUjvUqR0=
 github.com/DataDog/go-libddwaf/v4 v4.3.2 h1:YGvW2Of1C4e1yU+p7iibmhN2zEOgi9XEchbhQjBxb/A=

From d4e2b19834727b77fbd63a1261ab73c735ff9883 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 20 Oct 2025 10:28:59 +1100
Subject: [PATCH 008/242] Bump zzglob to v0.4.2

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index eb497952c6..5a715098fc 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ toolchain go1.24.5
 
 require (
 	cloud.google.com/go/compute/metadata v0.9.0
-	drjosh.dev/zzglob v0.4.1
+	drjosh.dev/zzglob v0.4.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2
 	github.com/DataDog/datadog-go/v5 v5.8.0
diff --git a/go.sum b/go.sum
index 535c8d2a19..2ee71a6dcd 100644
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
-drjosh.dev/zzglob v0.4.1 h1:HIf8v8REooNw//kGDUgSqPG5uMY7D69/jfZTUjmRT6A=
-drjosh.dev/zzglob v0.4.1/go.mod h1:SbYDdesQC13iyGiEwV8dJfJbyz7/Qiawrd5ODdJQCoo=
+drjosh.dev/zzglob v0.4.2 h1:q+e5Cp6SFCyz+Yurhk/edSrTKEk3tn60vzoaXLmtiBo=
+drjosh.dev/zzglob v0.4.2/go.mod h1:SbYDdesQC13iyGiEwV8dJfJbyz7/Qiawrd5ODdJQCoo=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 h1:KpMC6LFL7mqpExyMC9jVOYRiVhLmamjeZfRsUpB7l4s=

From 0d14838e352c0cad7dfba20144aea4448915a823 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Oct 2025 00:22:08 +0000
Subject: [PATCH 009/242] build(deps): bump the golang-x group with 5 updates

Bumps the golang-x group with 5 updates:

| Package | From | To |
| --- | --- | --- |
| [golang.org/x/crypto](https://github.com/golang/crypto) | `0.42.0` | `0.43.0` |
| [golang.org/x/net](https://github.com/golang/net) | `0.45.0` | `0.46.0` |
| [golang.org/x/oauth2](https://github.com/golang/oauth2) | `0.31.0` | `0.32.0` |
| [golang.org/x/sys](https://github.com/golang/sys) | `0.36.0` | `0.37.0` |
| [golang.org/x/term](https://github.com/golang/term) | `0.35.0` | `0.36.0` |


Updates `golang.org/x/crypto` from 0.42.0 to 0.43.0
- [Commits](https://github.com/golang/crypto/compare/v0.42.0...v0.43.0)

Updates `golang.org/x/net` from 0.45.0 to 0.46.0
- [Commits](https://github.com/golang/net/compare/v0.45.0...v0.46.0)

Updates `golang.org/x/oauth2` from 0.31.0 to 0.32.0
- [Commits](https://github.com/golang/oauth2/compare/v0.31.0...v0.32.0)

Updates `golang.org/x/sys` from 0.36.0 to 0.37.0
- [Commits](https://github.com/golang/sys/compare/v0.36.0...v0.37.0)

Updates `golang.org/x/term` from 0.35.0 to 0.36.0
- [Commits](https://github.com/golang/term/compare/v0.35.0...v0.36.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.43.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/net
  dependency-version: 0.46.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/oauth2
  dependency-version: 0.32.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/sys
  dependency-version: 0.37.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/term
  dependency-version: 0.36.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 16 ++++++++--------
 go.sum | 32 ++++++++++++++++----------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/go.mod b/go.mod
index 5e81d02db6..d74c4f2558 100644
--- a/go.mod
+++ b/go.mod
@@ -51,12 +51,12 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
 	go.opentelemetry.io/otel/sdk v1.38.0
 	go.opentelemetry.io/otel/trace v1.38.0
-	golang.org/x/crypto v0.42.0
-	golang.org/x/net v0.45.0
-	golang.org/x/oauth2 v0.31.0
+	golang.org/x/crypto v0.43.0
+	golang.org/x/net v0.46.0
+	golang.org/x/oauth2 v0.32.0
 	golang.org/x/sync v0.17.0
-	golang.org/x/sys v0.36.0
-	golang.org/x/term v0.35.0
+	golang.org/x/sys v0.37.0
+	golang.org/x/term v0.36.0
 	google.golang.org/api v0.252.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.6
 	gopkg.in/yaml.v3 v3.0.1
@@ -181,10 +181,10 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
-	golang.org/x/mod v0.27.0 // indirect
-	golang.org/x/text v0.29.0 // indirect
+	golang.org/x/mod v0.28.0 // indirect
+	golang.org/x/text v0.30.0 // indirect
 	golang.org/x/time v0.13.0 // indirect
-	golang.org/x/tools v0.36.0 // indirect
+	golang.org/x/tools v0.37.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251002232023-7c0ddcbb5797 // indirect
diff --git a/go.sum b/go.sum
index 4496a28b49..6ddcec61e1 100644
--- a/go.sum
+++ b/go.sum
@@ -459,24 +459,24 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
-golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
+golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
+golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
-golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
+golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U=
+golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
-golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
-golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo=
-golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
+golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
+golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
+golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -496,15 +496,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
-golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
-golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
+golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
-golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
+golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
 golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
 golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -512,8 +512,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
-golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
+golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
+golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From b0be890b3ebc9c7dca1d370da53885e68e465f60 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Oct 2025 00:22:27 +0000
Subject: [PATCH 010/242] build(deps): bump the container-images group across 3
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `f1e7f14` to `ce7c987`

Updates `buildkite/agent-base` from `d1ed232` to `80f3117`

Updates `buildkite/agent-base` from `f89600e` to `cac895b`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index ac950acc29..dfa28216d5 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:f1e7f1461bc1fe72efa5befcee02a43321c7e3f3658ec5f61918470c6c970993
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:ce7c9875df8990a8e6422c48604d81b074feeefa96843724f0bff6d7d07f1f86
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 933f8f71f2..dc128ea7ad 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:d1ed2320a26f7f34cd02d09e99ee7ad6879c1c8bd3f009cca4bf7975cdc067bf
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:80f31170e2499fe3f31a34f8017b463711aa5fce0f14ff22ace94bb1e3953a53
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index c73697fd1f..5b7ae16645 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:f89600e2bd6e03c702fa400bfbc5dc3cd3f99a57726fc79fc18efe6d5c804dd5
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:cac895b78d2eb6fc8df165d19104cdfa0a95cd51b6f7df3d74cb8ae9517823f0
 ARG TARGETOS
 ARG TARGETARCH
 

From 3db5820b04bbd86a43cbfdf0c375ddfe7e1ecd81 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Oct 2025 00:26:38 +0000
Subject: [PATCH 011/242] build(deps): bump the cloud-providers group across 1
 directory with 6 updates

Bumps the cloud-providers group with 5 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [github.com/Azure/azure-sdk-for-go/sdk/storage/azblob](https://github.com/Azure/azure-sdk-for-go) | `1.6.2` | `1.6.3` |
| [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.39.2` | `1.39.3` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.12` | `1.31.13` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.245.2` | `1.257.2` |
| [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) | `1.45.6` | `1.46.0` |



Updates `github.com/Azure/azure-sdk-for-go/sdk/storage/azblob` from 1.6.2 to 1.6.3
- [Release notes](https://github.com/Azure/azure-sdk-for-go/releases)
- [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/sdk-breaking-changes-guide-migration.md)
- [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/storage/azblob/v1.6.2...sdk/storage/azblob/v1.6.3)

Updates `github.com/aws/aws-sdk-go-v2` from 1.39.2 to 1.39.3
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.39.2...v1.39.3)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.12 to 1.31.13
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.12...config/v1.31.13)

Updates `github.com/aws/aws-sdk-go-v2/feature/ec2/imds` from 1.18.9 to 1.18.10
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.9...config/v1.18.10)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.245.2 to 1.257.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.245.2...service/ec2/v1.257.2)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.45.6 to 1.46.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/kms/v1.45.6...service/s3/v1.46.0)

---
updated-dependencies:
- dependency-name: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob
  dependency-version: 1.6.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2
  dependency-version: 1.39.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.31.13
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/ec2/imds
  dependency-version: 1.18.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.257.2
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.46.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 32 ++++++++++++++---------------
 go.sum | 64 +++++++++++++++++++++++++++++-----------------------------
 2 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/go.mod b/go.mod
index 5e81d02db6..17920098f6 100644
--- a/go.mod
+++ b/go.mod
@@ -8,15 +8,15 @@ require (
 	cloud.google.com/go/compute/metadata v0.9.0
 	drjosh.dev/zzglob v0.4.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.39.2
-	github.com/aws/aws-sdk-go-v2/config v1.31.12
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.9
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.245.2
-	github.com/aws/aws-sdk-go-v2/service/kms v1.45.6
+	github.com/aws/aws-sdk-go-v2 v1.39.3
+	github.com/aws/aws-sdk-go-v2/config v1.31.13
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2
+	github.com/aws/aws-sdk-go-v2/service/kms v1.46.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
 	github.com/buildkite/go-pipeline v0.16.0
@@ -91,16 +91,16 @@ require (
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.29.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.38.6 // indirect
-	github.com/aws/smithy-go v1.23.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 // indirect
+	github.com/aws/smithy-go v1.23.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/buildkite/test-engine-client v1.6.0 // indirect
diff --git a/go.sum b/go.sum
index 4496a28b49..7c0e3458bd 100644
--- a/go.sum
+++ b/go.sum
@@ -16,8 +16,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDo
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI=
@@ -80,36 +80,36 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I=
-github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY=
-github.com/aws/aws-sdk-go-v2/config v1.31.12 h1:pYM1Qgy0dKZLHX2cXslNacbcEFMkDMl+Bcj5ROuS6p8=
-github.com/aws/aws-sdk-go-v2/config v1.31.12/go.mod h1:/MM0dyD7KSDPR+39p9ZNVKaHDLb9qnfDurvVS2KAhN8=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.16 h1:4JHirI4zp958zC026Sm+V4pSDwW4pwLefKrc0bF2lwI=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.16/go.mod h1:qQMtGx9OSw7ty1yLclzLxXCRbrkjWAM7JnObZjmCB7I=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.9 h1:Mv4Bc0mWmv6oDuSWTKnk+wgeqPL5DRFu5bQL9BGPQ8Y=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.9/go.mod h1:IKlKfRppK2a1y0gy1yH6zD+yX5uplJ6UuPlgd48dJiQ=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 h1:se2vOWGD3dWQUtfn4wEjRQJb1HK1XsNIt825gskZ970=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9/go.mod h1:hijCGH2VfbZQxqCDN7bwz/4dzxV+hkyhjawAtdPWKZA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 h1:6RBnKZLkJM4hQ+kN6E7yWFveOTg8NLPHAkqrs4ZPlTU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9/go.mod h1:V9rQKRmK7AWuEsOMnHzKj8WyrIir1yUJbZxDuZLFvXI=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.245.2 h1:P94OfRObDwjklbvdJTGuRZXeGYF7Bv5NNUo+I628kKQ=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.245.2/go.mod h1:D8Wb993SJuFQ10Lp95Vod8VTpYjJz4v0LeW4rEI471c=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 h1:5r34CgVOD4WZudeEKZ9/iKpiT6cM1JyEROpXjOcdWv8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9/go.mod h1:dB12CEbNWPbzO2uC6QSWHteqOg4JfBVJOojbAoAUb5I=
-github.com/aws/aws-sdk-go-v2/service/kms v1.45.6 h1:Br3kil4j7RPW+7LoLVkYt8SuhIWlg6ylmbmzXJ7PgXY=
-github.com/aws/aws-sdk-go-v2/service/kms v1.45.6/go.mod h1:FKXkHzw1fJZtg1P1qoAIiwen5thz/cDRTTDCIu8ljxc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.6 h1:A1oRkiSQOWstGh61y4Wc/yQ04sqrQZr1Si/oAXj20/s=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.6/go.mod h1:5PfYspyCU5Vw1wNPsxi15LZovOnULudOQuVxphSflQA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.1 h1:5fm5RTONng73/QA73LhCNR7UT9RpFH3hR6HWL6bIgVY=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.1/go.mod h1:xBEjWD13h+6nq+z4AkqSfSvqRKFgDIQeaMguAJndOWo=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.6 h1:p3jIvqYwUZgu/XYeI48bJxOhvm47hZb5HUQ0tn6Q9kA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.6/go.mod h1:WtKK+ppze5yKPkZ0XwqIVWD4beCwv056ZbPQNoeHqM8=
-github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE=
-github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/aws/aws-sdk-go-v2 v1.39.3 h1:h7xSsanJ4EQJXG5iuW4UqgP7qBopLpj84mpkNx3wPjM=
+github.com/aws/aws-sdk-go-v2 v1.39.3/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
+github.com/aws/aws-sdk-go-v2/config v1.31.13 h1:wcqQB3B0PgRPUF5ZE/QL1JVOyB0mbPevHFoAMpemR9k=
+github.com/aws/aws-sdk-go-v2/config v1.31.13/go.mod h1:ySB5D5ybwqGbT6c3GszZ+u+3KvrlYCUQNo62+hkKOFk=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.17 h1:skpEwzN/+H8cdrrtT8y+rvWJGiWWv0DeNAe+4VTf+Vs=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.17/go.mod h1:Ed+nXsaYa5uBINovJhcAWkALvXw2ZLk36opcuiSZfJM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 h1:UuGVOX48oP4vgQ36oiKmW9RuSeT8jlgQgBFQD+HUiHY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10/go.mod h1:vM/Ini41PzvudT4YkQyE/+WiQJiQ6jzeDyU8pQKwCac=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 h1:mj/bdWleWEh81DtpdHKkw41IrS+r3uw1J/VQtbwYYp8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10/go.mod h1:7+oEMxAZWP8gZCyjcm9VicI0M61Sx4DJtcGfKYv2yKQ=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 h1:wh+/mn57yhUrFtLIxyFPh2RgxgQz/u+Yrf7hiHGHqKY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10/go.mod h1:7zirD+ryp5gitJJ2m1BBux56ai8RIRDykXZrJSp540w=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2 h1:D8MCemFa8rt09x7o6Fkm2T7ThVbRPrD91R+LKhVEnVU=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2/go.mod h1:Q/kZ++hvhasMpQU37I7daQh07ZqTa++isjj1aPi4zvM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 h1:DRND0dkCKtJzCj4Xl4OpVbXZgfttY5q712H9Zj7qc/0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10/go.mod h1:tGGNmJKOTernmR2+VJ0fCzQRurcPZj9ut60Zu5Fi6us=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.0 h1:vSXYridw+tT3AHuK1PWdJto2qEc30/wG/fm8dmCHHis=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.0/go.mod h1:YXPskkMuiMgp6qUG96NSTl7UpideOQT/Kx0u9Y1MKn0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 h1:fspVFg6qMx0svs40YgRmE7LZXh9VRZvTT35PfdQR6FM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.29.7/go.mod h1:BQTKL3uMECaLaUV3Zc2L4Qybv8C6BIXjuu1dOPyxTQs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 h1:scVnW+NLXasGOhy7HhkdT9AGb6kjgW7fJ5xYkUaqHs0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2/go.mod h1:FRNCY3zTEWZXBKm2h5UBUPvCVDOecTad9KhynDyGBc0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 h1:VEO5dqFkMsl8QZ2yHsFDJAIZLAkEbaYDB+xdKi0Feic=
+github.com/aws/aws-sdk-go-v2/service/sts v1.38.7/go.mod h1:L1xxV3zAdB+qVrVW/pBIrIAnHFWHo6FBbFe4xOGsG/o=
+github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
+github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE=
 github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY=
 github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=

From 44287350343a3c7ff15ffd7482cfb87f9a8eae0c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Oct 2025 00:28:44 +0000
Subject: [PATCH 012/242] build(deps): bump docker/library/golang

Bumps the container-images group with 1 update in the /.buildkite directory: docker/library/golang.


Updates `docker/library/golang` from 1.24.8 to 1.24.9

---
updated-dependencies:
- dependency-name: docker/library/golang
  dependency-version: 1.24.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-compile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index 9ea40a1795..1848e5dc71 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.8@sha256:273d4e65baa782dbe293c9192d600b72b17d415c1429e16bed99efcc5e61efb8
+FROM public.ecr.aws/docker/library/golang:1.24.9@sha256:02ce1d7ea7825dccb7cd10222e44e7c0565a08c5a38795e50fbf43936484507b
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest

From 860b5d9afb111a2b07c7a51471b97fbd5eb225d1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Oct 2025 00:42:47 +0000
Subject: [PATCH 013/242] build(deps): bump github.com/gofrs/flock from 0.12.1
 to 0.13.0

Bumps [github.com/gofrs/flock](https://github.com/gofrs/flock) from 0.12.1 to 0.13.0.
- [Release notes](https://github.com/gofrs/flock/releases)
- [Commits](https://github.com/gofrs/flock/compare/v0.12.1...v0.13.0)

---
updated-dependencies:
- dependency-name: github.com/gofrs/flock
  dependency-version: 0.13.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 47c12ad22d..e3ab670177 100644
--- a/go.mod
+++ b/go.mod
@@ -29,7 +29,7 @@ require (
 	github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2
 	github.com/gliderlabs/ssh v0.3.8
 	github.com/go-chi/chi/v5 v5.2.3
-	github.com/gofrs/flock v0.12.1
+	github.com/gofrs/flock v0.13.0
 	github.com/google/go-cmp v0.7.0
 	github.com/google/go-querystring v1.1.0
 	github.com/google/uuid v1.6.0
diff --git a/go.sum b/go.sum
index 7fd02a04f4..6a12540019 100644
--- a/go.sum
+++ b/go.sum
@@ -189,8 +189,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
-github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
+github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
+github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=

From e90e288be0b2d27fc5d919e0bd78094fd345175b Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 20 Oct 2025 13:25:50 +1100
Subject: [PATCH 014/242] Add --literal flag to artifact upload

---
 clicommand/artifact_upload.go      | 22 ++++++++-----
 internal/artifact/uploader.go      | 36 +++++++++++++++++----
 internal/artifact/uploader_test.go | 52 ++++++++++++++++++++++++++++++
 3 files changed, 95 insertions(+), 15 deletions(-)

diff --git a/clicommand/artifact_upload.go b/clicommand/artifact_upload.go
index 0196358b29..605231872a 100644
--- a/clicommand/artifact_upload.go
+++ b/clicommand/artifact_upload.go
@@ -77,6 +77,7 @@ type ArtifactUploadConfig struct {
 	ContentType string `cli:"content-type"`
 
 	// Uploader flags
+	Literal                   bool `cli:"literal"`
 	GlobResolveFollowSymlinks bool `cli:"glob-resolve-follow-symlinks"`
 	UploadSkipSymlinks        bool `cli:"upload-skip-symlinks"`
 	NoMultipartUpload         bool `cli:"no-multipart-artifact-upload"`
@@ -102,6 +103,11 @@ var ArtifactUploadCommand = cli.Command{
 			Usage:  "A specific Content-Type to set for the artifacts (otherwise detected)",
 			EnvVar: "BUILDKITE_ARTIFACT_CONTENT_TYPE",
 		},
+		cli.BoolFlag{
+			Name:   "literal",
+			Usage:  "Disables parsing of the upload paths as glob patterns; each path will be treated as a single literal file path",
+			EnvVar: "BUILDKITE_AGENT_ARTIFACT_LITERAL",
+		},
 		cli.BoolFlag{
 			Name:   "glob-resolve-follow-symlinks",
 			Usage:  "Follow symbolic links to directories while resolving globs. Note: this will not prevent symlinks to files from being uploaded. Use --upload-skip-symlinks to do that",
@@ -129,15 +135,15 @@ var ArtifactUploadCommand = cli.Command{
 
 		// Setup the uploader
 		uploader := artifact.NewUploader(l, client, artifact.UploaderConfig{
-			JobID:        cfg.Job,
-			Paths:        cfg.UploadPaths,
-			Destination:  cfg.Destination,
-			ContentType:  cfg.ContentType,
-			DebugHTTP:    cfg.DebugHTTP,
-			TraceHTTP:    cfg.TraceHTTP,
-			DisableHTTP2: cfg.NoHTTP2,
-
+			JobID:          cfg.Job,
+			Paths:          cfg.UploadPaths,
+			Destination:    cfg.Destination,
+			ContentType:    cfg.ContentType,
+			DebugHTTP:      cfg.DebugHTTP,
+			TraceHTTP:      cfg.TraceHTTP,
+			DisableHTTP2:   cfg.NoHTTP2,
 			AllowMultipart: !cfg.NoMultipartUpload,
+			Literal:        cfg.Literal,
 
 			// If the deprecated flag was set to true, pretend its replacement was set to true too
 			// this works as long as the user only sets one of the two flags
diff --git a/internal/artifact/uploader.go b/internal/artifact/uploader.go
index 9860210395..706c3ecfdf 100644
--- a/internal/artifact/uploader.go
+++ b/internal/artifact/uploader.go
@@ -49,6 +49,9 @@ type UploaderConfig struct {
 	TraceHTTP    bool
 	DisableHTTP2 bool
 
+	// When true, disables parsing Paths as globs.
+	Literal bool
+
 	// Whether to follow symbolic links when resolving globs
 	GlobResolveFollowSymlinks bool
 
@@ -171,8 +174,14 @@ func (a *Uploader) collect(ctx context.Context) ([]*api.Artifact, error) {
 		}()
 	}
 
-	// Start resolving globs into files.
-	if err := a.glob(wctx, filesCh); err != nil {
+	fileFinder := a.glob
+	if a.conf.Literal {
+		fileFinder = a.literal
+	}
+
+	// Start resolving globs (or not) and sending file paths to workers.
+	a.logger.Debug("Searching for %s", a.conf.Paths)
+	if err := fileFinder(wctx, filesCh); err != nil {
 		cancel(err)
 	}
 
@@ -195,17 +204,30 @@ type artifactCollector struct {
 	artifacts []*api.Artifact
 }
 
+func (a *Uploader) literal(ctx context.Context, filesCh chan<- string) error {
+	// literal is solely responsible for writing to the channel
+	defer close(filesCh)
+
+	for path := range strings.SplitSeq(a.conf.Paths, ArtifactPathDelimiter) {
+		path = strings.TrimSpace(path)
+		if path == "" {
+			continue
+		}
+		filesCh <- path
+	}
+	return nil
+}
+
 // glob resolves the globs (patterns with * and ** in them).
 func (a *Uploader) glob(ctx context.Context, filesCh chan<- string) error {
 	// glob is solely responsible for writing to the channel.
 	defer close(filesCh)
 
-	// New zzglob library. Do all globs at once with MultiGlob, which takes
-	// care of any necessary parallelism under the hood.
-	a.logger.Debug("Searching for %s", a.conf.Paths)
+	// Do all globs at once with MultiGlob, which takes care of any necessary
+	// parallelism under the hood.
 	var patterns []*zzglob.Pattern
-	for _, globPath := range strings.Split(a.conf.Paths, ArtifactPathDelimiter) {
-		globPath := strings.TrimSpace(globPath)
+	for globPath := range strings.SplitSeq(a.conf.Paths, ArtifactPathDelimiter) {
+		globPath = strings.TrimSpace(globPath)
 		if globPath == "" {
 			continue
 		}
diff --git a/internal/artifact/uploader_test.go b/internal/artifact/uploader_test.go
index c507881e74..d19187c235 100644
--- a/internal/artifact/uploader_test.go
+++ b/internal/artifact/uploader_test.go
@@ -1,6 +1,7 @@
 package artifact
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -10,6 +11,7 @@ import (
 	"github.com/buildkite/agent/v3/api"
 	"github.com/buildkite/agent/v3/internal/experiments"
 	"github.com/buildkite/agent/v3/logger"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -314,3 +316,53 @@ func TestCollectMatchesUploadSymlinks(t *testing.T) {
 		paths,
 	)
 }
+
+func TestCollect_Literal(t *testing.T) {
+	t.Parallel()
+	ctx := t.Context()
+
+	uploader := NewUploader(logger.Discard, nil, UploaderConfig{
+		Paths: strings.Join([]string{
+			filepath.Join("fixtures", "links", "folder-link", "terminator2.jpg"),
+			filepath.Join("fixtures", "gifs", "Smile.gif"),
+		}, ";"),
+		Literal: true,
+	})
+
+	artifacts, err := uploader.collect(ctx)
+	if err != nil {
+		t.Fatalf("uploader.Collect() error = %v", err)
+	}
+
+	got := []string{}
+	for _, a := range artifacts {
+		got = append(got, a.Path)
+	}
+	want := []string{
+		filepath.Join("fixtures", "links", "folder-link", "terminator2.jpg"),
+		filepath.Join("fixtures", "gifs", "Smile.gif"),
+	}
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Errorf("uploader.collect artifact paths diff (-got +want)\n%s", diff)
+	}
+}
+
+func TestCollect_LiteralPathNotFound(t *testing.T) {
+	t.Parallel()
+	ctx := t.Context()
+
+	uploader := NewUploader(logger.Discard, nil, UploaderConfig{
+		// When parsed as a glob, it finds multiple files.
+		// When used literally, it finds nothing.
+		Paths:   filepath.Join("fixtures", "**", "*.jpg"),
+		Literal: true,
+	})
+
+	var pathErr *os.PathError
+	if _, err := uploader.collect(ctx); !errors.As(err, &pathErr) {
+		t.Fatalf("uploader.collect() error = %v, want %T", err, pathErr)
+	}
+	if pathErr.Op != "open" {
+		t.Errorf("uploader.collect() error Op = %q, want open", pathErr.Op)
+	}
+}

From f7406675da32f3d1fc57152807e01a70ad5b9f20 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 20 Oct 2025 13:51:56 +1100
Subject: [PATCH 015/242] Add --delimiter flag to artifact upload

---
 clicommand/artifact_upload.go      | 16 ++++++++++++----
 internal/artifact/uploader.go      | 29 +++++++++++++++++++----------
 internal/artifact/uploader_test.go | 10 +++++++++-
 3 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/clicommand/artifact_upload.go b/clicommand/artifact_upload.go
index 605231872a..0403ff7609 100644
--- a/clicommand/artifact_upload.go
+++ b/clicommand/artifact_upload.go
@@ -77,10 +77,11 @@ type ArtifactUploadConfig struct {
 	ContentType string `cli:"content-type"`
 
 	// Uploader flags
-	Literal                   bool `cli:"literal"`
-	GlobResolveFollowSymlinks bool `cli:"glob-resolve-follow-symlinks"`
-	UploadSkipSymlinks        bool `cli:"upload-skip-symlinks"`
-	NoMultipartUpload         bool `cli:"no-multipart-artifact-upload"`
+	Literal                   bool   `cli:"literal"`
+	Delimiter                 string `cli:"delimiter"`
+	GlobResolveFollowSymlinks bool   `cli:"glob-resolve-follow-symlinks"`
+	UploadSkipSymlinks        bool   `cli:"upload-skip-symlinks"`
+	NoMultipartUpload         bool   `cli:"no-multipart-artifact-upload"`
 
 	// deprecated
 	FollowSymlinks bool `cli:"follow-symlinks" deprecated-and-renamed-to:"GlobResolveFollowSymlinks"`
@@ -108,6 +109,12 @@ var ArtifactUploadCommand = cli.Command{
 			Usage:  "Disables parsing of the upload paths as glob patterns; each path will be treated as a single literal file path",
 			EnvVar: "BUILDKITE_AGENT_ARTIFACT_LITERAL",
 		},
+		cli.StringFlag{
+			Name:   "delimiter",
+			Usage:  "Changes the delimiter used to split the upload paths into multiple paths; it can be more than 1 character. When set to the empty string, no splitting occurs",
+			EnvVar: "BUILDKITE_AGENT_ARTIFACT_DELIMITER",
+			Value:  ";",
+		},
 		cli.BoolFlag{
 			Name:   "glob-resolve-follow-symlinks",
 			Usage:  "Follow symbolic links to directories while resolving globs. Note: this will not prevent symlinks to files from being uploaded. Use --upload-skip-symlinks to do that",
@@ -144,6 +151,7 @@ var ArtifactUploadCommand = cli.Command{
 			DisableHTTP2:   cfg.NoHTTP2,
 			AllowMultipart: !cfg.NoMultipartUpload,
 			Literal:        cfg.Literal,
+			Delimiter:      cfg.Delimiter,
 
 			// If the deprecated flag was set to true, pretend its replacement was set to true too
 			// this works as long as the user only sets one of the two flags
diff --git a/internal/artifact/uploader.go b/internal/artifact/uploader.go
index 706c3ecfdf..d54e95afea 100644
--- a/internal/artifact/uploader.go
+++ b/internal/artifact/uploader.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"iter"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -26,10 +27,7 @@ import (
 	"github.com/dustin/go-humanize"
 )
 
-const (
-	ArtifactPathDelimiter    = ";"
-	ArtifactFallbackMimeType = "binary/octet-stream"
-)
+const ArtifactFallbackMimeType = "binary/octet-stream"
 
 type UploaderConfig struct {
 	// The ID of the Job
@@ -49,9 +47,12 @@ type UploaderConfig struct {
 	TraceHTTP    bool
 	DisableHTTP2 bool
 
-	// When true, disables parsing Paths as globs.
+	// When true, disables parsing Paths as globs; treat each path literally.
 	Literal bool
 
+	// The delimiter used to split Paths into multiple paths/globs.
+	Delimiter string
+
 	// Whether to follow symbolic links when resolving globs
 	GlobResolveFollowSymlinks bool
 
@@ -181,7 +182,7 @@ func (a *Uploader) collect(ctx context.Context) ([]*api.Artifact, error) {
 
 	// Start resolving globs (or not) and sending file paths to workers.
 	a.logger.Debug("Searching for %s", a.conf.Paths)
-	if err := fileFinder(wctx, filesCh); err != nil {
+	if err := fileFinder(wctx, a.paths(), filesCh); err != nil {
 		cancel(err)
 	}
 
@@ -204,11 +205,19 @@ type artifactCollector struct {
 	artifacts []*api.Artifact
 }
 
-func (a *Uploader) literal(ctx context.Context, filesCh chan<- string) error {
+func (a *Uploader) paths() iter.Seq[string] {
+	if a.conf.Delimiter == "" {
+		// Don't do any splitting.
+		return slices.Values([]string{a.conf.Paths})
+	}
+	return strings.SplitSeq(a.conf.Paths, a.conf.Delimiter)
+}
+
+func (a *Uploader) literal(ctx context.Context, paths iter.Seq[string], filesCh chan<- string) error {
 	// literal is solely responsible for writing to the channel
 	defer close(filesCh)
 
-	for path := range strings.SplitSeq(a.conf.Paths, ArtifactPathDelimiter) {
+	for path := range paths {
 		path = strings.TrimSpace(path)
 		if path == "" {
 			continue
@@ -219,14 +228,14 @@ func (a *Uploader) literal(ctx context.Context, filesCh chan<- string) error {
 }
 
 // glob resolves the globs (patterns with * and ** in them).
-func (a *Uploader) glob(ctx context.Context, filesCh chan<- string) error {
+func (a *Uploader) glob(ctx context.Context, paths iter.Seq[string], filesCh chan<- string) error {
 	// glob is solely responsible for writing to the channel.
 	defer close(filesCh)
 
 	// Do all globs at once with MultiGlob, which takes care of any necessary
 	// parallelism under the hood.
 	var patterns []*zzglob.Pattern
-	for globPath := range strings.SplitSeq(a.conf.Paths, ArtifactPathDelimiter) {
+	for globPath := range paths {
 		globPath = strings.TrimSpace(globPath)
 		if globPath == "" {
 			continue
diff --git a/internal/artifact/uploader_test.go b/internal/artifact/uploader_test.go
index d19187c235..e59e7de5f2 100644
--- a/internal/artifact/uploader_test.go
+++ b/internal/artifact/uploader_test.go
@@ -81,6 +81,7 @@ func TestCollect(t *testing.T) {
 			filepath.Join("fixtures", "**/*.jpg"),
 			filepath.Join(root, "fixtures", "**/*.gif"),
 		),
+		Delimiter: ";",
 	})
 
 	// For the normalised-upload-paths experiment, uploaded artifact paths are
@@ -164,6 +165,7 @@ func TestCollectThatDoesntMatchAnyFiles(t *testing.T) {
 			filepath.Join("mkmf.log"),
 			filepath.Join("log", "mkmf.log"),
 		}, ";"),
+		Delimiter: ";",
 	})
 
 	artifacts, err := uploader.collect(ctx)
@@ -184,6 +186,7 @@ func TestCollectWithSomeGlobsThatDontMatchAnything(t *testing.T) {
 			filepath.Join("dontmatchanything.zip"),
 			filepath.Join("fixtures", "**", "*.jpg"),
 		}, ";"),
+		Delimiter: ";",
 	})
 
 	artifacts, err := uploader.collect(ctx)
@@ -207,6 +210,7 @@ func TestCollectWithSomeGlobsThatDontMatchAnythingFollowingSymlinks(t *testing.T
 			filepath.Join("fixtures", "links", "folder-link", "dontmatchanything", "**", "*.jpg"),
 			filepath.Join("fixtures", "**", "*.jpg"),
 		}, ";"),
+		Delimiter:                 ";",
 		GlobResolveFollowSymlinks: true,
 	})
 
@@ -229,6 +233,7 @@ func TestCollectWithDuplicateMatches(t *testing.T) {
 			filepath.Join("fixtures", "**", "*.jpg"),
 			filepath.Join("fixtures", "folder", "Commando.jpg"), // dupe
 		}, ";"),
+		Delimiter: ";",
 	})
 
 	artifacts, err := uploader.collect(ctx)
@@ -261,6 +266,7 @@ func TestCollectWithDuplicateMatchesFollowingSymlinks(t *testing.T) {
 			filepath.Join("fixtures", "**", "*.jpg"),
 			filepath.Join("fixtures", "folder", "Commando.jpg"), // dupe
 		}, ";"),
+		Delimiter:                 ";",
 		GlobResolveFollowSymlinks: true,
 	})
 
@@ -294,6 +300,7 @@ func TestCollectMatchesUploadSymlinks(t *testing.T) {
 		Paths: strings.Join([]string{
 			filepath.Join("fixtures", "**", "*.jpg"),
 		}, ";"),
+		Delimiter:          ";",
 		UploadSkipSymlinks: true,
 	})
 
@@ -326,7 +333,8 @@ func TestCollect_Literal(t *testing.T) {
 			filepath.Join("fixtures", "links", "folder-link", "terminator2.jpg"),
 			filepath.Join("fixtures", "gifs", "Smile.gif"),
 		}, ";"),
-		Literal: true,
+		Delimiter: ";",
+		Literal:   true,
 	})
 
 	artifacts, err := uploader.collect(ctx)

From bcec64813f3f9d9f817330b564065f4fa8b344f5 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 16 Oct 2025 12:24:05 +1100
Subject: [PATCH 016/242] Review of signal handling and grace periods

---
 agent/agent_configuration.go       |  57 ++++---
 agent/agent_worker.go              |   2 +-
 agent/run_job.go                   |  66 ++------
 clicommand/agent_start.go          | 246 +++++++++++++++--------------
 clicommand/global.go               |   6 +-
 clicommand/kubernetes_bootstrap.go |  44 ++++--
 6 files changed, 202 insertions(+), 219 deletions(-)

diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go
index 36227db1d3..1556e94c82 100644
--- a/agent/agent_configuration.go
+++ b/agent/agent_configuration.go
@@ -8,35 +8,34 @@ import (
 // AgentConfiguration is the run-time configuration for an agent that
 // has been loaded from the config file and command-line params
 type AgentConfiguration struct {
-	ConfigPath                         string
-	BootstrapScript                    string
-	BuildPath                          string
-	HooksPath                          string
-	AdditionalHooksPaths               []string
-	SocketsPath                        string
-	GitMirrorsPath                     string
-	GitMirrorsLockTimeout              int
-	GitMirrorsSkipUpdate               bool
-	PluginsPath                        string
-	GitCheckoutFlags                   string
-	GitCloneFlags                      string
-	GitCloneMirrorFlags                string
-	GitCleanFlags                      string
-	GitFetchFlags                      string
-	GitSubmodules                      bool
-	AllowedRepositories                []*regexp.Regexp
-	AllowedPlugins                     []*regexp.Regexp
-	AllowedEnvironmentVariables        []*regexp.Regexp
-	SSHKeyscan                         bool
-	CommandEval                        bool
-	PluginsEnabled                     bool
-	PluginValidation                   bool
-	PluginsAlwaysCloneFresh            bool
-	LocalHooksEnabled                  bool
-	StrictSingleHooks                  bool
-	RunInPty                           bool
-	KubernetesExec                     bool
-	KubernetesLogCollectionGracePeriod time.Duration
+	ConfigPath                  string
+	BootstrapScript             string
+	BuildPath                   string
+	HooksPath                   string
+	AdditionalHooksPaths        []string
+	SocketsPath                 string
+	GitMirrorsPath              string
+	GitMirrorsLockTimeout       int
+	GitMirrorsSkipUpdate        bool
+	PluginsPath                 string
+	GitCheckoutFlags            string
+	GitCloneFlags               string
+	GitCloneMirrorFlags         string
+	GitCleanFlags               string
+	GitFetchFlags               string
+	GitSubmodules               bool
+	AllowedRepositories         []*regexp.Regexp
+	AllowedPlugins              []*regexp.Regexp
+	AllowedEnvironmentVariables []*regexp.Regexp
+	SSHKeyscan                  bool
+	CommandEval                 bool
+	PluginsEnabled              bool
+	PluginValidation            bool
+	PluginsAlwaysCloneFresh     bool
+	LocalHooksEnabled           bool
+	StrictSingleHooks           bool
+	RunInPty                    bool
+	KubernetesExec              bool
 
 	SigningJWKSFile  string // Where to find the key to sign pipeline uploads with (passed through to jobs, they might be uploading pipelines)
 	SigningJWKSKeyID string // The key ID to sign pipeline uploads with
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 1087a08325..7db8a28265 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -518,7 +518,7 @@ func (a *AgentWorker) Stop(graceful bool) {
 				a.logger.Info("Gracefully stopping agent. Since there is no job running, the agent will disconnect immediately")
 			}
 		}
-	} else {
+	} else { // ungraceful
 		// If there's a job running, kill it, then disconnect
 		if a.jobRunner != nil {
 			a.logger.Info("Forcefully stopping agent. The current job will be canceled before disconnecting...")
diff --git a/agent/run_job.go b/agent/run_job.go
index 43d493c058..5cc83a2177 100644
--- a/agent/run_job.go
+++ b/agent/run_job.go
@@ -373,11 +373,6 @@ One or more containers connected to the agent, but then stopped communicating wi
 func (r *JobRunner) cleanup(ctx context.Context, wg *sync.WaitGroup, exit core.ProcessExit, ignoreAgentInDispatches *bool) {
 	finishedAt := time.Now()
 
-	// In Kubernetes mode, wait for command containers to finish before stopping log collection
-	if r.conf.KubernetesExec {
-		r.waitForKubernetesProcessesToComplete(ctx)
-	}
-
 	// Flush the job logs. If the process is never started, then logs from prior to the attempt to
 	// start the process will still be buffered. Also, there may still be logs in the buffer that
 	// were left behind because the uploader goroutine exited before it could flush them.
@@ -432,50 +427,6 @@ func (r *JobRunner) cleanup(ctx context.Context, wg *sync.WaitGroup, exit core.P
 	r.agentLogger.Info("Finished job %s for build at %s", r.conf.Job.ID, r.conf.Job.Env["BUILDKITE_BUILD_URL"])
 }
 
-// waitForKubernetesProcessesToComplete waits for Kubernetes command containers to finish
-// before stopping log collection, ensuring post-command hook logs are captured.
-func (r *JobRunner) waitForKubernetesProcessesToComplete(ctx context.Context) {
-	r.agentLogger.Debug("[JobRunner] Waiting for Kubernetes processes to complete before stopping log collection")
-
-	// Guard against nil process
-	if r.process == nil {
-		r.agentLogger.Debug("[JobRunner] No process to wait for")
-		return
-	}
-
-	// Wait for the process to actually start before waiting for it to complete
-	select {
-	case <-r.process.Started():
-		// Process has started, we can now safely wait for completion
-	case <-ctx.Done():
-		r.agentLogger.Debug("[JobRunner] Context cancelled before process started, skipping wait")
-		return
-	}
-
-	gracePeriod := r.conf.AgentConfiguration.KubernetesLogCollectionGracePeriod
-
-	waitCtx := ctx
-	if gracePeriod >= 0 {
-		r.agentLogger.Debug("[JobRunner] Using log collection grace period: %v", gracePeriod)
-		var cancel context.CancelFunc
-		waitCtx, cancel = context.WithTimeout(ctx, gracePeriod)
-		defer cancel()
-	} else {
-		r.agentLogger.Debug("[JobRunner] No log collection grace period configured, waiting until process completes")
-	}
-
-	select {
-	case <-r.process.Done():
-		r.agentLogger.Debug("[JobRunner] Kubernetes processes completed, stopping log collection")
-	case <-waitCtx.Done():
-		if ctx.Err() != nil {
-			r.agentLogger.Info("[JobRunner] Parent context cancelled while waiting for Kubernetes processes")
-		} else {
-			r.agentLogger.Info("[JobRunner] Timeout waiting for Kubernetes processes, stopping log collection")
-		}
-	}
-}
-
 // streamJobLogsAfterProcessStart waits for the process to start, then grabs the job output
 // every few seconds and sends it back to Buildkite.
 func (r *JobRunner) streamJobLogsAfterProcessStart(ctx context.Context, wg *sync.WaitGroup) {
@@ -575,9 +526,9 @@ func (r *JobRunner) Cancel() error {
 	}
 
 	r.agentLogger.Info(
-		"Canceling job %s with a grace period of %ds %s",
+		"Canceling job %s with a signal grace period of %v %s",
 		r.conf.Job.ID,
-		r.conf.AgentConfiguration.CancelGracePeriod,
+		r.conf.AgentConfiguration.SignalGracePeriod,
 		reason,
 	)
 
@@ -589,9 +540,16 @@ func (r *JobRunner) Cancel() error {
 	}
 
 	select {
-	// Grace period for cancelling
-	case <-time.After(time.Second * time.Duration(r.conf.AgentConfiguration.CancelGracePeriod)):
-		r.agentLogger.Info("Job %s hasn't stopped in time, terminating", r.conf.Job.ID)
+	// Grace period between Interrupt and Terminate = the signal grace period.
+	// Extra time between the end of the signal grace period and the end of the
+	// cancel grace period is the time we (agent side) needs to upload logs and
+	// disconnect (if the agent is exiting).
+	case <-time.After(r.conf.AgentConfiguration.SignalGracePeriod):
+		r.agentLogger.Info(
+			"Job %s hasn't stopped within %v, terminating",
+			r.conf.Job.ID,
+			r.conf.AgentConfiguration.SignalGracePeriod,
+		)
 
 		// Terminate the process as we've exceeded our context
 		return r.process.Terminate()
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index c70416e86b..76bae731b4 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -29,7 +29,6 @@ import (
 	"github.com/buildkite/agent/v3/internal/awslib"
 	awssigner "github.com/buildkite/agent/v3/internal/cryptosigner/aws"
 	"github.com/buildkite/agent/v3/internal/experiments"
-	"github.com/buildkite/agent/v3/internal/job"
 	"github.com/buildkite/agent/v3/internal/job/hook"
 	"github.com/buildkite/agent/v3/internal/osutil"
 	"github.com/buildkite/agent/v3/internal/shell"
@@ -62,8 +61,6 @@ Example:
     $ buildkite-agent start --token xxx`
 
 var (
-	minGracePeriod = 10
-
 	verificationFailureBehaviors = []string{agent.VerificationBehaviourBlock, agent.VerificationBehaviourWarn}
 
 	buildkiteSetEnvironmentVariables = []*regexp.Regexp{
@@ -180,11 +177,10 @@ type AgentStartConfig struct {
 	TracingPropagateTraceparent bool   `cli:"tracing-propagate-traceparent"`
 
 	// Other shared flags
-	StrictSingleHooks                  bool          `cli:"strict-single-hooks"`
-	KubernetesExec                     bool          `cli:"kubernetes-exec"`
-	KubernetesLogCollectionGracePeriod time.Duration `cli:"kubernetes-log-collection-grace-period"`
-	TraceContextEncoding               string        `cli:"trace-context-encoding"`
-	NoMultipartArtifactUpload          bool          `cli:"no-multipart-artifact-upload"`
+	StrictSingleHooks         bool   `cli:"strict-single-hooks"`
+	KubernetesExec            bool   `cli:"kubernetes-exec"`
+	TraceContextEncoding      string `cli:"trace-context-encoding"`
+	NoMultipartArtifactUpload bool   `cli:"no-multipart-artifact-upload"`
 
 	// API config
 	DebugHTTP bool   `cli:"debug-http"`
@@ -194,14 +190,15 @@ type AgentStartConfig struct {
 	NoHTTP2   bool   `cli:"no-http2"`
 
 	// Deprecated
-	NoSSHFingerprintVerification bool     `cli:"no-automatic-ssh-fingerprint-verification" deprecated-and-renamed-to:"NoSSHKeyscan"`
-	MetaData                     []string `cli:"meta-data" deprecated-and-renamed-to:"Tags"`
-	MetaDataEC2                  bool     `cli:"meta-data-ec2" deprecated-and-renamed-to:"TagsFromEC2"`
-	MetaDataEC2Tags              bool     `cli:"meta-data-ec2-tags" deprecated-and-renamed-to:"TagsFromEC2Tags"`
-	MetaDataGCP                  bool     `cli:"meta-data-gcp" deprecated-and-renamed-to:"TagsFromGCP"`
-	TagsFromEC2                  bool     `cli:"tags-from-ec2" deprecated-and-renamed-to:"TagsFromEC2MetaData"`
-	TagsFromGCP                  bool     `cli:"tags-from-gcp" deprecated-and-renamed-to:"TagsFromGCPMetaData"`
-	DisconnectAfterJobTimeout    int      `cli:"disconnect-after-job-timeout" deprecated:"Use disconnect-after-idle-timeout instead"`
+	KubernetesLogCollectionGracePeriod time.Duration `cli:"kubernetes-log-collection-grace-period"`
+	NoSSHFingerprintVerification       bool          `cli:"no-automatic-ssh-fingerprint-verification" deprecated-and-renamed-to:"NoSSHKeyscan"`
+	MetaData                           []string      `cli:"meta-data" deprecated-and-renamed-to:"Tags"`
+	MetaDataEC2                        bool          `cli:"meta-data-ec2" deprecated-and-renamed-to:"TagsFromEC2"`
+	MetaDataEC2Tags                    bool          `cli:"meta-data-ec2-tags" deprecated-and-renamed-to:"TagsFromEC2Tags"`
+	MetaDataGCP                        bool          `cli:"meta-data-gcp" deprecated-and-renamed-to:"TagsFromGCP"`
+	TagsFromEC2                        bool          `cli:"tags-from-ec2" deprecated-and-renamed-to:"TagsFromEC2MetaData"`
+	TagsFromGCP                        bool          `cli:"tags-from-gcp" deprecated-and-renamed-to:"TagsFromGCPMetaData"`
+	DisconnectAfterJobTimeout          int           `cli:"disconnect-after-job-timeout" deprecated:"Use disconnect-after-idle-timeout instead"`
 }
 
 func (asc AgentStartConfig) Features(ctx context.Context) []string {
@@ -754,11 +751,11 @@ var AgentStartCommand = cli.Command{
 		RedactedVars,
 		StrictSingleHooksFlag,
 		KubernetesExecFlag,
-		KubernetesLogCollectionGracePeriodFlag,
 		TraceContextEncodingFlag,
 		NoMultipartArtifactUploadFlag,
 
 		// Deprecated flags which will be removed in v4
+		KubernetesLogCollectionGracePeriodFlag,
 		cli.StringSliceFlag{
 			Name:   "meta-data",
 			Value:  &cli.StringSlice{},
@@ -921,8 +918,6 @@ var AgentStartCommand = cli.Command{
 			return err
 		}
 
-		kubernetesLogCollectionGracePeriod := cfg.KubernetesLogCollectionGracePeriod
-
 		if _, err := tracetools.ParseEncoding(cfg.TraceContextEncoding); err != nil {
 			return fmt.Errorf("while parsing trace context encoding: %v", err)
 		}
@@ -1015,51 +1010,50 @@ var AgentStartCommand = cli.Command{
 
 		// AgentConfiguration is the runtime configuration for an agent
 		agentConf := agent.AgentConfiguration{
-			BootstrapScript:                    cfg.BootstrapScript,
-			BuildPath:                          cfg.BuildPath,
-			SocketsPath:                        cfg.SocketsPath,
-			GitMirrorsPath:                     cfg.GitMirrorsPath,
-			GitMirrorsLockTimeout:              cfg.GitMirrorsLockTimeout,
-			GitMirrorsSkipUpdate:               cfg.GitMirrorsSkipUpdate,
-			HooksPath:                          cfg.HooksPath,
-			AdditionalHooksPaths:               cfg.AdditionalHooksPaths,
-			PluginsPath:                        cfg.PluginsPath,
-			GitCheckoutFlags:                   cfg.GitCheckoutFlags,
-			GitCloneFlags:                      cfg.GitCloneFlags,
-			GitCloneMirrorFlags:                cfg.GitCloneMirrorFlags,
-			GitCleanFlags:                      cfg.GitCleanFlags,
-			GitFetchFlags:                      cfg.GitFetchFlags,
-			GitSubmodules:                      !cfg.NoGitSubmodules,
-			SSHKeyscan:                         !cfg.NoSSHKeyscan,
-			CommandEval:                        !cfg.NoCommandEval,
-			PluginsEnabled:                     !cfg.NoPlugins,
-			PluginValidation:                   !cfg.NoPluginValidation,
-			PluginsAlwaysCloneFresh:            cfg.PluginsAlwaysCloneFresh,
-			LocalHooksEnabled:                  !cfg.NoLocalHooks,
-			AllowedEnvironmentVariables:        allowedEnvironmentVariables,
-			StrictSingleHooks:                  cfg.StrictSingleHooks,
-			RunInPty:                           !cfg.NoPTY,
-			ANSITimestamps:                     !cfg.NoANSITimestamps,
-			TimestampLines:                     cfg.TimestampLines,
-			DisconnectAfterJob:                 cfg.DisconnectAfterJob,
-			DisconnectAfterIdleTimeout:         cfg.DisconnectAfterIdleTimeout,
-			DisconnectAfterUptime:              cfg.DisconnectAfterUptime,
-			CancelGracePeriod:                  cfg.CancelGracePeriod,
-			SignalGracePeriod:                  signalGracePeriod,
-			EnableJobLogTmpfile:                cfg.EnableJobLogTmpfile,
-			JobLogPath:                         cfg.JobLogPath,
-			WriteJobLogsToStdout:               cfg.WriteJobLogsToStdout,
-			LogFormat:                          cfg.LogFormat,
-			Shell:                              cfg.Shell,
-			RedactedVars:                       cfg.RedactedVars,
-			AcquireJob:                         cfg.AcquireJob,
-			TracingBackend:                     cfg.TracingBackend,
-			TracingServiceName:                 cfg.TracingServiceName,
-			TracingPropagateTraceparent:        cfg.TracingPropagateTraceparent,
-			TraceContextEncoding:               cfg.TraceContextEncoding,
-			AllowMultipartArtifactUpload:       !cfg.NoMultipartArtifactUpload,
-			KubernetesExec:                     cfg.KubernetesExec,
-			KubernetesLogCollectionGracePeriod: kubernetesLogCollectionGracePeriod,
+			BootstrapScript:              cfg.BootstrapScript,
+			BuildPath:                    cfg.BuildPath,
+			SocketsPath:                  cfg.SocketsPath,
+			GitMirrorsPath:               cfg.GitMirrorsPath,
+			GitMirrorsLockTimeout:        cfg.GitMirrorsLockTimeout,
+			GitMirrorsSkipUpdate:         cfg.GitMirrorsSkipUpdate,
+			HooksPath:                    cfg.HooksPath,
+			AdditionalHooksPaths:         cfg.AdditionalHooksPaths,
+			PluginsPath:                  cfg.PluginsPath,
+			GitCheckoutFlags:             cfg.GitCheckoutFlags,
+			GitCloneFlags:                cfg.GitCloneFlags,
+			GitCloneMirrorFlags:          cfg.GitCloneMirrorFlags,
+			GitCleanFlags:                cfg.GitCleanFlags,
+			GitFetchFlags:                cfg.GitFetchFlags,
+			GitSubmodules:                !cfg.NoGitSubmodules,
+			SSHKeyscan:                   !cfg.NoSSHKeyscan,
+			CommandEval:                  !cfg.NoCommandEval,
+			PluginsEnabled:               !cfg.NoPlugins,
+			PluginValidation:             !cfg.NoPluginValidation,
+			PluginsAlwaysCloneFresh:      cfg.PluginsAlwaysCloneFresh,
+			LocalHooksEnabled:            !cfg.NoLocalHooks,
+			AllowedEnvironmentVariables:  allowedEnvironmentVariables,
+			StrictSingleHooks:            cfg.StrictSingleHooks,
+			RunInPty:                     !cfg.NoPTY,
+			ANSITimestamps:               !cfg.NoANSITimestamps,
+			TimestampLines:               cfg.TimestampLines,
+			DisconnectAfterJob:           cfg.DisconnectAfterJob,
+			DisconnectAfterIdleTimeout:   cfg.DisconnectAfterIdleTimeout,
+			DisconnectAfterUptime:        cfg.DisconnectAfterUptime,
+			CancelGracePeriod:            cfg.CancelGracePeriod,
+			SignalGracePeriod:            signalGracePeriod,
+			EnableJobLogTmpfile:          cfg.EnableJobLogTmpfile,
+			JobLogPath:                   cfg.JobLogPath,
+			WriteJobLogsToStdout:         cfg.WriteJobLogsToStdout,
+			LogFormat:                    cfg.LogFormat,
+			Shell:                        cfg.Shell,
+			RedactedVars:                 cfg.RedactedVars,
+			AcquireJob:                   cfg.AcquireJob,
+			TracingBackend:               cfg.TracingBackend,
+			TracingServiceName:           cfg.TracingServiceName,
+			TracingPropagateTraceparent:  cfg.TracingPropagateTraceparent,
+			TraceContextEncoding:         cfg.TraceContextEncoding,
+			AllowMultipartArtifactUpload: !cfg.NoMultipartArtifactUpload,
+			KubernetesExec:               cfg.KubernetesExec,
 
 			SigningJWKSFile:  cfg.SigningJWKSFile,
 			SigningJWKSKeyID: cfg.SigningJWKSKeyID,
@@ -1300,7 +1294,15 @@ var AgentStartCommand = cli.Command{
 		}
 
 		// Handle process signals
-		signals := handlePoolSignals(ctx, l, pool, cancel, cfg.CancelGracePeriod)
+		poolSigs := &poolSignals{
+			log:               l,
+			pool:              pool,
+			cancelGracePeriod: time.Duration(cfg.CancelGracePeriod) * time.Second,
+			// Under Kubernetes, there is no user interactively signalling us,
+			// so on SIGTERM, stop un-gracefully.
+			skipGraceful: cfg.KubernetesExec,
+		}
+		signals := poolSigs.handle(ctx)
 		defer close(signals)
 
 		l.Info("Starting %d Agent(s)", cfg.Spawn)
@@ -1374,68 +1376,76 @@ func parseAndValidateJWKS(ctx context.Context, keysetType, path string) (jwk.Set
 	return jwks, nil
 }
 
-func handlePoolSignals(ctx context.Context, l logger.Logger, pool *agent.AgentPool, cancel context.CancelFunc, cancelGracePeriod int) chan os.Signal {
+type poolSignals struct {
+	log               logger.Logger
+	pool              *agent.AgentPool
+	cancelGracePeriod time.Duration
+	skipGraceful      bool
+}
+
+func (ps *poolSignals) handle(ctx context.Context) chan os.Signal {
 	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt,
+	signal.Notify(
+		signals,
+		os.Interrupt,
 		syscall.SIGHUP,
 		syscall.SIGTERM,
 		syscall.SIGINT,
-		syscall.SIGQUIT)
+		syscall.SIGQUIT,
+	)
 
-	go func() {
-		_, setStatus, done := status.AddSimpleItem(ctx, "Handle Pool Signals")
-		defer done()
-		setStatus("⏳ Waiting for a signal")
-
-		var interruptCount int
-
-		for sig := range signals {
-			l.Debug("Received signal `%v`", sig)
-			setStatus(fmt.Sprintf("Received signal `%v`", sig))
-
-			switch sig {
-			case syscall.SIGQUIT:
-				l.Debug("Received signal `%s`", sig.String())
-				pool.Stop(false)
-			case syscall.SIGTERM, syscall.SIGINT:
-				l.Debug("Received signal `%s`", sig.String())
-				if interruptCount == 0 {
-					interruptCount++
-					l.Info("Received CTRL-C, send again to forcefully kill the agent(s)")
-					pool.Stop(true)
-				} else {
-					l.Info("Forcefully stopping running jobs and stopping the agent(s) in %d seconds", cancelGracePeriod)
-
-					gracefulContext, _ := job.WithGracePeriod(ctx, time.Duration(max(cancelGracePeriod, minGracePeriod))*time.Second)
-
-					go func() {
-						l.Info("Forced agent(s) to stop")
-						pool.Stop(false) // one last chance to stop
-
-						// Wait half the grace period before cancelling the context
-						time.Sleep(time.Duration(max(cancelGracePeriod/2, minGracePeriod/2)) * time.Second)
-
-						l.Info("Cancelling all internal tasks and API requests")
-						cancel() // cancel the context to stop all network operations
-					}()
-
-					// Once pending retries and requests are cancelled,
-					// the main goroutine should exit before this grace period expires, ending the program.
-					// If that doesn't happen, exit 1 below.
-					<-gracefulContext.Done()
-					l.Info("exiting with status 1")
-
-					// this should only be called if the context cancellation and forceful stop fails
-					os.Exit(1)
+	go ps.handleLoop(ctx, signals)
+	return signals
+}
+
+func (ps *poolSignals) handleLoop(ctx context.Context, signals chan os.Signal) {
+	_, setStatus, done := status.AddSimpleItem(ctx, "Handle Pool Signals")
+	defer done()
+	setStatus("⏳ Waiting for a signal")
+
+	interruptCount := 0
+	if ps.skipGraceful {
+		interruptCount = 1
+	}
 
+	for sig := range signals {
+		ps.log.Debug("Received signal `%v`", sig)
+		setStatus(fmt.Sprintf("Received signal `%v`", sig))
+
+		switch sig {
+		case syscall.SIGQUIT:
+			ps.pool.Stop(false)
+
+		case syscall.SIGTERM, syscall.SIGINT:
+			interruptCount++
+			switch interruptCount {
+			case 1:
+				ps.log.Info("Received CTRL-C, send again to forcefully kill the agent(s)")
+				ps.pool.Stop(true)
+
+			case 2:
+				ps.log.Info("Forcefully stopping running jobs and stopping the agent(s) in %d seconds", ps.cancelGracePeriod)
+				if !ps.skipGraceful {
+					ps.log.Info("Press Ctrl-C one more time to exit immediately without disconnecting")
 				}
-			default:
-				l.Debug("Ignoring signal `%s`", sig.String())
+				ps.pool.Stop(false) // one last chance to stop
+
+				go func() {
+					time.Sleep(ps.cancelGracePeriod)
+					// We get here if the main goroutine hasn't returned yet.
+					ps.log.Info("Timed out waiting for agents to exit; exiting immediately with status 1")
+					os.Exit(1)
+				}()
+
+			case 3:
+				ps.log.Info("Exiting immediately with status 1")
+				os.Exit(1)
 			}
-		}
-	}()
 
-	return signals
+		default:
+			ps.log.Debug("Ignoring signal `%s`", sig.String())
+		}
+	}
 }
 
 func agentStartupHook(log logger.Logger, cfg AgentStartConfig) error {
diff --git a/clicommand/global.go b/clicommand/global.go
index 679dfc1710..cfe8869091 100644
--- a/clicommand/global.go
+++ b/clicommand/global.go
@@ -117,10 +117,8 @@ var (
 	}
 
 	KubernetesLogCollectionGracePeriodFlag = cli.DurationFlag{
-		Name: "kubernetes-log-collection-grace-period",
-		Usage: "How long to wait for Kubernetes processes to complete before stopping log " +
-			"collection during graceful termination. This should be less than the pod's " +
-			"terminationGracePeriodSeconds to allow time for final log upload",
+		Name:   "kubernetes-log-collection-grace-period",
+		Usage:  "Deprecated, do not use",
 		EnvVar: "BUILDKITE_KUBERNETES_LOG_COLLECTION_GRACE_PERIOD",
 		Value:  50 * time.Second,
 	}
diff --git a/clicommand/kubernetes_bootstrap.go b/clicommand/kubernetes_bootstrap.go
index 44c5dbf769..0f013fe18f 100644
--- a/clicommand/kubernetes_bootstrap.go
+++ b/clicommand/kubernetes_bootstrap.go
@@ -121,9 +121,10 @@ var KubernetesBootstrapCommand = cli.Command{
 			}
 			cancelSignal = cs
 		}
-		cgp := environ.GetInt("BUILDKITE_CANCEL_GRACE_PERIOD", defaultCancelGracePeriodSecs)
-		sgp := environ.GetInt("BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS", defaultSignalGracePeriodSecs)
-		signalGracePeriod, err := signalGracePeriod(cgp, sgp)
+		cancelGracePeriodSecs := environ.GetInt("BUILDKITE_CANCEL_GRACE_PERIOD", defaultCancelGracePeriodSecs)
+		cancelGracePeriod := time.Duration(cancelGracePeriodSecs) * time.Second
+		signalGracePeriodSecs := environ.GetInt("BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS", defaultSignalGracePeriodSecs)
+		signalGracePeriod, err := signalGracePeriod(cancelGracePeriodSecs, signalGracePeriodSecs)
 		if err != nil {
 			return err
 		}
@@ -162,9 +163,23 @@ var KubernetesBootstrapCommand = cli.Command{
 			// is in state interrupted or the connection died or ...), we should
 			// cancel the job.
 			if err != nil {
-				l.Error("Error waiting for client interrupt: %v", err)
+				l.Error("kubernetes-bootstrap: Error waiting for client interrupt: %v; cancelling work", err)
+			} else {
+				l.Warn("kubernetes-bootstrap: Either the job was cancelled or the pod is being deleted; cancelling work")
 			}
+			// The context cancellation handler in process.Run first calls
+			// Interrupt, waits for its signalGracePeriod, and then calls
+			// Terminate.
 			cancel()
+			// If we're cancelling because the job was cancelled in the UI, we
+			// should self-exit after cancelGracePeriod to be sure.
+			// (If we're cancelling because the pod is being deleted, Kubernetes
+			// enforces it after terminationGracePeriodSeconds, so self-exiting
+			// in that case is superfluous.)
+			time.Sleep(cancelGracePeriod)
+			// We get here if the main goroutine hasn't returned yet.
+			l.Info("kubernetes-bootstrap: Timed out waiting for subprocess to exit; exiting immediately with status 1")
+			os.Exit(1)
 		}); err != nil {
 			return fmt.Errorf("connecting to k8s socket: %w", err)
 		}
@@ -189,29 +204,32 @@ var KubernetesBootstrapCommand = cli.Command{
 			SignalGracePeriod: signalGracePeriod,
 		})
 
-		// We aren't expecting the user to Ctrl-C the process (we're in a k8s
-		// pod), but Kubernetes might send signals.
-		// Forward them to the subprocess.
+		// We aren't expecting the user to Ctrl-C the process (we're in k8s),
+		// but Kubernetes might send signals.
+		// All the containers in the pod get SIGTERM when the pod is deleted,
+		// followed up by SIGKILL after ~TerminationGracePeriodSeconds.
+		// Instead of forwarding Kubernetes's SIGTERM to the subprocess
+		// ourselves, we'll instead swallow the signals, and wait until the
+		// agent container interrupts us via the Unix socket.
 		signals := make(chan os.Signal, 1)
-		signal.Notify(signals,
+		signal.Notify(
+			signals,
 			os.Interrupt,
 			syscall.SIGHUP,
 			syscall.SIGTERM,
 			syscall.SIGINT,
 			syscall.SIGQUIT,
 		)
-
 		go func() {
-			defer signal.Stop(signals)
-			// Forward signals to the subprocess.
 			for {
 				select {
 				case <-ctx.Done():
 					return
 				case <-proc.Done():
 					return
-				case <-signals:
-					proc.Interrupt()
+				case sig := <-signals:
+					// Log but otherwise swallow the signal
+					l.Info("kubernetes-bootstrap: Received %v; awaiting interrupt from agent", sig)
 				}
 			}
 		}()

From d0b0e8c2596810630f69806b58d62c406e67354c Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 16 Oct 2025 12:24:05 +1100
Subject: [PATCH 017/242] Exit directly on panicking signals

---
 clicommand/bootstrap.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go
index 322e2049b6..fbef2a745d 100644
--- a/clicommand/bootstrap.go
+++ b/clicommand/bootstrap.go
@@ -568,6 +568,13 @@ var BootstrapCommand = cli.Command{
 		// If cancelled and our child process returns a non-zero, we should terminate
 		// ourselves with the same signal so that our caller can detect and handle appropriately
 		if cancelled && runtime.GOOS != "windows" {
+			// Per https://pkg.go.dev/os/signal:
+			// "A SIGQUIT, SIGILL, SIGTRAP, SIGABRT, SIGSTKFLT, SIGEMT, or
+			// SIGSYS signal causes the program to exit with a stack dump."
+			// Of these, `received` can only be SIGQUIT.
+			if received == syscall.SIGQUIT {
+				return &SilentExitError{code: 131} // 128 + 3 (SIGQUIT).
+			}
 			if err := signalSelf(l, received); err != nil {
 				l.Error("Failed to signal self: %v", err)
 			}

From 805338955a9bd62d0f57b04d7971a8081c373cca Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Oct 2025 15:30:08 +0000
Subject: [PATCH 018/242] build(deps): bump gopkg.in/DataDog/dd-trace-go.v1
 from 1.74.6 to 1.74.7

Bumps gopkg.in/DataDog/dd-trace-go.v1 from 1.74.6 to 1.74.7.

---
updated-dependencies:
- dependency-name: gopkg.in/DataDog/dd-trace-go.v1
  dependency-version: 1.74.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 13 ++++++-------
 go.sum | 30 ++++++++++++++----------------
 2 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/go.mod b/go.mod
index e3ab670177..06e64d1c5d 100644
--- a/go.mod
+++ b/go.mod
@@ -58,7 +58,7 @@ require (
 	golang.org/x/sys v0.37.0
 	golang.org/x/term v0.36.0
 	google.golang.org/api v0.252.0
-	gopkg.in/DataDog/dd-trace-go.v1 v1.74.6
+	gopkg.in/DataDog/dd-trace-go.v1 v1.74.7
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
 )
@@ -69,7 +69,6 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
-	github.com/DataDog/appsec-internal-go v1.13.0 // indirect
 	github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/obfuscate v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/proto v0.67.0 // indirect
@@ -78,7 +77,7 @@ require (
 	github.com/DataDog/datadog-agent/pkg/util/log v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/version v0.67.0 // indirect
-	github.com/DataDog/dd-trace-go/v2 v2.2.3 // indirect
+	github.com/DataDog/dd-trace-go/v2 v2.3.0 // indirect
 	github.com/DataDog/go-libddwaf/v4 v4.3.2 // indirect
 	github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20250721125240-fdf1ef85b633 // indirect
 	github.com/DataDog/go-sqllexer v0.1.6 // indirect
@@ -87,7 +86,7 @@ require (
 	github.com/DataDog/sketches-go v1.4.7 // indirect
 	github.com/Masterminds/semver/v3 v3.3.1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/agnivade/levenshtein v1.1.1 // indirect
+	github.com/agnivade/levenshtein v1.2.1 // indirect
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
@@ -111,7 +110,6 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
 	github.com/dnephin/pflag v1.0.7 // indirect
-	github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect
 	github.com/ebitengine/purego v0.8.4 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -160,10 +158,11 @@ require (
 	github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shirou/gopsutil/v4 v4.25.8 // indirect
+	github.com/theckman/httpforwarded v0.4.0 // indirect
 	github.com/tinylib/msgp v1.2.5 // indirect
 	github.com/tklauser/go-sysconf v0.3.15 // indirect
 	github.com/tklauser/numcpus v0.10.0 // indirect
-	github.com/vektah/gqlparser/v2 v2.5.19 // indirect
+	github.com/vektah/gqlparser/v2 v2.5.25 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/collector/component v1.31.0 // indirect
@@ -180,7 +179,7 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
+	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
 	golang.org/x/mod v0.28.0 // indirect
 	golang.org/x/text v0.30.0 // indirect
 	golang.org/x/time v0.13.0 // indirect
diff --git a/go.sum b/go.sum
index 6a12540019..f38f1279af 100644
--- a/go.sum
+++ b/go.sum
@@ -23,8 +23,6 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo
 github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/DataDog/appsec-internal-go v1.13.0 h1:aO6DmHYsAU8BNFuvYJByhMKGgcQT3WAbj9J/sgAJxtA=
-github.com/DataDog/appsec-internal-go v1.13.0/go.mod h1:9YppRCpElfGX+emXOKruShFYsdPq7WEPq/Fen4tYYpk=
 github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0 h1:2mEwRWvhIPHMPK4CMD8iKbsrYBxeMBSuuCXumQAwShU=
 github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0/go.mod h1:ejJHsyJTG7NU6c6TDbF7dmckD3g+AUGSdiSXy+ZyaCE=
 github.com/DataDog/datadog-agent/pkg/obfuscate v0.67.0 h1:NcvyDVIUA0NbBDbp7QJnsYhoBv548g8bXq886795mCQ=
@@ -43,8 +41,8 @@ github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyL
 github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8=
 github.com/DataDog/datadog-go/v5 v5.8.1 h1:+GOES5W9zpKlhwHptZVW2C0NLVf7ilr7pHkDcbNvpIc=
 github.com/DataDog/datadog-go/v5 v5.8.1/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
-github.com/DataDog/dd-trace-go/v2 v2.2.3 h1:6RvVdY9suR/rYYYZHjx4txrtSYcRZ5u5Cs2sXMsIBf4=
-github.com/DataDog/dd-trace-go/v2 v2.2.3/go.mod h1:1LcqWELgQwgk6x7sO0MXUgsvxcAVjxSA423cUjvUqR0=
+github.com/DataDog/dd-trace-go/v2 v2.3.0 h1:0Y5kx+Wbod0z8moY0vUbKl6OM0oIV4zAynsVmsq+XT8=
+github.com/DataDog/dd-trace-go/v2 v2.3.0/go.mod h1:yFomJ/rqKNLDbS9ohIDibdz8q9GK0MUSSkBdVDCibGA=
 github.com/DataDog/go-libddwaf/v4 v4.3.2 h1:YGvW2Of1C4e1yU+p7iibmhN2zEOgi9XEchbhQjBxb/A=
 github.com/DataDog/go-libddwaf/v4 v4.3.2/go.mod h1:/AZqP6zw3qGJK5mLrA0PkfK3UQDk1zCI2fUNCt4xftE=
 github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20250721125240-fdf1ef85b633 h1:ZRLR9Lbym748e8RznWzmSoK+OfV+8qW6SdNYA4/IqdA=
@@ -66,8 +64,8 @@ github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lpr
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
-github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
+github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
+github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
 github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y=
 github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8=
 github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw=
@@ -152,8 +150,8 @@ github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbj
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
-github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
+github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo=
+github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
 github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk=
 github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -161,8 +159,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S6Dco8FtAhEI/qkg/00H6RdEGC+MCy5GPiQ+xweNRFE=
 github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
-github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8AOC29zl2mnhQjwyLxxOTuhUazWRsg=
-github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4/go.mod h1:I5sHm0Y0T1u5YjlyqC5GVArM7aNZRUYtTjmJ8mPJFds=
 github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
 github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
@@ -360,6 +356,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/theckman/httpforwarded v0.4.0 h1:N55vGJT+6ojTnLY3LQCNliJC4TW0P0Pkeys1G1WpX2w=
+github.com/theckman/httpforwarded v0.4.0/go.mod h1:GVkFynv6FJreNbgH/bpOU9ITDZ7a5WuzdNCtIMI1pVI=
 github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po=
 github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
 github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
@@ -368,8 +366,8 @@ github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfj
 github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
 github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ=
 github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo=
-github.com/vektah/gqlparser/v2 v2.5.19 h1:bhCPCX1D4WWzCDvkPl4+TP1N8/kLrWnp43egplt7iSg=
-github.com/vektah/gqlparser/v2 v2.5.19/go.mod h1:y7kvl5bBlDeuWIvLtA9849ncyvx6/lj06RsMrEjVy3U=
+github.com/vektah/gqlparser/v2 v2.5.25 h1:FmWtFEa+invTIzWlWK6Vk7BVEZU/97QBzeI8Z1JjGt8=
+github.com/vektah/gqlparser/v2 v2.5.25/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo=
 github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI=
 github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
 github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
@@ -461,8 +459,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
 golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
-golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
-golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
+golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
+golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -536,8 +534,8 @@ google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
 google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
-gopkg.in/DataDog/dd-trace-go.v1 v1.74.6 h1:VBxCK/WkaNjsM9Ygn57scwmiwMqF0gEbuE4C5c2TU5E=
-gopkg.in/DataDog/dd-trace-go.v1 v1.74.6/go.mod h1:jQL1vSDZhH+DJWUOYjkRQ+kU1HUXPvUK41gS1AvHOTE=
+gopkg.in/DataDog/dd-trace-go.v1 v1.74.7 h1:Vz7lua569eApE/1suZueI8ixJZqJCSbWavE9SuTDV9c=
+gopkg.in/DataDog/dd-trace-go.v1 v1.74.7/go.mod h1:yLnCS8nIkriRHfQW+YPDcNG+8ehXYNVio3NSEqy1Iqk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

From c89453bd0ee9f4dc021c0faa95deae444b4557e9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Oct 2025 15:50:58 +0000
Subject: [PATCH 019/242] build(deps): bump the container-images group across 3
 directories with 2 updates

Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/sidecar directory: docker/library/busybox.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `7fa09f7` to `fd70c60`

Updates `docker/library/busybox` from `254e613` to `5b9c2e4`

Updates `buildkite/agent-base` from `9287774` to `de28c3f`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: docker/library/busybox
  dependency-version: 1-musl
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/sidecar/Dockerfile      | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 8e59900cd9..bad96a435a 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:7fa09f71a5246693cf09e7905222b5b6776a6ecd38cee2a4d5fbeab6c708ec21
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:fd70c602f586e9613cce31ee8f3f090180a59340a0737d7c26cd40e3493fccbe
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/sidecar/Dockerfile b/packaging/docker/sidecar/Dockerfile
index a0f5927c4e..a25ebae160 100644
--- a/packaging/docker/sidecar/Dockerfile
+++ b/packaging/docker/sidecar/Dockerfile
@@ -12,6 +12,6 @@ RUN mkdir /buildkite \
 COPY buildkite-agent.cfg /buildkite/
 COPY buildkite-agent-$TARGETOS-$TARGETARCH /buildkite/bin/buildkite-agent
 
-FROM public.ecr.aws/docker/library/busybox:1-musl@sha256:254e6134b1bf813b34e920bc4235864a54079057d51ae6db9a4f2328f261c2ad
+FROM public.ecr.aws/docker/library/busybox:1-musl@sha256:5b9c2e4df019f56a2cbb0d7b748208c44cc77c03f793ae1d4bdbdf3e41b044cd
 COPY --from=0 /buildkite /buildkite
 VOLUME /buildkite
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index f9fa60dcf0..f46fff7c11 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:9287774d93203409df9a596f4b9e4a13cd5a6b297e31f61fcc708c1ef72e2c4e
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:de28c3f38091efbcf192cc04dbd4ab07e365b73c0b4d3fb40d661ca4b2a89e04
 ARG TARGETOS
 ARG TARGETARCH
 

From f48392495cdcc06f8ef52c9ef83e0e081da7c8f9 Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Tue, 21 Oct 2025 15:54:09 +1100
Subject: [PATCH 020/242] PS-1295: inject OpenTelemetry context to all child
 processes

---
 internal/shell/shell.go | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/internal/shell/shell.go b/internal/shell/shell.go
index bb05d9aab7..94947b050f 100644
--- a/internal/shell/shell.go
+++ b/internal/shell/shell.go
@@ -28,6 +28,9 @@ import (
 	"github.com/buildkite/shellwords"
 	"github.com/gofrs/flock"
 	"github.com/opentracing/opentracing-go"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 )
 
 const lockRetryDuration = time.Second
@@ -498,16 +501,32 @@ func WithStringSearch(m map[string]bool) RunCommandOpt { return func(c *runConfi
 // injectTraceCtx adds tracing information to the given env vars to support
 // distributed tracing across jobs/builds.
 func (s *Shell) injectTraceCtx(ctx context.Context, env *env.Environment) {
-	span := opentracing.SpanFromContext(ctx)
-	// Not all shell runs will have tracing (nor do they really need to).
-	if span == nil {
+	// OpenTracing path (for Datadog backend)
+	if span := opentracing.SpanFromContext(ctx); span != nil {
+		if err := tracetools.EncodeTraceContext(span, env.Dump(), s.traceContextCodec); err != nil {
+			if s.debug {
+				s.Warningf("Failed to encode trace context: %v", err)
+			}
+		}
 		return
 	}
-	if err := tracetools.EncodeTraceContext(span, env.Dump(), s.traceContextCodec); err != nil {
-		if s.debug {
-			s.Warningf("Failed to encode trace context: %v", err)
+
+	// OpenTelemetry path (for OpenTelemetry backend)
+	if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
+		carrier := propagation.MapCarrier{}
+		otel.GetTextMapPropagator().Inject(ctx, carrier)
+
+		// Transform HTTP header names to environment variable names per
+		// https://opentelemetry.io/docs/specs/otel/context/env-carriers/
+		// Examples: "traceparent" -> "TRACEPARENT", "X-B3-TraceId" -> "X_B3_TRACEID"
+		//
+		// It remains unclear whether various ecosystems are well equipped handling normalized env vars.
+		// But it will be trivial to conform to the standard.
+		// We shall see how community responds to this.
+		for k, v := range carrier {
+			envKey := strings.ToUpper(strings.ReplaceAll(k, "-", "_"))
+			env.Set(envKey, v)
 		}
-		return
 	}
 }
 

From c20f289fda1d6d5e95fc7bfb096f7ef773c9eaa2 Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Tue, 21 Oct 2025 10:41:48 +1100
Subject: [PATCH 021/242] PS-1302: refresh checkout root file handle after
 checkout hook

---
 internal/job/checkout.go | 112 +++++++++++++++++++++++++--------------
 1 file changed, 71 insertions(+), 41 deletions(-)

diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index 1ba865fb60..3746a66b1a 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -95,14 +95,9 @@ func (e *Executor) createCheckoutDir() error {
 		}
 	}
 
-	root, err := os.OpenRoot(checkoutPath)
-	if err != nil {
-		return fmt.Errorf("opening checkout path as root: %w", err)
+	if err := e.refreshCheckoutRoot(); err != nil {
+		return err
 	}
-	// This cleanup is largely ornamental, since the executor pointer only
-	// becomes unreachable when the bootstrap exits.
-	runtime.AddCleanup(e, func(r *os.Root) { r.Close() }, root)
-	e.checkoutRoot = root
 
 	if e.shell.Getwd() != checkoutPath {
 		if err := e.shell.Chdir(checkoutPath); err != nil {
@@ -113,6 +108,26 @@ func (e *Executor) createCheckoutDir() error {
 	return nil
 }
 
+// refreshCheckoutRoot refreshes e.checkoutRoot
+func (e *Executor) refreshCheckoutRoot() error {
+	checkoutPath, _ := e.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
+	if e.checkoutRoot != nil {
+		if err := e.checkoutRoot.Close(); err != nil {
+			// While it's unlikely, it's not a blocking error
+			e.shell.Warningf("unable to close existing checkoutRoot during refreshCheckoutRoot: %w", err)
+		}
+	}
+	root, err := os.OpenRoot(checkoutPath)
+	if err != nil {
+		return fmt.Errorf("opening checkout path as root: %w", err)
+	}
+	// This cleanup is largely ornamental, since the executor pointer only
+	// becomes unreachable when the bootstrap exits.
+	runtime.AddCleanup(e, func(r *os.Root) { r.Close() }, root)
+	e.checkoutRoot = root
+	return nil
+}
+
 // CheckoutPhase creates the build directory and makes sure we're running the
 // build at the right commit.
 func (e *Executor) CheckoutPhase(ctx context.Context) error {
@@ -156,6 +171,52 @@ func (e *Executor) CheckoutPhase(ctx context.Context) error {
 		return err
 	}
 
+	if err := e.checkout(ctx); err != nil {
+		return err
+	}
+
+	err = e.sendCommitToBuildkite(ctx)
+	if err != nil {
+		e.shell.OptionalWarningf("git-commit-resolution-failed", "Couldn't send commit information to Buildkite: %v", err)
+	}
+
+	// Store the current value of BUILDKITE_BUILD_CHECKOUT_PATH, so we can detect if
+	// one of the post-checkout hooks changed it.
+	previousCheckoutPath, exists := e.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
+	if !exists {
+		e.shell.Printf("Could not determine previous checkout path from BUILDKITE_BUILD_CHECKOUT_PATH")
+	}
+
+	// Run post-checkout hooks
+	if err := e.executeGlobalHook(ctx, "post-checkout"); err != nil {
+		return err
+	}
+
+	if err := e.executeLocalHook(ctx, "post-checkout"); err != nil {
+		return err
+	}
+
+	if err := e.executePluginHook(ctx, "post-checkout", e.pluginCheckouts); err != nil {
+		return err
+	}
+
+	// Capture the new checkout path so we can see if it's changed.
+	newCheckoutPath, _ := e.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
+
+	// If the working directory has been changed by a hook, log and switch to it
+	if previousCheckoutPath != "" && previousCheckoutPath != newCheckoutPath {
+		e.shell.Headerf("A post-checkout hook has changed the working directory to \"%s\"", newCheckoutPath)
+
+		if err := e.shell.Chdir(newCheckoutPath); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// checkout runs checkout hook or default checkout logic
+func (e *Executor) checkout(ctx context.Context) error {
 	// There can only be one checkout hook, either plugin or global, in that order
 	switch {
 	case e.hasPluginHook("checkout"):
@@ -257,43 +318,12 @@ func (e *Executor) CheckoutPhase(ctx context.Context) error {
 		}
 	}
 
-	err = e.sendCommitToBuildkite(ctx)
-	if err != nil {
-		e.shell.OptionalWarningf("git-commit-resolution-failed", "Couldn't send commit information to Buildkite: %v", err)
-	}
-
-	// Store the current value of BUILDKITE_BUILD_CHECKOUT_PATH, so we can detect if
-	// one of the post-checkout hooks changed it.
-	previousCheckoutPath, exists := e.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
-	if !exists {
-		e.shell.Printf("Could not determine previous checkout path from BUILDKITE_BUILD_CHECKOUT_PATH")
-	}
-
-	// Run post-checkout hooks
-	if err := e.executeGlobalHook(ctx, "post-checkout"); err != nil {
-		return err
-	}
-
-	if err := e.executeLocalHook(ctx, "post-checkout"); err != nil {
-		return err
-	}
-
-	if err := e.executePluginHook(ctx, "post-checkout", e.pluginCheckouts); err != nil {
+	// After everything, we need to refresh checkout root.
+	// This is because checkout hook might re-create the checkout root folder entirely, deprecating e.checkoutRoot.
+	if err := e.refreshCheckoutRoot(); err != nil {
 		return err
 	}
 
-	// Capture the new checkout path so we can see if it's changed.
-	newCheckoutPath, _ := e.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
-
-	// If the working directory has been changed by a hook, log and switch to it
-	if previousCheckoutPath != "" && previousCheckoutPath != newCheckoutPath {
-		e.shell.Headerf("A post-checkout hook has changed the working directory to \"%s\"", newCheckoutPath)
-
-		if err := e.shell.Chdir(newCheckoutPath); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 

From 129389a68c6370581f7a448e5c0f8dc6a2a1fad5 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 16 Oct 2025 12:24:05 +1100
Subject: [PATCH 022/242] Defer k8s socket close

---
 clicommand/kubernetes_bootstrap.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clicommand/kubernetes_bootstrap.go b/clicommand/kubernetes_bootstrap.go
index 0f013fe18f..7fc4cd9af3 100644
--- a/clicommand/kubernetes_bootstrap.go
+++ b/clicommand/kubernetes_bootstrap.go
@@ -92,6 +92,7 @@ var KubernetesBootstrapCommand = cli.Command{
 		if err != nil {
 			return fmt.Errorf("error connecting to kubernetes runner: %w", err)
 		}
+		defer socket.Close()
 
 		// Start with the registration response env, then override with our
 		// existing env.

From 217ea51f50325e39cc173bccb1a006d7fbfcb9b5 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 16 Oct 2025 12:24:05 +1100
Subject: [PATCH 023/242] Fix duration in "Forcefully stopping" log

---
 clicommand/agent_start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 76bae731b4..2801305221 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1424,7 +1424,7 @@ func (ps *poolSignals) handleLoop(ctx context.Context, signals chan os.Signal) {
 				ps.pool.Stop(true)
 
 			case 2:
-				ps.log.Info("Forcefully stopping running jobs and stopping the agent(s) in %d seconds", ps.cancelGracePeriod)
+				ps.log.Info("Forcefully stopping running jobs and stopping the agent(s) in %v", ps.cancelGracePeriod)
 				if !ps.skipGraceful {
 					ps.log.Info("Press Ctrl-C one more time to exit immediately without disconnecting")
 				}

From 19f9ef0f5d83af858b5d513e5091e6bb6659d343 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 16 Oct 2025 12:24:05 +1100
Subject: [PATCH 024/242] Don't cancel RunJob context on agent stop

---
 agent/agent_worker.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 7db8a28265..8f9a9033ff 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -664,18 +664,26 @@ func (a *AgentWorker) Ping(ctx context.Context) (job *api.Job, action string, er
 // state. If the job is in an unassignable state, it will return an error immediately.
 // Otherwise, it will retry every 3s for 30 s. The whole operation will timeout after 5 min.
 func (a *AgentWorker) AcquireAndRunJob(ctx context.Context, jobId string) error {
-	ctx, cancel := context.WithCancel(ctx)
+	// Note: Context.Cancel is a blunt instrument. It will (for example)
+	// prevent the final API calls to upload remaining logs and mark the job
+	// finished.
+	// But we do want to abort the retry loop in AcquireJob early if possible.
+	// So, use context cancellation to abort AcquireJob on agent stop, but not
+	// RunJob.
+	// The agent's signal handler handles cancellation after a job has begun.
+	acquireCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	go func() {
 		<-a.stop
 		cancel()
 	}()
 
-	job, err := a.client.AcquireJob(ctx, jobId)
+	job, err := a.client.AcquireJob(acquireCtx, jobId)
 	if err != nil {
 		return fmt.Errorf("failed to acquire job: %w", err)
 	}
 
-	// Now that we've acquired the job, let's run it
+	// Now that we've acquired the job, let's run it.
 	return a.RunJob(ctx, job, nil)
 }
 

From 0ef5709cae36dcea8a60146539f3d5abe36fe8e7 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 16 Oct 2025 12:24:05 +1100
Subject: [PATCH 025/242] Various comments and internal API changes

---
 agent/agent_worker.go | 15 ++++++++++---
 agent/job_runner.go   | 23 +++++++++----------
 agent/run_job.go      | 51 +++++++++++++++++++++++++++----------------
 3 files changed, 56 insertions(+), 33 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 8f9a9033ff..8ee0e2a6ba 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -87,7 +87,8 @@ type AgentWorker struct {
 	// The signal to use for cancellation
 	cancelSig process.Signal
 
-	// Stop controls
+	// Stop controls. Note that Stopping != Cancelling. See the [Stop] method
+	// for an explanation.
 	stopOnce sync.Once // prevents double-closing the channel
 	stop     chan struct{}
 
@@ -502,7 +503,15 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 }
 
 // Stops the agent from accepting new work and cancels any current work it's
-// running
+// running. Whether or not Stop also cancels the current job depends on the
+// `graceful` argument.
+//
+//   - *Graceful* stop: allow the current job to finish without interruption
+//     (but don't accept new jobs) before exiting
+//   - *Ungraceful* stop: cancel the current job (jobRunner.CancelAndStop),
+//     wait a grace period for it to exit, and then exit ourselves
+//
+// In both cases, the ping loop should cease running.
 func (a *AgentWorker) Stop(graceful bool) {
 	if graceful {
 		select {
@@ -526,7 +535,7 @@ func (a *AgentWorker) Stop(graceful bool) {
 			// Kill the current job. Doesn't do anything if the job
 			// is already being killed, so it's safe to call
 			// multiple times.
-			err := a.jobRunner.CancelAndStop()
+			err := a.jobRunner.Cancel(true /* agent is stopping */)
 			if err != nil {
 				a.logger.Error("Unexpected error canceling job (err: %s)", err)
 			}
diff --git a/agent/job_runner.go b/agent/job_runner.go
index 35fb1984a2..f5a310d1fc 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -13,6 +13,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/buildkite/agent/v3/api"
@@ -115,18 +116,16 @@ type JobRunner struct {
 	// jobLogs is an io.Writer that sends data to the job logs
 	jobLogs io.Writer
 
-	// If the job is being cancelled
-	cancelled bool
+	// Job cancellation control
+	cancelLock sync.Mutex // prevent concurrent calls to Cancel
+
+	// State flags
+	cancelled     atomic.Bool // job is cancelled?
+	agentStopping atomic.Bool // agent is stopping?
 
 	// When the job was started
 	startedAt time.Time
 
-	// If the agent is being stopped
-	stopped bool
-
-	// A lock to protect concurrent calls to cancel
-	cancelLock sync.Mutex
-
 	// Files containing a copy of the job env
 	envShellFile *os.File
 	envJSONFile  *os.File
@@ -811,15 +810,17 @@ func (r *JobRunner) jobCancellationChecker(ctx context.Context, wg *sync.WaitGro
 		if err != nil {
 			if response != nil && response.StatusCode == 401 {
 				r.agentLogger.Error("Invalid access token, cancelling job %s", r.conf.Job.ID)
-				if err := r.Cancel(); err != nil {
+				if err := r.Cancel(false /* agent is not necessarily stopping */); err != nil {
 					r.agentLogger.Error("Failed to cancel the process (job: %s): %v", r.conf.Job.ID, err)
 				}
 			} else {
 				// We don't really care if it fails, we'll just try again soon anyway
 				r.agentLogger.Warn("Problem with getting job state %s (%s)", r.conf.Job.ID, err)
 			}
-		} else if jobState.State == "canceling" || jobState.State == "canceled" {
-			if err := r.Cancel(); err != nil {
+			continue // the loop
+		}
+		if jobState.State == "canceling" || jobState.State == "canceled" {
+			if err := r.Cancel(false /* agent is not necessarily stopping */); err != nil {
 				r.agentLogger.Error("Unexpected error canceling process as requested by server (job: %s) (err: %s)", r.conf.Job.ID, err)
 			}
 		}
diff --git a/agent/run_job.go b/agent/run_job.go
index 5cc83a2177..1f72c637f3 100644
--- a/agent/run_job.go
+++ b/agent/run_job.go
@@ -71,6 +71,10 @@ func (e *missingKeyError) Error() string {
 
 // Run runs the job.
 func (r *JobRunner) Run(ctx context.Context, ignoreAgentInDispatches *bool) (err error) {
+	if r.cancelled.Load() {
+		return errors.New("job already cancelled before running")
+	}
+
 	r.agentLogger.Info("Starting job %s for build at %s", r.conf.Job.ID, r.conf.Job.Env["BUILDKITE_BUILD_URL"])
 
 	ctx, done := status.AddItem(ctx, "Job Runner", "", nil)
@@ -328,9 +332,9 @@ func (r *JobRunner) runJob(ctx context.Context) core.ProcessExit {
 	// start. Normally such errors are hidden in the Kubernetes events. Let's feed them up
 	// to the user as they may be the caused by errors in the pipeline definition.
 	k8sProcess, isK8s := r.process.(*kubernetes.Runner)
-	if isK8s && !r.stopped {
+	if isK8s && !r.agentStopping.Load() {
 		switch {
-		case r.cancelled && k8sProcess.AnyClientIn(kubernetes.StateNotYetConnected):
+		case r.cancelled.Load() && k8sProcess.AnyClientIn(kubernetes.StateNotYetConnected):
 			fmt.Fprint(r.jobLogs, `+++ Unknown container exit status
 One or more containers never connected to the agent. Perhaps the container image specified in your podSpec could not be pulled (ImagePullBackOff)?
 `)
@@ -349,12 +353,12 @@ One or more containers connected to the agent, but then stopped communicating wi
 	}
 
 	switch {
-	case r.stopped:
-		// The agent is being gracefully stopped, and we signaled the job to end. Often due
-		// to pending host shutdown or EC2 spot instance termination
+	case r.agentStopping.Load():
+		// The agent is being ungracefully stopped, and we signaled the job to
+		// end. Often due to pending host shutdown or EC2 spot instance termination
 		exit.SignalReason = SignalReasonAgentStop
 
-	case r.cancelled:
+	case r.cancelled.Load():
 		// The job was signaled because it was cancelled via the buildkite web UI
 		exit.SignalReason = SignalReasonCancel
 
@@ -500,18 +504,26 @@ func (r *JobRunner) streamJobLogsAfterProcessStart(ctx context.Context, wg *sync
 	// The final output after the process has finished is processed in Run().
 }
 
-func (r *JobRunner) CancelAndStop() error {
-	r.cancelLock.Lock()
-	r.stopped = true
-	r.cancelLock.Unlock()
-	return r.Cancel()
-}
-
-func (r *JobRunner) Cancel() error {
+// Cancel cancels the job. It can be summarised as:
+//   - Send the process an Interrupt. When run via a subprocess, this translates
+//     into SIGTERM. When run via the k8s socket, this transitions the connected
+//     client to RunStateInterrupt.
+//   - Wait for the signal grace period.
+//   - If the job hasn't exited, send the process a Terminate. This is either
+//     SIGKILL or closing the k8s socket server.
+//
+// Cancel blocks until this process is complete.
+// The `agentStopping` arg mainly affects logged messages.
+func (r *JobRunner) Cancel(agentStopping bool) error {
 	r.cancelLock.Lock()
 	defer r.cancelLock.Unlock()
 
-	if r.cancelled {
+	// In case the user clicks "Cancel" in the UI while the agent happens to be
+	// stopping, only go from !stopping -> stopping.
+	r.agentStopping.Store(r.agentStopping.Load() || agentStopping)
+
+	// Return early if already cancelled.
+	if !r.cancelled.CompareAndSwap(false, true) {
 		return nil
 	}
 
@@ -521,7 +533,7 @@ func (r *JobRunner) Cancel() error {
 	}
 
 	reason := ""
-	if r.stopped {
+	if r.agentStopping.Load() {
 		reason = "(agent stopping)"
 	}
 
@@ -532,9 +544,10 @@ func (r *JobRunner) Cancel() error {
 		reason,
 	)
 
-	r.cancelled = true
-
-	// First we interrupt the process (ctrl-c or SIGINT)
+	// First we interrupt the process with the configured CancelSignal.
+	// At some point in the past, for subprocesses, the default was intended to
+	// be SIGINT, but you will find that the cancel-signal flag default and
+	// the process package's default are both SIGTERM.
 	if err := r.process.Interrupt(); err != nil {
 		return err
 	}

From 1b2ea19d587b127eb89645f660eb3937f87a031b Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 21 Oct 2025 16:46:34 +1100
Subject: [PATCH 026/242] Concurrently stop all workers

---
 agent/agent_pool.go       | 18 ++++++++++++++++--
 agent/agent_worker.go     | 26 ++++++++++++++------------
 agent/run_job.go          |  2 +-
 clicommand/agent_start.go | 30 +++++++++++++++++++-----------
 4 files changed, 50 insertions(+), 26 deletions(-)

diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index 575a59500e..c34056112d 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"sync"
 
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/status"
@@ -50,7 +51,7 @@ func (ap *AgentPool) StartStatusServer(ctx context.Context, l logger.Logger, add
 	}()
 }
 
-// Start kicks off the parallel AgentWorkers and waits for them to finish
+// Start kicks off the parallel AgentWorkers and waits for them to finish.
 func (r *AgentPool) Start(ctx context.Context) error {
 	ctx, setStat, done := status.AddSimpleItem(ctx, "Agent Pool")
 	defer done()
@@ -87,10 +88,23 @@ func (r *AgentPool) runWorker(ctx context.Context, worker *AgentWorker) error {
 	return worker.Start(ctx, r.idleMonitor)
 }
 
+// Stop stops all workers in the pool simultaneously. It blocks until Stop has
+// returned for every worker - note that passing `false` means each Stop will
+// wait for job cancellation.
 func (r *AgentPool) Stop(graceful bool) {
+	var wg sync.WaitGroup
+	wg.Add(len(r.workers))
 	for _, worker := range r.workers {
-		worker.Stop(graceful)
+		// Because Stop calls the job runner's Cancel, which can block, tell all
+		// the workers to stop simultaneously.
+		// The number of concurrent Stops is bounded by the spawn count, and
+		// there already exists a handful of goroutines per worker.
+		go func() {
+			worker.Stop(graceful)
+			wg.Done()
+		}()
 	}
+	wg.Wait()
 }
 
 func (ap *AgentPool) statusJSONHandler(l logger.Logger) http.HandlerFunc {
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 8ee0e2a6ba..f731d07c63 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -8,6 +8,7 @@ import (
 	"math/rand/v2"
 	"net/http"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/buildkite/agent/v3/api"
@@ -97,7 +98,7 @@ type AgentWorker struct {
 
 	// When this worker runs a job, we'll store an instance of the
 	// JobRunner here
-	jobRunner *JobRunner
+	jobRunner atomic.Pointer[JobRunner]
 
 	// Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection.
 	agentStdout io.Writer
@@ -507,9 +508,10 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 // `graceful` argument.
 //
 //   - *Graceful* stop: allow the current job to finish without interruption
-//     (but don't accept new jobs) before exiting
+//     (but don't accept new jobs) before exiting. Does not block.
 //   - *Ungraceful* stop: cancel the current job (jobRunner.CancelAndStop),
-//     wait a grace period for it to exit, and then exit ourselves
+//     wait a grace period for it to exit, and then exit ourselves. This blocks
+//     until the job is cancelled.
 //
 // In both cases, the ping loop should cease running.
 func (a *AgentWorker) Stop(graceful bool) {
@@ -521,21 +523,21 @@ func (a *AgentWorker) Stop(graceful bool) {
 		default:
 			// If we have a job, tell the user that we'll wait for
 			// it to finish before disconnecting
-			if a.jobRunner != nil {
+			if a.jobRunner.Load() != nil {
 				a.logger.Info("Gracefully stopping agent. Waiting for current job to finish before disconnecting...")
 			} else {
 				a.logger.Info("Gracefully stopping agent. Since there is no job running, the agent will disconnect immediately")
 			}
 		}
 	} else { // ungraceful
-		// If there's a job running, kill it, then disconnect
-		if a.jobRunner != nil {
+		// If there's a job running, kill it, then disconnect.
+		if jr := a.jobRunner.Load(); jr != nil {
 			a.logger.Info("Forcefully stopping agent. The current job will be canceled before disconnecting...")
 
 			// Kill the current job. Doesn't do anything if the job
 			// is already being killed, so it's safe to call
 			// multiple times.
-			err := a.jobRunner.Cancel(true /* agent is stopping */)
+			err := jr.Cancel(true /* agent is stopping */)
 			if err != nil {
 				a.logger.Error("Unexpected error canceling job (err: %s)", err)
 			}
@@ -765,11 +767,11 @@ func (a *AgentWorker) RunJob(ctx context.Context, acceptResponse *api.Job, ignor
 	if err != nil {
 		return fmt.Errorf("Failed to initialize job: %w", err)
 	}
-	a.jobRunner = jr
-	defer func() {
-		// No more job, no more runner.
-		a.jobRunner = nil
-	}()
+	if !a.jobRunner.CompareAndSwap(nil, jr) {
+		return fmt.Errorf("Agent worker already has a job running")
+	}
+	// No more job, no more runner.
+	defer a.jobRunner.Store(nil)
 
 	// Start running the job
 	if err := jr.Run(ctx, ignoreAgentInDispatches); err != nil {
diff --git a/agent/run_job.go b/agent/run_job.go
index 1f72c637f3..da3bc41ac9 100644
--- a/agent/run_job.go
+++ b/agent/run_job.go
@@ -555,7 +555,7 @@ func (r *JobRunner) Cancel(agentStopping bool) error {
 	select {
 	// Grace period between Interrupt and Terminate = the signal grace period.
 	// Extra time between the end of the signal grace period and the end of the
-	// cancel grace period is the time we (agent side) needs to upload logs and
+	// cancel grace period is the time we (agent side) need to upload logs and
 	// disconnect (if the agent is exiting).
 	case <-time.After(r.conf.AgentConfiguration.SignalGracePeriod):
 		r.agentLogger.Info(
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 2801305221..55403da0a7 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1408,34 +1408,42 @@ func (ps *poolSignals) handleLoop(ctx context.Context, signals chan os.Signal) {
 		interruptCount = 1
 	}
 
+	ungracefulStop := func() {
+		// We shouldn't block the signal handler loop either by waiting
+		// for the jobs to cancel or by waiting for the cancel grace
+		// period to expire.
+		go ps.pool.Stop(false /* ungraceful */) // one last chance to stop
+		go func() {
+			// Assuming cancelling jobs takes the full cancel grace period,
+			// allow 1 second to send agent disconnects.
+			time.Sleep(ps.cancelGracePeriod + 1*time.Second)
+			// We get here if the main goroutine hasn't returned yet.
+			ps.log.Info("Timed out waiting for agents to exit; exiting immediately with status 1")
+			os.Exit(1)
+		}()
+	}
+
 	for sig := range signals {
 		ps.log.Debug("Received signal `%v`", sig)
 		setStatus(fmt.Sprintf("Received signal `%v`", sig))
 
 		switch sig {
 		case syscall.SIGQUIT:
-			ps.pool.Stop(false)
+			ungracefulStop()
 
 		case syscall.SIGTERM, syscall.SIGINT:
 			interruptCount++
 			switch interruptCount {
 			case 1:
 				ps.log.Info("Received CTRL-C, send again to forcefully kill the agent(s)")
-				ps.pool.Stop(true)
+				ps.pool.Stop(true /* graceful */)
 
 			case 2:
 				ps.log.Info("Forcefully stopping running jobs and stopping the agent(s) in %v", ps.cancelGracePeriod)
 				if !ps.skipGraceful {
-					ps.log.Info("Press Ctrl-C one more time to exit immediately without disconnecting")
+					ps.log.Info("Press Ctrl-C one more time to exit immediately without disconnecting - note that agents will be considered lost!")
 				}
-				ps.pool.Stop(false) // one last chance to stop
-
-				go func() {
-					time.Sleep(ps.cancelGracePeriod)
-					// We get here if the main goroutine hasn't returned yet.
-					ps.log.Info("Timed out waiting for agents to exit; exiting immediately with status 1")
-					os.Exit(1)
-				}()
+				ungracefulStop()
 
 			case 3:
 				ps.log.Info("Exiting immediately with status 1")

From 64385feb367535ab3aa9bbe0a56e3baf99165430 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 22 Oct 2025 11:48:09 +1100
Subject: [PATCH 027/242] Remove bool args

---
 agent/agent_pool.go       | 21 ++++++---
 agent/agent_worker.go     | 91 ++++++++++++++++++++-------------------
 agent/job_runner.go       |  6 +--
 agent/run_job.go          | 32 ++++++++++----
 clicommand/agent_start.go |  4 +-
 5 files changed, 89 insertions(+), 65 deletions(-)

diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index c34056112d..f31b8153eb 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -88,19 +88,26 @@ func (r *AgentPool) runWorker(ctx context.Context, worker *AgentWorker) error {
 	return worker.Start(ctx, r.idleMonitor)
 }
 
-// Stop stops all workers in the pool simultaneously. It blocks until Stop has
-// returned for every worker - note that passing `false` means each Stop will
-// wait for job cancellation.
-func (r *AgentPool) Stop(graceful bool) {
+// StopGracefully stops all workers in the pool gracefully.
+func (r *AgentPool) StopGracefully() {
+	for _, worker := range r.workers {
+		worker.StopGracefully()
+	}
+}
+
+// StopUngracefully stops all workers in the pool ungracefully. It blocks until
+// all workers have returned from stopping, which means waiting for job
+// cancellation to finish.
+func (r *AgentPool) StopUngracefully() {
 	var wg sync.WaitGroup
 	wg.Add(len(r.workers))
 	for _, worker := range r.workers {
-		// Because Stop calls the job runner's Cancel, which can block, tell all
-		// the workers to stop simultaneously.
+		// Because StopUngracefully calls the job runner's Cancel, which blocks,
+		// concurrently stop all the workers.
 		// The number of concurrent Stops is bounded by the spawn count, and
 		// there already exists a handful of goroutines per worker.
 		go func() {
-			worker.Stop(graceful)
+			worker.StopUngracefully()
 			wg.Done()
 		}()
 	}
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index f731d07c63..2e0901d199 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -391,7 +391,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 
 		switch action {
 		case "disconnect":
-			a.Stop(false)
+			a.StopUngracefully()
 			return nil
 
 		case "pause":
@@ -503,56 +503,57 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 	}
 }
 
-// Stops the agent from accepting new work and cancels any current work it's
-// running. Whether or not Stop also cancels the current job depends on the
-// `graceful` argument.
-//
-//   - *Graceful* stop: allow the current job to finish without interruption
-//     (but don't accept new jobs) before exiting. Does not block.
-//   - *Ungraceful* stop: cancel the current job (jobRunner.CancelAndStop),
-//     wait a grace period for it to exit, and then exit ourselves. This blocks
-//     until the job is cancelled.
-//
-// In both cases, the ping loop should cease running.
-func (a *AgentWorker) Stop(graceful bool) {
-	if graceful {
-		select {
-		case <-a.stop:
-			a.logger.Warn("Agent is already gracefully stopping...")
+// StopGracefully stops the agent from accepting new work. It allows the current
+// job to finish without interruption. Does not block.
+func (a *AgentWorker) StopGracefully() {
+	select {
+	case <-a.stop:
+		a.logger.Warn("Agent is already gracefully stopping...")
+		return
 
-		default:
-			// If we have a job, tell the user that we'll wait for
-			// it to finish before disconnecting
-			if a.jobRunner.Load() != nil {
-				a.logger.Info("Gracefully stopping agent. Waiting for current job to finish before disconnecting...")
-			} else {
-				a.logger.Info("Gracefully stopping agent. Since there is no job running, the agent will disconnect immediately")
-			}
-		}
-	} else { // ungraceful
-		// If there's a job running, kill it, then disconnect.
-		if jr := a.jobRunner.Load(); jr != nil {
-			a.logger.Info("Forcefully stopping agent. The current job will be canceled before disconnecting...")
-
-			// Kill the current job. Doesn't do anything if the job
-			// is already being killed, so it's safe to call
-			// multiple times.
-			err := jr.Cancel(true /* agent is stopping */)
-			if err != nil {
-				a.logger.Error("Unexpected error canceling job (err: %s)", err)
-			}
-		} else {
-			a.logger.Info("Forcefully stopping agent. Since there is no job running, the agent will disconnect immediately")
-		}
+	default:
+		// continue below
+	}
+
+	// If we have a job, tell the user that we'll wait for it to finish
+	// before disconnecting
+	if a.jobRunner.Load() != nil {
+		a.logger.Info("Gracefully stopping agent. Waiting for current job to finish before disconnecting...")
+	} else {
+		a.logger.Info("Gracefully stopping agent. Since there is no job running, the agent will disconnect immediately")
 	}
 
 	a.stopOnce.Do(func() {
-		// Use the closure of the stop channel as a signal to the main run loop in Start()
-		// to stop looping and terminate
+		// Use the closure of the stop channel as a signal to the main run
+		// loop in Start() to stop looping and terminate
 		close(a.stop)
 	})
 }
 
+// StopUngracefully stops the agent from accepting new work and cancels any
+// existing job. It blocks until the job is cancelled, if there is one.
+func (a *AgentWorker) StopUngracefully() {
+	a.stopOnce.Do(func() {
+		// Use the closure of the stop channel as a signal to the main run
+		// loop in Start() to stop looping and terminate
+		close(a.stop)
+	})
+
+	// If there's a job running, kill it, then disconnect.
+	if jr := a.jobRunner.Load(); jr != nil {
+		a.logger.Info("Forcefully stopping agent. The current job will be canceled before disconnecting...")
+
+		// Kill the current job. Doesn't do anything if the job
+		// is already being killed, so it's safe to call
+		// multiple times.
+		if err := jr.Cancel(CancelReasonAgentStopping); err != nil {
+			a.logger.Error("Unexpected error canceling job (err: %s)", err)
+		}
+	} else {
+		a.logger.Info("Forcefully stopping agent. Since there is no job running, the agent will disconnect immediately")
+	}
+}
+
 // Connects the agent to the Buildkite Agent API, retrying up to 10 times if it
 // fails.
 func (a *AgentWorker) Connect(ctx context.Context) error {
@@ -572,7 +573,7 @@ func (a *AgentWorker) Heartbeat(ctx context.Context) error {
 		b, resp, err := a.apiClient.Heartbeat(ctx)
 		if err != nil {
 			if resp != nil && !api.IsRetryableStatus(resp) {
-				a.Stop(false)
+				a.StopUngracefully()
 				r.Break()
 				return nil, &errUnrecoverable{action: "Heartbeat", response: resp, err: err}
 			}
@@ -622,7 +623,7 @@ func (a *AgentWorker) Ping(ctx context.Context) (job *api.Job, action string, er
 		// The reason we do this after the disconnect check is because the backend can (and does) send disconnect actions in
 		// responses with non-retryable statuses
 		if resp != nil && !api.IsRetryableStatus(resp) {
-			a.Stop(false)
+			a.StopUngracefully()
 			return nil, action, &errUnrecoverable{action: "Ping", response: resp, err: pingErr}
 		}
 
diff --git a/agent/job_runner.go b/agent/job_runner.go
index f5a310d1fc..fd866cea20 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -121,7 +121,7 @@ type JobRunner struct {
 
 	// State flags
 	cancelled     atomic.Bool // job is cancelled?
-	agentStopping atomic.Bool // agent is stopping?
+	agentStopping atomic.Bool
 
 	// When the job was started
 	startedAt time.Time
@@ -810,7 +810,7 @@ func (r *JobRunner) jobCancellationChecker(ctx context.Context, wg *sync.WaitGro
 		if err != nil {
 			if response != nil && response.StatusCode == 401 {
 				r.agentLogger.Error("Invalid access token, cancelling job %s", r.conf.Job.ID)
-				if err := r.Cancel(false /* agent is not necessarily stopping */); err != nil {
+				if err := r.Cancel(CancelReasonInvalidToken); err != nil {
 					r.agentLogger.Error("Failed to cancel the process (job: %s): %v", r.conf.Job.ID, err)
 				}
 			} else {
@@ -820,7 +820,7 @@ func (r *JobRunner) jobCancellationChecker(ctx context.Context, wg *sync.WaitGro
 			continue // the loop
 		}
 		if jobState.State == "canceling" || jobState.State == "canceled" {
-			if err := r.Cancel(false /* agent is not necessarily stopping */); err != nil {
+			if err := r.Cancel(CancelReasonJobState); err != nil {
 				r.agentLogger.Error("Unexpected error canceling process as requested by server (job: %s) (err: %s)", r.conf.Job.ID, err)
 			}
 		}
diff --git a/agent/run_job.go b/agent/run_job.go
index da3bc41ac9..e28ea52aa6 100644
--- a/agent/run_job.go
+++ b/agent/run_job.go
@@ -514,13 +514,13 @@ func (r *JobRunner) streamJobLogsAfterProcessStart(ctx context.Context, wg *sync
 //
 // Cancel blocks until this process is complete.
 // The `agentStopping` arg mainly affects logged messages.
-func (r *JobRunner) Cancel(agentStopping bool) error {
+func (r *JobRunner) Cancel(reason CancelReason) error {
 	r.cancelLock.Lock()
 	defer r.cancelLock.Unlock()
 
 	// In case the user clicks "Cancel" in the UI while the agent happens to be
 	// stopping, only go from !stopping -> stopping.
-	r.agentStopping.Store(r.agentStopping.Load() || agentStopping)
+	r.agentStopping.Store(r.agentStopping.Load() || reason == CancelReasonAgentStopping)
 
 	// Return early if already cancelled.
 	if !r.cancelled.CompareAndSwap(false, true) {
@@ -532,13 +532,8 @@ func (r *JobRunner) Cancel(agentStopping bool) error {
 		return nil
 	}
 
-	reason := ""
-	if r.agentStopping.Load() {
-		reason = "(agent stopping)"
-	}
-
 	r.agentLogger.Info(
-		"Canceling job %s with a signal grace period of %v %s",
+		"Canceling job %s with a signal grace period of %v (%s)",
 		r.conf.Job.ID,
 		r.conf.AgentConfiguration.SignalGracePeriod,
 		reason,
@@ -572,3 +567,24 @@ func (r *JobRunner) Cancel(agentStopping bool) error {
 		return nil
 	}
 }
+
+// CancelReason captures the reason why Cancel is called.
+type CancelReason int
+
+const (
+	CancelReasonJobState CancelReason = iota
+	CancelReasonAgentStopping
+	CancelReasonInvalidToken
+)
+
+func (r CancelReason) String() string {
+	switch r {
+	case CancelReasonJobState:
+		return "job cancelled on Buildkite"
+	case CancelReasonAgentStopping:
+		return "agent is stopping"
+	case CancelReasonInvalidToken:
+		return "access token is invalid"
+	}
+	return "unknown"
+}
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 55403da0a7..c839414ce9 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1412,7 +1412,7 @@ func (ps *poolSignals) handleLoop(ctx context.Context, signals chan os.Signal) {
 		// We shouldn't block the signal handler loop either by waiting
 		// for the jobs to cancel or by waiting for the cancel grace
 		// period to expire.
-		go ps.pool.Stop(false /* ungraceful */) // one last chance to stop
+		go ps.pool.StopUngracefully() // one last chance to stop
 		go func() {
 			// Assuming cancelling jobs takes the full cancel grace period,
 			// allow 1 second to send agent disconnects.
@@ -1436,7 +1436,7 @@ func (ps *poolSignals) handleLoop(ctx context.Context, signals chan os.Signal) {
 			switch interruptCount {
 			case 1:
 				ps.log.Info("Received CTRL-C, send again to forcefully kill the agent(s)")
-				ps.pool.Stop(true /* graceful */)
+				ps.pool.StopGracefully()
 
 			case 2:
 				ps.log.Info("Forcefully stopping running jobs and stopping the agent(s) in %v", ps.cancelGracePeriod)

From 0b571493d4e4b754fba0b8169bf56e61e06d7142 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 22 Oct 2025 13:33:52 +1100
Subject: [PATCH 028/242] Fix client lost due to sleep

---
 clicommand/kubernetes_bootstrap.go | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/clicommand/kubernetes_bootstrap.go b/clicommand/kubernetes_bootstrap.go
index 7fc4cd9af3..4858cdb2b2 100644
--- a/clicommand/kubernetes_bootstrap.go
+++ b/clicommand/kubernetes_bootstrap.go
@@ -172,15 +172,19 @@ var KubernetesBootstrapCommand = cli.Command{
 			// Interrupt, waits for its signalGracePeriod, and then calls
 			// Terminate.
 			cancel()
-			// If we're cancelling because the job was cancelled in the UI, we
-			// should self-exit after cancelGracePeriod to be sure.
-			// (If we're cancelling because the pod is being deleted, Kubernetes
-			// enforces it after terminationGracePeriodSeconds, so self-exiting
-			// in that case is superfluous.)
-			time.Sleep(cancelGracePeriod)
-			// We get here if the main goroutine hasn't returned yet.
-			l.Info("kubernetes-bootstrap: Timed out waiting for subprocess to exit; exiting immediately with status 1")
-			os.Exit(1)
+			// If we block the StatusLoop goroutine, the client will be
+			// considered missing after a short while.
+			go func() {
+				// If we're cancelling because the job was cancelled in the UI, we
+				// should self-exit after cancelGracePeriod to be sure.
+				// (If we're cancelling because the pod is being deleted, Kubernetes
+				// enforces it after terminationGracePeriodSeconds, so self-exiting
+				// in that case is superfluous.)
+				time.Sleep(cancelGracePeriod)
+				// We get here if the main goroutine hasn't returned yet.
+				l.Info("kubernetes-bootstrap: Timed out waiting for subprocess to exit; exiting immediately with status 1")
+				os.Exit(1)
+			}()
 		}); err != nil {
 			return fmt.Errorf("connecting to k8s socket: %w", err)
 		}

From e0337a6f15aa8b4a0f10419d051538ccdf5ede84 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 22 Oct 2025 13:58:40 +1100
Subject: [PATCH 029/242] Bump version and changelog for v3.110.0

---
 CHANGELOG.md    | 38 ++++++++++++++++++++++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 40d7daa512..b41d8dd5a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,44 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.110.0](https://github.com/buildkite/agent/tree/v3.110.0) (2025-10-22)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.109.1...v3.110.0)
+
+### Added
+- Configurable chunks interval [#3521](https://github.com/buildkite/agent/pull/3521) (@catkins)
+- Inject OpenTelemetry context to all child processes [#3548](https://github.com/buildkite/agent/pull/3548) (@zhming0)
+  - This is done using [environment variables](https://opentelemetry.io/docs/specs/otel/context/env-carriers/). This may interfere with existing OTel environment variables if they are manually added some other way.
+- Add --literal and --delimiter flags to artifact upload [#3543](https://github.com/buildkite/agent/pull/3543) (@DrJosh9000)
+
+### Changed
+Various improvements and fixes to do with signal and cancel grace periods, and signal handling, most notably:
+- When cancelling a job, the timeout before sending a SIGKILL to the job has changed from cancel-grace-period to signal-grace-period (`--signal-grace-period-seconds` flag, `BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS` env var) to allow the agent some extra time to upload job logs and mark the job as finished. By default, signal-grace-period is 1 second shorter than cancel-grace-period. You may wish to increase cancel-grace-period accordingly.
+- When SIGQUIT is handled by the bootstrap, the exit code is now 131, and it no longer dumps a stacktrace.
+- The recently-added `--kubernetes-log-collection-grace-period` flag is now deprecated. Instead, use `--cancel-grace-period`.
+- When running the agent interactively, you can now Ctrl-C a third time to exit immediately.
+- In Kubernetes mode, the agent now begins shutting down on the first SIGTERM. The kubernetes-bootstrap now swallows SIGTERM with a logged message, and waits for the agent container to send an interrupt.
+- When the agent is cancelling jobs because it is stopping, all jobs start cancellation simultaneously. This allows the agent to exit sooner when multiple workers (`--spawn` flag) are used.
+See [#3549](https://github.com/buildkite/agent/pull/3549), [#3547](https://github.com/buildkite/agent/pull/3547), [#3534](https://github.com/buildkite/agent/pull/3534) (@DrJosh9000)
+
+### Fixed
+- Refresh checkout root file handle after checkout hook [#3546](https://github.com/buildkite/agent/pull/3546) (@zhming0)
+- Bump zzglob to v0.4.2 to fix uploading artifact paths containing `~` [#3539](https://github.com/buildkite/agent/pull/3539) (@DrJosh9000)
+
+### Internal
+- Docs: Add examples for step update commands for priority and notify attributes [#3532](https://github.com/buildkite/agent/pull/3532) (@tomowatt)
+- Docs: Update URLs in agent cfg comments [#3536](https://github.com/buildkite/agent/pull/3536) (@petetomasik)
+
+### Dependency updates
+- Upgrade Datadog-go to v5.8.1 to work around mod checksum issues [#3538](https://github.com/buildkite/agent/pull/3538) (@dannyfallon)
+- build(deps): bump the container-images group across 3 directories with 2 updates [#3545](https://github.com/buildkite/agent/pull/3545) (@dependabot[bot])
+- build(deps): bump gopkg.in/DataDog/dd-trace-go.v1 from 1.74.6 to 1.74.7 [#3544](https://github.com/buildkite/agent/pull/3544) (@dependabot[bot])
+- build(deps): bump github.com/gofrs/flock from 0.12.1 to 0.13.0 [#3523](https://github.com/buildkite/agent/pull/3523) (@dependabot[bot])
+- build(deps): bump docker/library/golang from 1.24.8 to 1.24.9 in /.buildkite in the container-images group across 1 directory [#3542](https://github.com/buildkite/agent/pull/3542) (@dependabot[bot])
+- build(deps): bump the cloud-providers group across 1 directory with 6 updates [#3541](https://github.com/buildkite/agent/pull/3541) (@dependabot[bot])
+- build(deps): bump the container-images group across 3 directories with 1 update [#3540](https://github.com/buildkite/agent/pull/3540) (@dependabot[bot])
+- build(deps): bump the golang-x group with 5 updates [#3525](https://github.com/buildkite/agent/pull/3525) (@dependabot[bot])
+
+
 ## [v3.109.1](https://github.com/buildkite/agent/tree/v3.109.1) (2025-10-15)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.109.0...v3.109.1)
 
diff --git a/version/VERSION b/version/VERSION
index 65a0f1cdbb..a23b9aae41 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.109.1
+3.110.0

From d3a22a3df5180becf3ea425d217378211adb0064 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Mon, 27 Oct 2025 10:57:54 +1100
Subject: [PATCH 030/242] Enforce that command descriptions indent using
 spaces, not tabs

---
 clicommand/config_completeness_test.go | 42 ++++++++++++++++++++++++++
 clicommand/step_cancel.go              |  2 +-
 clicommand/step_update.go              |  6 ++--
 3 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/clicommand/config_completeness_test.go b/clicommand/config_completeness_test.go
index 705e7b00f5..80c2d17a14 100644
--- a/clicommand/config_completeness_test.go
+++ b/clicommand/config_completeness_test.go
@@ -1,6 +1,7 @@
 package clicommand
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -93,6 +94,47 @@ func TestAllCommandConfigStructsHaveCorrespondingCLIFlags(t *testing.T) {
 	}
 }
 
+func TestDescriptionsAreIndentedUsingSpaces(t *testing.T) {
+	t.Parallel()
+
+	for name, command := range commandsByFullName(t, BuildkiteAgentCommands) {
+		if command.Description == "" {
+			t.Fatalf("command %q has no description; please add one", name)
+		}
+
+		lines := strings.Split(command.Description, "\n")
+		for i, line := range lines {
+			if strings.HasPrefix(line, "\t") {
+				fullCommandName := "buildkite-agent " + name
+				t.Errorf("line %d of description for command %q contains tab characters; please use spaces for indentation in command descriptions", i, fullCommandName)
+			}
+		}
+	}
+}
+
+// cli.Command.FullName() doesn't actually print the full name of a command when its a subcommand,
+// so we need to build a map of full command names to cli.Command structs ourselves
+func commandsByFullName(t *testing.T, commands []cli.Command) map[string]cli.Command {
+	t.Helper()
+
+	result := make(map[string]cli.Command)
+
+	for _, command := range commands {
+		if len(command.Subcommands) == 0 {
+			result[command.FullName()] = command
+		}
+
+		for _, subcommand := range command.Subcommands {
+			subcommands := commandsByFullName(t, []cli.Command{subcommand})
+			for subcommandName, cmd := range subcommands {
+				result[fmt.Sprintf("%s %s", command.FullName(), subcommandName)] = cmd
+			}
+		}
+	}
+
+	return result
+}
+
 func TestAllCommandsAreTestedForConfigCompleteness(t *testing.T) {
 	t.Parallel()
 
diff --git a/clicommand/step_cancel.go b/clicommand/step_cancel.go
index e097a95017..96e97bf0a5 100644
--- a/clicommand/step_cancel.go
+++ b/clicommand/step_cancel.go
@@ -25,7 +25,7 @@ Example:
     $ buildkite-agent step cancel --step "key"
     $ buildkite-agent step cancel --step "key" --force
     $ buildkite-agent step cancel --step "key" --force --force-grace-period-seconds 30
-		`
+`
 
 type StepCancelConfig struct {
 	GlobalConfig
diff --git a/clicommand/step_update.go b/clicommand/step_update.go
index 205a674dd5..e8693db203 100644
--- a/clicommand/step_update.go
+++ b/clicommand/step_update.go
@@ -38,9 +38,9 @@ Example:
     $ buildkite-agent step update "label" " (add to end of label)" --append
     $ buildkite-agent step update "label" < ./tmp/some-new-label
     $ ./script/label-generator | buildkite-agent step update "label"
-	$ buildkite-agent step update "priority" 10 --step "my-step-key"
-	$ buildkite-agent step update "notify" '[{"github_commit_status": {"context": "my-context"}}]' --append
-	$ buildkite-agent step update "notify" '[{"slack": "my-slack-workspace#my-channel"}]' --append
+    $ buildkite-agent step update "priority" 10 --step "my-step-key"
+    $ buildkite-agent step update "notify" '[{"github_commit_status": {"context": "my-context"}}]' --append
+    $ buildkite-agent step update "notify" '[{"slack": "my-slack-workspace#my-channel"}]' --append
 `
 
 type StepUpdateConfig struct {

From a98d7db09c9ab13dfe9122fc6c8486a8ac8f283d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 Oct 2025 15:51:18 +0000
Subject: [PATCH 031/242] build(deps): bump the cloud-providers group with 6
 updates

Bumps the cloud-providers group with 6 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.39.3` | `1.39.4` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.13` | `1.31.15` |
| [github.com/aws/aws-sdk-go-v2/feature/ec2/imds](https://github.com/aws/aws-sdk-go-v2) | `1.18.10` | `1.18.11` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.257.2` | `1.258.1` |
| [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) | `1.46.0` | `1.46.2` |
| [google.golang.org/api](https://github.com/googleapis/google-api-go-client) | `0.252.0` | `0.253.0` |


Updates `github.com/aws/aws-sdk-go-v2` from 1.39.3 to 1.39.4
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.39.3...v1.39.4)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.13 to 1.31.15
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.13...config/v1.31.15)

Updates `github.com/aws/aws-sdk-go-v2/feature/ec2/imds` from 1.18.10 to 1.18.11
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.10...config/v1.18.11)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.257.2 to 1.258.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.257.2...service/ec2/v1.258.1)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.46.0 to 1.46.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.46.0...service/rds/v1.46.2)

Updates `google.golang.org/api` from 0.252.0 to 0.253.0
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.252.0...v0.253.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2
  dependency-version: 1.39.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.31.15
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/ec2/imds
  dependency-version: 1.18.11
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.258.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.46.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: google.golang.org/api
  dependency-version: 0.253.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 32 ++++++++++++++---------------
 go.sum | 64 +++++++++++++++++++++++++++++-----------------------------
 2 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/go.mod b/go.mod
index 06e64d1c5d..38f10046b4 100644
--- a/go.mod
+++ b/go.mod
@@ -12,11 +12,11 @@ require (
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.39.3
-	github.com/aws/aws-sdk-go-v2/config v1.31.13
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2
-	github.com/aws/aws-sdk-go-v2/service/kms v1.46.0
+	github.com/aws/aws-sdk-go-v2 v1.39.4
+	github.com/aws/aws-sdk-go-v2/config v1.31.15
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1
+	github.com/aws/aws-sdk-go-v2/service/kms v1.46.2
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
 	github.com/buildkite/go-pipeline v0.16.0
@@ -57,7 +57,7 @@ require (
 	golang.org/x/sync v0.17.0
 	golang.org/x/sys v0.37.0
 	golang.org/x/term v0.36.0
-	google.golang.org/api v0.252.0
+	google.golang.org/api v0.253.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.7
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
@@ -90,15 +90,15 @@ require (
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.17 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect
 	github.com/aws/smithy-go v1.23.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
@@ -182,12 +182,12 @@ require (
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
 	golang.org/x/mod v0.28.0 // indirect
 	golang.org/x/text v0.30.0 // indirect
-	golang.org/x/time v0.13.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.37.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251002232023-7c0ddcbb5797 // indirect
-	google.golang.org/grpc v1.75.1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f // indirect
+	google.golang.org/grpc v1.76.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
diff --git a/go.sum b/go.sum
index f38f1279af..d9987f1918 100644
--- a/go.sum
+++ b/go.sum
@@ -78,34 +78,34 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.39.3 h1:h7xSsanJ4EQJXG5iuW4UqgP7qBopLpj84mpkNx3wPjM=
-github.com/aws/aws-sdk-go-v2 v1.39.3/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
-github.com/aws/aws-sdk-go-v2/config v1.31.13 h1:wcqQB3B0PgRPUF5ZE/QL1JVOyB0mbPevHFoAMpemR9k=
-github.com/aws/aws-sdk-go-v2/config v1.31.13/go.mod h1:ySB5D5ybwqGbT6c3GszZ+u+3KvrlYCUQNo62+hkKOFk=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.17 h1:skpEwzN/+H8cdrrtT8y+rvWJGiWWv0DeNAe+4VTf+Vs=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.17/go.mod h1:Ed+nXsaYa5uBINovJhcAWkALvXw2ZLk36opcuiSZfJM=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 h1:UuGVOX48oP4vgQ36oiKmW9RuSeT8jlgQgBFQD+HUiHY=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10/go.mod h1:vM/Ini41PzvudT4YkQyE/+WiQJiQ6jzeDyU8pQKwCac=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 h1:mj/bdWleWEh81DtpdHKkw41IrS+r3uw1J/VQtbwYYp8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10/go.mod h1:7+oEMxAZWP8gZCyjcm9VicI0M61Sx4DJtcGfKYv2yKQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 h1:wh+/mn57yhUrFtLIxyFPh2RgxgQz/u+Yrf7hiHGHqKY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10/go.mod h1:7zirD+ryp5gitJJ2m1BBux56ai8RIRDykXZrJSp540w=
+github.com/aws/aws-sdk-go-v2 v1.39.4 h1:qTsQKcdQPHnfGYBBs+Btl8QwxJeoWcOcPcixK90mRhg=
+github.com/aws/aws-sdk-go-v2 v1.39.4/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
+github.com/aws/aws-sdk-go-v2/config v1.31.15 h1:gE3M4xuNXfC/9bG4hyowGm/35uQTi7bUKeYs5e/6uvU=
+github.com/aws/aws-sdk-go-v2/config v1.31.15/go.mod h1:HvnvGJoE2I95KAIW8kkWVPJ4XhdrlvwJpV6pEzFQa8o=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.19 h1:Jc1zzwkSY1QbkEcLujwqRTXOdvW8ppND3jRBb/VhBQc=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.19/go.mod h1:DIfQ9fAk5H0pGtnqfqkbSIzky82qYnGvh06ASQXXg6A=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 h1:X7X4YKb+c0rkI6d4uJ5tEMxXgCZ+jZ/D6mvkno8c8Uw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11/go.mod h1:EqM6vPZQsZHYvC4Cai35UDg/f5NCEU+vp0WfbVqVcZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11/go.mod h1:7bUb2sSr2MZ3M/N+VyETLTQtInemHXb/Fl3s8CLzm0Y=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2 h1:D8MCemFa8rt09x7o6Fkm2T7ThVbRPrD91R+LKhVEnVU=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2/go.mod h1:Q/kZ++hvhasMpQU37I7daQh07ZqTa++isjj1aPi4zvM=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1 h1:D8cBaI1TsIF+cbB8qPmiZWsMqGsbs1/e7qYQ0NMDscY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1/go.mod h1:DT0XByGaNaOff3CtLVmj3jKcMeVDfOj5DkLD39UPJY0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 h1:DRND0dkCKtJzCj4Xl4OpVbXZgfttY5q712H9Zj7qc/0=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10/go.mod h1:tGGNmJKOTernmR2+VJ0fCzQRurcPZj9ut60Zu5Fi6us=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.0 h1:vSXYridw+tT3AHuK1PWdJto2qEc30/wG/fm8dmCHHis=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.0/go.mod h1:YXPskkMuiMgp6qUG96NSTl7UpideOQT/Kx0u9Y1MKn0=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 h1:fspVFg6qMx0svs40YgRmE7LZXh9VRZvTT35PfdQR6FM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.7/go.mod h1:BQTKL3uMECaLaUV3Zc2L4Qybv8C6BIXjuu1dOPyxTQs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 h1:scVnW+NLXasGOhy7HhkdT9AGb6kjgW7fJ5xYkUaqHs0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2/go.mod h1:FRNCY3zTEWZXBKm2h5UBUPvCVDOecTad9KhynDyGBc0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 h1:VEO5dqFkMsl8QZ2yHsFDJAIZLAkEbaYDB+xdKi0Feic=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.7/go.mod h1:L1xxV3zAdB+qVrVW/pBIrIAnHFWHo6FBbFe4xOGsG/o=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 h1:GpMf3z2KJa4RnJ0ew3Hac+hRFYLZ9DDjfgXjuW+pB54=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11/go.mod h1:6MZP3ZI4QQsgUCFTwMZA2V0sEriNQ8k2hmoHF3qjimQ=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.2 h1:hz2rJseQXnVQtVbByFpeSCNJBBU7oFN+yenW4biJtvs=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.2/go.mod h1:E4ink1KCQgqIe2pHFD9E+b5CNXovm50rQbWFuh0cM+I=
+github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.29.8/go.mod h1:mbef/pgKhtKRwrigPPs7SSSKZgytzP8PQ6P6JAAdqyM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 h1:S5GuJZpYxE0lKeMHKn+BRTz6PTFpgThyJ+5mYfux7BM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3/go.mod h1:X4OF+BTd7HIb3L+tc4UlWHVrpgwZZIVENU15pRDVTI0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 h1:Ekml5vGg6sHSZLZJQJagefnVe6PmqC2oiRkBq4F7fU0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.38.9/go.mod h1:/e15V+o1zFHWdH3u7lpI3rVBcxszktIKuHKCY2/py+k=
 github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
 github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE=
@@ -503,8 +503,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
 golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
-golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
-golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -520,18 +520,18 @@ golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSm
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.252.0 h1:xfKJeAJaMwb8OC9fesr369rjciQ704AjU/psjkKURSI=
-google.golang.org/api v0.252.0/go.mod h1:dnHOv81x5RAmumZ7BWLShB/u7JZNeyalImxHmtTHxqw=
+google.golang.org/api v0.253.0 h1:apU86Eq9Q2eQco3NsUYFpVTfy7DwemojL7LmbAj7g/I=
+google.golang.org/api v0.253.0/go.mod h1:PX09ad0r/4du83vZVAaGg7OaeyGnaUmT/CYPNvtLCbw=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
 google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
 google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251002232023-7c0ddcbb5797 h1:CirRxTOwnRWVLKzDNrs0CXAaVozJoR4G9xvdRecrdpk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251002232023-7c0ddcbb5797/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ=
-google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
-google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f h1:1FTH6cpXFsENbPR5Bu8NQddPSaUUE6NA2XdZdDSAJK4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
+google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/DataDog/dd-trace-go.v1 v1.74.7 h1:Vz7lua569eApE/1suZueI8ixJZqJCSbWavE9SuTDV9c=

From 33d243984db4eb47fb260fe1fd36d2cb5ba6a5cd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 Oct 2025 16:36:10 +0000
Subject: [PATCH 032/242] build(deps): bump the container-images group across 4
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `ce7c987` to `1df9b22`

Updates `buildkite/agent-base` from `fd70c60` to `3532569`

Updates `buildkite/agent-base` from `80f3117` to `7427ebc`

Updates `buildkite/agent-base` from `de28c3f` to `e60eef5`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index bad96a435a..991e5cea95 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:fd70c602f586e9613cce31ee8f3f090180a59340a0737d7c26cd40e3493fccbe
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:3532569cd996e8f3f8985896e15f8d360c27424c56bfad985b7d26652f2bc15f
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index dfa28216d5..9c0641d274 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:ce7c9875df8990a8e6422c48604d81b074feeefa96843724f0bff6d7d07f1f86
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:1df9b2216279b31581a50f160d5acd23f10bb38607b1a0725db4f506ec44beda
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index dc128ea7ad..7a5f84aa26 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:80f31170e2499fe3f31a34f8017b463711aa5fce0f14ff22ace94bb1e3953a53
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:7427ebca3c4ab49bc98614c336708ad3bc6340e43d12e382eafa0e5fdc07b56d
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index f46fff7c11..13bc04281e 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:de28c3f38091efbcf192cc04dbd4ab07e365b73c0b4d3fb40d661ca4b2a89e04
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:e60eef5258c55ce337789385c0fb8db8488cdcae75243d7a2f3c17b7a2d3d5f8
 ARG TARGETOS
 ARG TARGETARCH
 

From e929e1b0c6abf52c6993bf029df981c435a94463 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 Oct 2025 16:42:05 +0000
Subject: [PATCH 033/242] build(deps): bump docker/library/golang

Bumps the container-images group with 1 update in the /.buildkite directory: docker/library/golang.


Updates `docker/library/golang` from `02ce1d7` to `5034fa4`

---
updated-dependencies:
- dependency-name: docker/library/golang
  dependency-version: 1.24.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-compile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index 1848e5dc71..b9e0568fa5 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.9@sha256:02ce1d7ea7825dccb7cd10222e44e7c0565a08c5a38795e50fbf43936484507b
+FROM public.ecr.aws/docker/library/golang:1.24.9@sha256:5034fa44b36163a4109b71ed75c67dbdcb52c3cd9750953befe00054315d9fd2
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest

From e3bf723350e461f25fe00991a9f8684cbeda17ff Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 28 Oct 2025 00:18:15 +0000
Subject: [PATCH 034/242] build(deps): bump gopkg.in/DataDog/dd-trace-go.v1
 from 1.74.7 to 1.74.8

Bumps gopkg.in/DataDog/dd-trace-go.v1 from 1.74.7 to 1.74.8.

---
updated-dependencies:
- dependency-name: gopkg.in/DataDog/dd-trace-go.v1
  dependency-version: 1.74.8
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 38f10046b4..a5a4798f8a 100644
--- a/go.mod
+++ b/go.mod
@@ -58,7 +58,7 @@ require (
 	golang.org/x/sys v0.37.0
 	golang.org/x/term v0.36.0
 	google.golang.org/api v0.253.0
-	gopkg.in/DataDog/dd-trace-go.v1 v1.74.7
+	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
 )
diff --git a/go.sum b/go.sum
index d9987f1918..2978ce6730 100644
--- a/go.sum
+++ b/go.sum
@@ -534,8 +534,8 @@ google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
 google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
-gopkg.in/DataDog/dd-trace-go.v1 v1.74.7 h1:Vz7lua569eApE/1suZueI8ixJZqJCSbWavE9SuTDV9c=
-gopkg.in/DataDog/dd-trace-go.v1 v1.74.7/go.mod h1:yLnCS8nIkriRHfQW+YPDcNG+8ehXYNVio3NSEqy1Iqk=
+gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 h1:h96ji92t9eXbPvSWhJ+lrPWetHiQNYlt48JKRO09NFA=
+gopkg.in/DataDog/dd-trace-go.v1 v1.74.8/go.mod h1:LpHbtHsCZBlm1HWrlVOUQcEXwMWZnU6yMvmtd1GvSDI=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

From f73091de3eb8ca1f399472b6f0e3fddb34b378fb Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark@wolfe.id.au>
Date: Tue, 28 Oct 2025 19:57:14 +1100
Subject: [PATCH 035/242] chore: go modernize to do a bit of a tidy up and
 remove some junk

Ran `go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest -fix -test ./...`
---
 api/jobs.go                           | 2 +-
 clicommand/global.go                  | 2 +-
 cliconfig/loader.go                   | 6 +++---
 internal/job/docker.go                | 4 ++--
 internal/job/executor.go              | 4 ++--
 internal/job/git.go                   | 4 ++--
 internal/job/knownhosts.go            | 2 +-
 internal/replacer/big_lipsum_test.go  | 2 +-
 internal/replacer/bm_redactor_test.go | 4 ++--
 internal/shell/logger_test.go         | 4 ++--
 process/ansi_test.go                  | 4 ++--
 process/main_test.go                  | 2 +-
 process/process_test.go               | 2 +-
 process/scanner_test.go               | 2 +-
 14 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/api/jobs.go b/api/jobs.go
index d17840cca2..c30f799571 100644
--- a/api/jobs.go
+++ b/api/jobs.go
@@ -13,7 +13,7 @@ type Job struct {
 	Endpoint              string                     `json:"endpoint"`
 	State                 string                     `json:"state,omitempty"`
 	Env                   map[string]string          `json:"env,omitempty"`
-	Step                  pipeline.CommandStep       `json:"step,omitempty"`
+	Step                  pipeline.CommandStep       `json:"step"`
 	MatrixPermutation     pipeline.MatrixPermutation `json:"matrix_permutation,omitempty"`
 	ChunksMaxSizeBytes    uint64                     `json:"chunks_max_size_bytes,omitempty"`
 	ChunksIntervalSeconds int                        `json:"chunks_interval_seconds,omitempty"`
diff --git a/clicommand/global.go b/clicommand/global.go
index cfe8869091..c33231b421 100644
--- a/clicommand/global.go
+++ b/clicommand/global.go
@@ -324,7 +324,7 @@ func UnsetConfigFromEnvironment(c *cli.Context) error {
 		}
 		// split comma delimited env
 		if envVars := f.String(); envVars != "" {
-			for _, env := range strings.Split(envVars, ",") {
+			for env := range strings.SplitSeq(envVars, ",") {
 				os.Unsetenv(env)
 			}
 		}
diff --git a/cliconfig/loader.go b/cliconfig/loader.go
index 610b5c9c69..9ec4b2eeea 100644
--- a/cliconfig/loader.go
+++ b/cliconfig/loader.go
@@ -343,10 +343,10 @@ func (l Loader) fieldValueIsEmpty(fieldName string) bool {
 
 func (l Loader) validateField(fieldName string, label string, validationRules string) error {
 	// Split up the validation rules
-	rules := strings.Split(validationRules, ",")
+	rules := strings.SplitSeq(validationRules, ",")
 
 	// Loop through each rule, and perform it
-	for _, rule := range rules {
+	for rule := range rules {
 		switch rule {
 		case "required":
 			if l.fieldValueIsEmpty(fieldName) {
@@ -428,7 +428,7 @@ func (l Loader) normalizeField(fieldName string, normalization string) error {
 
 			for _, value := range valueAsSlice {
 				// Split values with commas into fields
-				for _, normalized := range strings.Split(value, ",") {
+				for normalized := range strings.SplitSeq(value, ",") {
 					if normalized == "" {
 						continue
 					}
diff --git a/internal/job/docker.go b/internal/job/docker.go
index f61dbeb86c..e9b47a6284 100644
--- a/internal/job/docker.go
+++ b/internal/job/docker.go
@@ -144,8 +144,8 @@ func runDockerCompose(ctx context.Context, sh *shell.Shell, projectName string,
 	}
 
 	// composeFile might be multiple files, spaces or colons
-	for _, chunk := range strings.Fields(composeFile) {
-		for _, file := range strings.Split(chunk, ":") {
+	for chunk := range strings.FieldsSeq(composeFile) {
+		for file := range strings.SplitSeq(chunk, ":") {
 			args = append(args, "-f", file)
 		}
 	}
diff --git a/internal/job/executor.go b/internal/job/executor.go
index 33b942525a..1dc77f2d1e 100644
--- a/internal/job/executor.go
+++ b/internal/job/executor.go
@@ -872,7 +872,7 @@ func (e *Executor) setUp(ctx context.Context) error {
 		e.shell.Commentf("Your pipeline environment has protected environment variables set. " +
 			"These can only be set via hooks, plugins or the agent configuration.")
 
-		for _, env := range strings.Split(ignored, ",") {
+		for env := range strings.SplitSeq(ignored, ",") {
 			e.shell.Warningf("Ignored %s", env)
 		}
 
@@ -1307,7 +1307,7 @@ func (e *Executor) writeBatchScript(cmd string) (string, error) {
 
 	scriptContents := []string{"@echo off"}
 
-	for _, line := range strings.Split(cmd, "\n") {
+	for line := range strings.SplitSeq(cmd, "\n") {
 		if line != "" {
 			if shouldCallBatchLine(line) {
 				scriptContents = append(scriptContents, "call "+line)
diff --git a/internal/job/git.go b/internal/job/git.go
index fe0a5ad7e3..9cbb365802 100644
--- a/internal/job/git.go
+++ b/internal/job/git.go
@@ -250,10 +250,10 @@ func gitEnumerateSubmoduleURLs(ctx context.Context, sh *shell.Shell) ([]string,
 	}
 
 	// splits lines on null-bytes to gracefully handle line endings and repositories with newlines
-	lines := strings.Split(strings.TrimRight(output, "\x00"), "\x00")
+	lines := strings.SplitSeq(strings.TrimRight(output, "\x00"), "\x00")
 
 	// process each line
-	for _, line := range lines {
+	for line := range lines {
 		tokens := strings.SplitN(line, "\n", 2)
 		if len(tokens) != 2 {
 			return nil, fmt.Errorf("Failed to parse .gitmodules line %q", line)
diff --git a/internal/job/knownhosts.go b/internal/job/knownhosts.go
index a12cde8c24..15e2adea6c 100644
--- a/internal/job/knownhosts.go
+++ b/internal/job/knownhosts.go
@@ -79,7 +79,7 @@ func (kh *knownHosts) Contains(host string) (bool, error) {
 		if len(fields) != 3 {
 			continue
 		}
-		for _, addr := range strings.Split(fields[0], ",") {
+		for addr := range strings.SplitSeq(fields[0], ",") {
 			if addr == normalized || addr == knownhosts.HashHostname(normalized) {
 				return true, nil
 			}
diff --git a/internal/replacer/big_lipsum_test.go b/internal/replacer/big_lipsum_test.go
index 9bc1c5c054..6e17ada4bc 100644
--- a/internal/replacer/big_lipsum_test.go
+++ b/internal/replacer/big_lipsum_test.go
@@ -21,7 +21,7 @@ var bigLipsumSecrets []string
 func init() {
 	// Find 10 most frequent words in bigLipsum to use as "secrets"
 	hist := make(map[string]int)
-	for _, w := range strings.Fields(bigLipsum) {
+	for w := range strings.FieldsSeq(bigLipsum) {
 		hist[strings.Trim(w, ".,")]++
 	}
 	words := make([]string, 0, len(hist))
diff --git a/internal/replacer/bm_redactor_test.go b/internal/replacer/bm_redactor_test.go
index 148c14ade7..f8e84ff76b 100644
--- a/internal/replacer/bm_redactor_test.go
+++ b/internal/replacer/bm_redactor_test.go
@@ -8,9 +8,9 @@ import (
 )
 
 func BenchmarkBMRedactor(b *testing.B) {
-	b.ResetTimer()
+
 	r := NewBMRedactor(io.Discard, "[REDACTED]", bigLipsumSecrets)
-	for range b.N {
+	for b.Loop() {
 		if _, err := fmt.Fprintln(r, bigLipsum); err != nil {
 			b.Errorf("fmt.Fprintln(r, bigLipsum) error = %v", err)
 		}
diff --git a/internal/shell/logger_test.go b/internal/shell/logger_test.go
index e21ea00897..fadbd07f13 100644
--- a/internal/shell/logger_test.go
+++ b/internal/shell/logger_test.go
@@ -124,7 +124,7 @@ func BenchmarkDoubleFmt(b *testing.B) {
 		fmt.Fprintf(io.Discard, "%s", fmt.Sprintf(format, v...))
 		fmt.Fprintln(io.Discard)
 	}
-	for range b.N {
+	for b.Loop() {
 		logf("asdfghjkl %s %d %t", "hi", 42, true)
 	}
 }
@@ -134,7 +134,7 @@ func BenchmarkFmtConcat(b *testing.B) {
 	logf := func(format string, v ...any) {
 		fmt.Fprintf(io.Discard, format+"\n", v...)
 	}
-	for range b.N {
+	for b.Loop() {
 		logf("asdfghjkl %s %d %t", "hi", 42, true)
 	}
 }
diff --git a/process/ansi_test.go b/process/ansi_test.go
index 4dc8729c44..5e3deea574 100644
--- a/process/ansi_test.go
+++ b/process/ansi_test.go
@@ -63,8 +63,8 @@ func BenchmarkANSIParser(b *testing.B) {
 	if err != nil {
 		b.Fatalf("os.ReadFile(fixtures/npm.sh.raw) error = %v", err)
 	}
-	b.ResetTimer()
-	for range b.N {
+
+	for b.Loop() {
 		var p ansiParser
 		p.Write(npm)
 	}
diff --git a/process/main_test.go b/process/main_test.go
index 4f6bc12c6d..3053dcf226 100644
--- a/process/main_test.go
+++ b/process/main_test.go
@@ -17,7 +17,7 @@ import (
 func TestMain(m *testing.M) {
 	switch os.Getenv("TEST_MAIN") {
 	case "tester":
-		for _, line := range strings.Split(strings.TrimSuffix(longTestOutput, "\n"), "\n") {
+		for line := range strings.SplitSeq(strings.TrimSuffix(longTestOutput, "\n"), "\n") {
 			fmt.Printf("%s\n", line)
 			time.Sleep(time.Millisecond * 20)
 		}
diff --git a/process/process_test.go b/process/process_test.go
index 8f665c49e6..ad48510700 100644
--- a/process/process_test.go
+++ b/process/process_test.go
@@ -407,7 +407,7 @@ func assertProcessDoesntExist(t *testing.T, p *process.Process) {
 }
 
 func BenchmarkProcess(b *testing.B) {
-	for range b.N {
+	for b.Loop() {
 		proc := process.New(logger.Discard, process.Config{
 			Path: os.Args[0],
 			Env:  []string{"TEST_MAIN=output"},
diff --git a/process/scanner_test.go b/process/scanner_test.go
index 74aebdf966..d3d92f2ab5 100644
--- a/process/scanner_test.go
+++ b/process/scanner_test.go
@@ -27,7 +27,7 @@ func TestScanLines(t *testing.T) {
 	pr, pw := io.Pipe()
 
 	go func() {
-		for _, line := range strings.Split(strings.TrimSuffix(longTestOutput, "\n"), "\n") {
+		for line := range strings.SplitSeq(strings.TrimSuffix(longTestOutput, "\n"), "\n") {
 			fmt.Fprintf(pw, "%s\n", line)
 			time.Sleep(time.Millisecond * 10)
 		}

From e88a98251f1d8798d77b5f0202eff70161a7b1ac Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Wed, 22 Oct 2025 16:55:30 +1100
Subject: [PATCH 036/242] Add cache save and restore using
 github.com/buildkite/zstash

This change introduces some new sub commands.

* buildkite-agent cache save
* buildkite-agent cache restore

It is currently using some conventions from zstash which will change as it is integrated more with the agent.
---
 clicommand/cache.go         |  24 +++++
 clicommand/cache_restore.go | 202 ++++++++++++++++++++++++++++++++++++
 clicommand/cache_save.go    | 180 ++++++++++++++++++++++++++++++++
 clicommand/commands.go      |   9 ++
 go.mod                      |  26 +++--
 go.sum                      | 103 +++++++++++++++---
 6 files changed, 522 insertions(+), 22 deletions(-)
 create mode 100644 clicommand/cache.go
 create mode 100644 clicommand/cache_restore.go
 create mode 100644 clicommand/cache_save.go

diff --git a/clicommand/cache.go b/clicommand/cache.go
new file mode 100644
index 0000000000..6c1640001f
--- /dev/null
+++ b/clicommand/cache.go
@@ -0,0 +1,24 @@
+package clicommand
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/buildkite/zstash/cache"
+	"github.com/stretchr/testify/assert/yaml"
+)
+
+// use yaml to load cache configuration from file
+func loadCacheConfiguration(cacheConfigFile string) ([]cache.Cache, error) {
+	data, err := os.ReadFile(cacheConfigFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read cache config file: %w", err)
+	}
+
+	var caches []cache.Cache
+	if err := yaml.Unmarshal(data, &caches); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal cache config file: %w", err)
+	}
+
+	return caches, nil
+}
diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
new file mode 100644
index 0000000000..5abfa0299b
--- /dev/null
+++ b/clicommand/cache_restore.go
@@ -0,0 +1,202 @@
+package clicommand
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/buildkite/agent/v3/version"
+	"github.com/buildkite/zstash"
+	"github.com/buildkite/zstash/api"
+	"github.com/dustin/go-humanize"
+	"github.com/urfave/cli"
+)
+
+const cacheRestoreHelpDescription = `Usage:
+
+    buildkite-agent cache restore [options]
+
+Description:
+
+Restores files from the cache for the current build based on the cache configuration
+defined in your cache config file (defaults to .buildkite/cache.yml).
+
+The cache configuration file defines which files or directories should be restored
+and their associated cache keys. Caches are scoped by organization, pipeline, and
+branch. If an exact cache match is not found, the command will attempt to use
+fallback keys if defined in your cache configuration.
+
+Example:
+
+    $ buildkite-agent cache restore
+
+This will restore all caches defined in .buildkite/cache.yml. You can also restore
+specific caches by providing their IDs:
+
+    $ buildkite-agent cache restore --ids "node"
+
+The cache will be retrieved from the bucket specified by --bucket-url or your
+cache configuration.
+
+Configuration File Format:
+
+The cache configuration file should be in YAML format:
+
+    - id: node
+      key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
+	  fallback_keys:
+		- '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
+      paths:
+        - node_modules
+
+Cache Restoration Results:
+
+The command will report one of three outcomes for each cache:
+  - Cache hit: Exact key match found and restored
+  - Fallback used: No exact match, but a fallback key was found and restored
+  - Cache miss: No matching cache found
+
+The command automatically uses the following environment variables when available:
+  - BUILDKITE_BRANCH (for branch scoping)
+  - BUILDKITE_PIPELINE_SLUG (for pipeline scoping)
+  - BUILDKITE_ORGANIZATION_SLUG (for organization scoping)`
+
+type CacheRestoreConfig struct {
+	GlobalConfig
+	APIConfig
+
+	Ids             string `cli:"ids"`
+	BucketURL       string `cli:"bucket-url"`
+	Branch          string `cli:"branch" validate:"required"`
+	Pipeline        string `cli:"pipeline" validate:"required"`
+	Organization    string `cli:"organization" validate:"required"`
+	CacheConfigFile string `cli:"cache-config-file"`
+}
+
+var CacheRestoreCommand = cli.Command{
+	Name:        "restore",
+	Usage:       "Restores files from the cache",
+	Description: cacheRestoreHelpDescription,
+	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
+		cli.StringFlag{
+			Name:   "ids",
+			Value:  "",
+			Usage:  "Comma-separated list of cache IDs to restore (if empty, restores all caches)",
+			EnvVar: "BUILDKITE_CACHE_IDS",
+		},
+		cli.StringFlag{
+			Name:   "bucket-url",
+			Value:  "",
+			Usage:  "The URL of the bucket to retrieve caches from (e.g., s3://bucket-name)",
+			EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
+		},
+		cli.StringFlag{
+			Name:   "branch",
+			Value:  "",
+			Usage:  "Which branch should the cache be associated with",
+			EnvVar: "BUILDKITE_BRANCH",
+		},
+		cli.StringFlag{
+			Name:   "pipeline",
+			Value:  "",
+			Usage:  "The pipeline slug for this cache",
+			EnvVar: "BUILDKITE_PIPELINE_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "organization",
+			Value:  "",
+			Usage:  "The organization slug for this cache",
+			EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "cache-config-file",
+			Value:  ".buildkite/cache.yml",
+			Usage:  "Path to the cache configuration YAML file",
+			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
+		},
+	}),
+
+	Action: func(c *cli.Context) error {
+		ctx := context.Background()
+		ctx, cfg, l, _, done := setupLoggerAndConfig[CacheRestoreConfig](ctx, c)
+		defer done()
+
+		l.Info("Cache restore command executed")
+
+		apiCfg := loadAPIClientConfig(cfg, "AgentAccessToken")
+
+		// we are using the zstash api package here which has a different client constructor but uses the same values
+		client := api.NewClient(ctx, version.Version(), apiCfg.Endpoint, apiCfg.Token)
+
+		caches, err := loadCacheConfiguration(cfg.CacheConfigFile)
+		if err != nil {
+			return fmt.Errorf("failed to load cache configuration: %w", err)
+		}
+
+		cacheClient, err := zstash.NewCache(zstash.Config{
+			Client:       client,
+			BucketURL:    cfg.BucketURL,
+			Format:       "zip",
+			Branch:       cfg.Branch,
+			Pipeline:     cfg.Pipeline,
+			Organization: cfg.Organization,
+			Caches:       caches,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to create cache client: %w", err)
+		}
+
+		// split the ids by comma
+		cacheIDs := strings.Split(cfg.Ids, ",")
+
+		if len(cacheIDs) == 0 {
+			// Save all caches configured in the client
+			caches := cacheClient.ListCaches()
+			for _, cache := range caches {
+				cacheIDs = append(cacheIDs, cache.ID)
+			}
+		}
+
+		for _, cacheID := range cacheIDs {
+			result, err := cacheClient.Restore(ctx, cacheID)
+			if err != nil {
+				return fmt.Errorf("failed to restore cache %q: %w", cacheID, err)
+			}
+
+			switch {
+			case result.CacheHit:
+				// Cache was found and restored
+				l.Info("Cache restored", map[string]interface{}{
+					"cache_id":          cacheID,
+					"cache_key":         result.Key,
+					"archive_size":      humanize.Bytes(uint64(result.Archive.Size)),
+					"written_bytes":     humanize.Bytes(uint64(result.Archive.WrittenBytes)),
+					"written_entries":   fmt.Sprintf("%d", result.Archive.WrittenEntries),
+					"compression_ratio": fmt.Sprintf("%.2f", result.Archive.CompressionRatio),
+					"transfer_speed":    fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed),
+				})
+			case result.FallbackUsed:
+				// Cache was not found, but a fallback was used
+				l.Info("Cache restored (fallback used)", map[string]interface{}{
+					"cache_id":          cacheID,
+					"cache_key":         result.Key,
+					"fallback_used":     result.FallbackUsed,
+					"archive_size":      humanize.Bytes(uint64(result.Archive.Size)),
+					"written_bytes":     humanize.Bytes(uint64(result.Archive.WrittenBytes)),
+					"written_entries":   fmt.Sprintf("%d", result.Archive.WrittenEntries),
+					"compression_ratio": fmt.Sprintf("%.2f", result.Archive.CompressionRatio),
+					"transfer_speed":    fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed),
+				})
+			default:
+				// Cache was not found
+				l.Info("Cache not restored (not found)", map[string]interface{}{
+					"cache_id":  cacheID,
+					"cache_key": result.Key,
+				})
+			}
+		}
+
+		return nil
+	},
+}
diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
new file mode 100644
index 0000000000..474390b51d
--- /dev/null
+++ b/clicommand/cache_save.go
@@ -0,0 +1,180 @@
+package clicommand
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/buildkite/agent/v3/version"
+	"github.com/buildkite/zstash"
+	"github.com/buildkite/zstash/api"
+	"github.com/dustin/go-humanize"
+	"github.com/urfave/cli"
+)
+
+const cacheSaveHelpDescription = `Usage:
+
+    buildkite-agent cache save [options]
+
+Description:
+
+Saves files to the cache for the current build based on the cache configuration
+defined in your cache config file (defaults to .buildkite/cache.yml).
+
+The cache configuration file defines which files or directories should be cached
+and their associated cache keys. Caches are scoped by organization, pipeline, and
+branch.
+
+Example:
+
+    $ buildkite-agent cache save
+
+This will save all caches defined in .buildkite/cache.yml. You can also save
+specific caches by providing their IDs:
+
+    $ buildkite-agent cache save --ids "node"
+
+The cache will be stored in the bucket specified by --bucket-url or your
+cache configuration. If a cache with the same key already exists, it will
+not be overwritten.
+
+Configuration File Format:
+
+The cache configuration file should be in YAML format:
+
+    - id: node
+      key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
+	  fallback_keys:
+		- '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
+      paths:
+        - node_modules
+
+The command automatically uses the following environment variables when available:
+  - BUILDKITE_BRANCH (for branch scoping)
+  - BUILDKITE_PIPELINE_SLUG (for pipeline scoping)
+  - BUILDKITE_ORGANIZATION_SLUG (for organization scoping)`
+
+type CacheSaveConfig struct {
+	GlobalConfig
+	APIConfig
+
+	Ids             string `cli:"ids"`
+	BucketURL       string `cli:"bucket-url"`
+	Branch          string `cli:"branch" validate:"required"`
+	Pipeline        string `cli:"pipeline" validate:"required"`
+	Organization    string `cli:"organization" validate:"required"`
+	CacheConfigFile string `cli:"cache-config-file"`
+}
+
+var CacheSaveCommand = cli.Command{
+	Name:        "save",
+	Usage:       "Saves files to the cache",
+	Description: cacheSaveHelpDescription,
+	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
+		cli.StringFlag{
+			Name:   "ids",
+			Value:  "",
+			Usage:  "Comma-separated list of cache IDs to save (if empty, saves all caches)",
+			EnvVar: "BUILDKITE_CACHE_IDS",
+		},
+		cli.StringFlag{
+			Name:   "bucket-url",
+			Value:  "",
+			Usage:  "The URL of the bucket to store caches (e.g., s3://bucket-name)",
+			EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
+		},
+		cli.StringFlag{
+			Name:   "branch",
+			Value:  "",
+			Usage:  "Which branch should the cache be associated with",
+			EnvVar: "BUILDKITE_BRANCH",
+		},
+		cli.StringFlag{
+			Name:   "pipeline",
+			Value:  "",
+			Usage:  "The pipeline slug for this cache",
+			EnvVar: "BUILDKITE_PIPELINE_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "organization",
+			Value:  "",
+			Usage:  "The organization slug for this cache",
+			EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "cache-config-file",
+			Value:  ".buildkite/cache.yml",
+			Usage:  "Path to the cache configuration YAML file",
+			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
+		},
+	}),
+	Action: func(c *cli.Context) error {
+		ctx := context.Background()
+		ctx, cfg, l, _, done := setupLoggerAndConfig[CacheSaveConfig](ctx, c)
+		defer done()
+
+		l.Info("Cache save command executed")
+
+		apiCfg := loadAPIClientConfig(cfg, "AgentAccessToken")
+
+		// we are using the zstash api package here which has a different client constructor but uses the same values
+		client := api.NewClient(ctx, version.Version(), apiCfg.Endpoint, apiCfg.Token)
+
+		caches, err := loadCacheConfiguration(cfg.CacheConfigFile)
+		if err != nil {
+			return fmt.Errorf("failed to load cache configuration: %w", err)
+		}
+
+		cacheClient, err := zstash.NewCache(zstash.Config{
+			Client:       client,
+			BucketURL:    cfg.BucketURL,
+			Format:       "zip",
+			Branch:       cfg.Branch,
+			Pipeline:     cfg.Pipeline,
+			Organization: cfg.Organization,
+			Caches:       caches,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to create cache client: %w", err)
+		}
+
+		// split the ids by comma
+		cacheIDs := strings.Split(cfg.Ids, ",")
+
+		if len(cacheIDs) == 0 {
+			// Save all caches configured in the client
+			caches := cacheClient.ListCaches()
+			for _, cache := range caches {
+				cacheIDs = append(cacheIDs, cache.ID)
+			}
+		}
+
+		for _, cacheID := range cacheIDs {
+			result, err := cacheClient.Save(ctx, cacheID)
+			if err != nil {
+				return fmt.Errorf("failed to save cache %q: %w", cacheID, err)
+			}
+
+			switch {
+			case result.CacheCreated:
+				l.Info("Cache created", map[string]interface{}{
+					"cache_id":          cacheID,
+					"cache_key":         result.Key,
+					"archive_size":      humanize.Bytes(uint64(result.Archive.Size)),
+					"written_bytes":     humanize.Bytes(uint64(result.Archive.WrittenBytes)),
+					"written_entries":   fmt.Sprintf("%d", result.Archive.WrittenEntries),
+					"compression_ratio": fmt.Sprintf("%.2f", result.Archive.CompressionRatio),
+					"transfer_speed":    fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed),
+				})
+			default:
+				l.Info("Cache already exists, not saving", map[string]interface{}{
+					"cache_id":  cacheID,
+					"cache_key": result.Key,
+				})
+			}
+		}
+
+		return nil
+	},
+}
diff --git a/clicommand/commands.go b/clicommand/commands.go
index 26cad785a3..d092d60b09 100644
--- a/clicommand/commands.go
+++ b/clicommand/commands.go
@@ -43,6 +43,15 @@ var BuildkiteAgentCommands = []cli.Command{
 			BuildCancelCommand,
 		},
 	},
+	{
+		Name:     "cache",
+		Category: categoryJobCommands,
+		Usage:    "Manage build caches",
+		Subcommands: []cli.Command{
+			CacheSaveCommand,
+			CacheRestoreCommand,
+		},
+	},
 	{
 		Name:     "env",
 		Category: categoryJobCommands,
diff --git a/go.mod b/go.mod
index a5a4798f8a..9b420d20b0 100644
--- a/go.mod
+++ b/go.mod
@@ -23,6 +23,7 @@ require (
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
+	github.com/buildkite/zstash v0.4.1-0.20251022041055-8abe44698ee9
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
@@ -90,12 +91,18 @@ require (
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.19 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect
@@ -124,13 +131,15 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/wire v0.7.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/klauspost/compress v1.18.1 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.3 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
@@ -154,7 +163,9 @@ require (
 	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/qri-io/jsonpointer v0.1.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/rs/zerolog v1.34.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shirou/gopsutil/v4 v4.25.8 // indirect
@@ -163,29 +174,32 @@ require (
 	github.com/tklauser/go-sysconf v0.3.15 // indirect
 	github.com/tklauser/numcpus v0.10.0 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.25 // indirect
+	github.com/wolfeidau/quickzip v1.0.2 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/collector/component v1.31.0 // indirect
 	go.opentelemetry.io/collector/featuregate v1.31.0 // indirect
 	go.opentelemetry.io/collector/internal/telemetry v0.125.0 // indirect
 	go.opentelemetry.io/collector/pdata v1.31.0 // indirect
 	go.opentelemetry.io/collector/semconv v0.125.0 // indirect
 	go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
 	go.opentelemetry.io/otel/log v0.11.0 // indirect
 	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.8.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
+	gocloud.dev v0.43.0 // indirect
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
 	golang.org/x/mod v0.28.0 // indirect
 	golang.org/x/text v0.30.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.37.0 // indirect
-	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f // indirect
 	google.golang.org/grpc v1.76.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
diff --git a/go.sum b/go.sum
index 2978ce6730..2cbe6daa4a 100644
--- a/go.sum
+++ b/go.sum
@@ -1,9 +1,19 @@
+cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
+cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
+cloud.google.com/go v0.121.4 h1:cVvUiY0sX0xwyxPwdSU2KsF9knOVmtRyAMt8xou0iTs=
+cloud.google.com/go v0.121.4/go.mod h1:XEBchUiHFJbz4lKBZwYBDHV/rSyfFktk737TLDU089s=
 cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
 cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
+cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8=
+cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE=
+cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
+cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
+cloud.google.com/go/storage v1.55.0 h1:NESjdAToN9u1tmhVqhXCaCwYBuvEhZLLv0gBr+2znf0=
+cloud.google.com/go/storage v1.55.0/go.mod h1:ztSmTTwzsdXe5syLVS0YsbFxXuvEmEyZj7v7zChEmuY=
 drjosh.dev/zzglob v0.4.2 h1:q+e5Cp6SFCyz+Yurhk/edSrTKEk3tn60vzoaXLmtiBo=
 drjosh.dev/zzglob v0.4.2/go.mod h1:SbYDdesQC13iyGiEwV8dJfJbyz7/Qiawrd5ODdJQCoo=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w=
@@ -57,6 +67,12 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0 h1:5US5S
 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0/go.mod h1:VRo4D6rj92AExpVBlq3Gcuol9Nm1bber12KyxRjKGWw=
 github.com/DataDog/sketches-go v1.4.7 h1:eHs5/0i2Sdf20Zkj0udVFWuCrXGRFig2Dcfm5rtcTxc=
 github.com/DataDog/sketches-go v1.4.7/go.mod h1:eAmQ/EBmtSO+nQp7IZMZVRPT4BQTmIc5RZQ+deGlTPM=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo=
 github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs=
 github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU=
 github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
@@ -80,26 +96,38 @@ github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
 github.com/aws/aws-sdk-go-v2 v1.39.4 h1:qTsQKcdQPHnfGYBBs+Btl8QwxJeoWcOcPcixK90mRhg=
 github.com/aws/aws-sdk-go-v2 v1.39.4/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
 github.com/aws/aws-sdk-go-v2/config v1.31.15 h1:gE3M4xuNXfC/9bG4hyowGm/35uQTi7bUKeYs5e/6uvU=
 github.com/aws/aws-sdk-go-v2/config v1.31.15/go.mod h1:HvnvGJoE2I95KAIW8kkWVPJ4XhdrlvwJpV6pEzFQa8o=
 github.com/aws/aws-sdk-go-v2/credentials v1.18.19 h1:Jc1zzwkSY1QbkEcLujwqRTXOdvW8ppND3jRBb/VhBQc=
 github.com/aws/aws-sdk-go-v2/credentials v1.18.19/go.mod h1:DIfQ9fAk5H0pGtnqfqkbSIzky82qYnGvh06ASQXXg6A=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 h1:X7X4YKb+c0rkI6d4uJ5tEMxXgCZ+jZ/D6mvkno8c8Uw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11/go.mod h1:EqM6vPZQsZHYvC4Cai35UDg/f5NCEU+vp0WfbVqVcZc=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 h1:cTXRdLkpBanlDwISl+5chq5ui1d1YWg4PWMR9c3kXyw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84/go.mod h1:kwSy5X7tfIHN39uucmjQVs2LvDdXEjQucgQQEqCggEo=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11/go.mod h1:7bUb2sSr2MZ3M/N+VyETLTQtInemHXb/Fl3s8CLzm0Y=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 h1:FHw90xCTsofzk6vjU808TSuDtDfOOKPNdz5Weyc3tUI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10/go.mod h1:n8jdIE/8F3UYkg8O4IGkQpn2qUmapg/1K1yl29/uf/c=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1 h1:D8cBaI1TsIF+cbB8qPmiZWsMqGsbs1/e7qYQ0NMDscY=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1/go.mod h1:DT0XByGaNaOff3CtLVmj3jKcMeVDfOj5DkLD39UPJY0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 h1:ne+eepnDB2Wh5lHKzELgEncIqeVlQ1rSF9fEa4r5I+A=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1/go.mod h1:u0Jkg0L+dcG1ozUq21uFElmpbmjBnhHR5DELHIme4wg=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 h1:GpMf3z2KJa4RnJ0ew3Hac+hRFYLZ9DDjfgXjuW+pB54=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11/go.mod h1:6MZP3ZI4QQsgUCFTwMZA2V0sEriNQ8k2hmoHF3qjimQ=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 h1:DA+Hl5adieRyFvE7pCvBWm3VOZTRexGVkXw33SUqNoY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10/go.mod h1:L+A89dH3/gr8L4ecrdzuXUYd1znoko6myzndVGZx/DA=
 github.com/aws/aws-sdk-go-v2/service/kms v1.46.2 h1:hz2rJseQXnVQtVbByFpeSCNJBBU7oFN+yenW4biJtvs=
 github.com/aws/aws-sdk-go-v2/service/kms v1.46.2/go.mod h1:E4ink1KCQgqIe2pHFD9E+b5CNXovm50rQbWFuh0cM+I=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5 h1:FlGScxzCGNzT+2AvHT1ZGMvxTwAMa6gsooFb1pO/AiM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5/go.mod h1:N/iojY+8bW3MYol9NUMuKimpSbPEur75cuI1SmtonFM=
 github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU=
 github.com/aws/aws-sdk-go-v2/service/sso v1.29.8/go.mod h1:mbef/pgKhtKRwrigPPs7SSSKZgytzP8PQ6P6JAAdqyM=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 h1:S5GuJZpYxE0lKeMHKn+BRTz6PTFpgThyJ+5mYfux7BM=
@@ -128,6 +156,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
+github.com/buildkite/zstash v0.4.1-0.20251022041055-8abe44698ee9 h1:A4xF9pSy+8z0nBa4FQ8Lw0pA9/l1Z7K/628EVq8mCTU=
+github.com/buildkite/zstash v0.4.1-0.20251022041055-8abe44698ee9/go.mod h1:PiQBhgsu0RgM8mOVupX6UJrNxIL9dlcHSYDqZEMhot8=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -135,6 +165,9 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
+github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls=
+github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.19 h1:tUN6H7LWqNx4hQVxomd0CVsDwaDr9gaRQaI4GpSmrsA=
@@ -161,6 +194,11 @@ github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S
 github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
 github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
 github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
+github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
+github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
+github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
+github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -173,6 +211,8 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
 github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
+github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI=
+github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -185,6 +225,7 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
 github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -201,9 +242,15 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo=
+github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
+github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
+github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
+github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
@@ -213,6 +260,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4=
+github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18=
 github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
 github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
 github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
@@ -221,8 +270,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU=
 github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
@@ -241,8 +290,8 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
+github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -263,8 +312,11 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB
 github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc=
 github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -320,8 +372,13 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 h1:8zyjtFyKi5NJySVOJRiHmSN1vl6qugQ5n9C4X7WyY3U=
+github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216/go.mod h1:hnzuad9d2wdd3z8fC6UouHQK5qZxqv3F/E6MMzXc7q0=
 github.com/secure-systems-lab/go-securesystemslib v0.9.0 h1:rf1HIbL64nUpEIZnjLZ3mcNEL9NBPB0iuVjyxvq3LZc=
 github.com/secure-systems-lab/go-securesystemslib v0.9.0/go.mod h1:DVHKMcZ+V4/woA/peqr+L0joiRXbPpQ042GgJckkFgw=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
@@ -340,6 +397,8 @@ github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
 github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
+github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -372,13 +431,17 @@ github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNl
 github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
 github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
 github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+github.com/wolfeidau/quickzip v1.0.2 h1:QPc4CVE8ECYng87o63C4t6+6ihZk1howyrZWwklKjq8=
+github.com/wolfeidau/quickzip v1.0.2/go.mod h1:ZDvxJMzI2iVp6CavOqxFq1tORhRyuxCDFw4HO25RbuA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
+github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/collector/component v1.31.0 h1:9LzU8X1RhV3h8/QsAoTX23aFUfoJ3EUc9O/vK+hFpSI=
 go.opentelemetry.io/collector/component v1.31.0/go.mod h1:JbZl/KywXJxpUXPbt96qlEXJSym1zQ2hauMxYMuvlxM=
 go.opentelemetry.io/collector/component/componentstatus v0.125.0 h1:zlxGQZYd9kknRZSjRpOYW5SBjl0a5zYFYRPbreobXoU=
@@ -415,8 +478,12 @@ go.opentelemetry.io/collector/semconv v0.125.0 h1:SyRP617YGvNSWRSKMy7Lbk9RaJSR+q
 go.opentelemetry.io/collector/semconv v0.125.0/go.mod h1:te6VQ4zZJO5Lp8dM2XIhDxDiL45mwX0YAQQWRQ0Qr9U=
 go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU=
 go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
+go.opentelemetry.io/contrib/detectors/gcp v1.37.0 h1:B+WbN9RPsvobe6q4vP6KgM8/9plR/HNjgGBrfcOlweA=
+go.opentelemetry.io/contrib/detectors/gcp v1.37.0/go.mod h1:K5zQ3TT7p2ru9Qkzk0bKtCql0RGkPj9pRjpXgZJZ+rU=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY=
 go.opentelemetry.io/contrib/propagators/aws v1.38.0 h1:eRZ7asSbLc5dH7+TBzL6hFKb1dabz0IV51uUUwYRZts=
 go.opentelemetry.io/contrib/propagators/aws v1.38.0/go.mod h1:wXqc9NTGcXapBExHBDVLEZlByu6quiQL8w7Tjgv8TCg=
 go.opentelemetry.io/contrib/propagators/b3 v1.38.0 h1:uHsCCOSKl0kLrV2dLkFK+8Ywk9iKa/fptkytc6aFFEo=
@@ -443,8 +510,8 @@ go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6
 go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
 go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
 go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
-go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
-go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
+go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE=
+go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
@@ -454,6 +521,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+gocloud.dev v0.43.0 h1:aW3eq4RMyehbJ54PMsh4hsp7iX8cO/98ZRzJJOzN/5M=
+gocloud.dev v0.43.0/go.mod h1:eD8rkg7LhKUHrzkEdLTZ+Ty/vgPHPCd+yMQdfelQVu4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -492,8 +561,10 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
 golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -516,18 +587,18 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
 google.golang.org/api v0.253.0 h1:apU86Eq9Q2eQco3NsUYFpVTfy7DwemojL7LmbAj7g/I=
 google.golang.org/api v0.253.0/go.mod h1:PX09ad0r/4du83vZVAaGg7OaeyGnaUmT/CYPNvtLCbw=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
-google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
-google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
-google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
-google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
+google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 h1:Nt6z9UHqSlIdIGJdz6KhTIs2VRx/iOsA5iE8bmQNcxs=
+google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79/go.mod h1:kTmlBHMPqR5uCZPBvwa2B18mvubkjyY3CRLI0c6fj0s=
+google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 h1:8XJ4pajGwOlasW+L13MnEGA8W4115jJySQtVfS2/IBU=
+google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4/go.mod h1:NnuHhy+bxcg30o7FnVAZbXsPHUDQ9qKWAQKCD7VxFtk=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f h1:1FTH6cpXFsENbPR5Bu8NQddPSaUUE6NA2XdZdDSAJK4=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
 google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=

From 706e71581ebf9328deea230dc473e634b5b22a80 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 23 Oct 2025 09:49:02 +1100
Subject: [PATCH 037/242] fix: update the zstash library to latest main

---
 go.mod |  6 +-----
 go.sum | 53 ++++-------------------------------------------------
 2 files changed, 5 insertions(+), 54 deletions(-)

diff --git a/go.mod b/go.mod
index 9b420d20b0..8432d0de3e 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
-	github.com/buildkite/zstash v0.4.1-0.20251022041055-8abe44698ee9
+	github.com/buildkite/zstash v0.4.1-0.20251022224615-e5fb6fcef4ee
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
@@ -93,7 +93,6 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.19 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -131,7 +130,6 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/wire v0.7.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
@@ -187,12 +185,10 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
 	go.opentelemetry.io/otel/log v0.11.0 // indirect
 	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.8.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	gocloud.dev v0.43.0 // indirect
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
 	golang.org/x/mod v0.28.0 // indirect
 	golang.org/x/text v0.30.0 // indirect
diff --git a/go.sum b/go.sum
index 2cbe6daa4a..0841378969 100644
--- a/go.sum
+++ b/go.sum
@@ -1,19 +1,9 @@
-cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
-cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
-cloud.google.com/go v0.121.4 h1:cVvUiY0sX0xwyxPwdSU2KsF9knOVmtRyAMt8xou0iTs=
-cloud.google.com/go v0.121.4/go.mod h1:XEBchUiHFJbz4lKBZwYBDHV/rSyfFktk737TLDU089s=
 cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
 cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
-cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8=
-cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE=
-cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
-cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
-cloud.google.com/go/storage v1.55.0 h1:NESjdAToN9u1tmhVqhXCaCwYBuvEhZLLv0gBr+2znf0=
-cloud.google.com/go/storage v1.55.0/go.mod h1:ztSmTTwzsdXe5syLVS0YsbFxXuvEmEyZj7v7zChEmuY=
 drjosh.dev/zzglob v0.4.2 h1:q+e5Cp6SFCyz+Yurhk/edSrTKEk3tn60vzoaXLmtiBo=
 drjosh.dev/zzglob v0.4.2/go.mod h1:SbYDdesQC13iyGiEwV8dJfJbyz7/Qiawrd5ODdJQCoo=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w=
@@ -67,12 +57,6 @@ github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0 h1:5US5S
 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0/go.mod h1:VRo4D6rj92AExpVBlq3Gcuol9Nm1bber12KyxRjKGWw=
 github.com/DataDog/sketches-go v1.4.7 h1:eHs5/0i2Sdf20Zkj0udVFWuCrXGRFig2Dcfm5rtcTxc=
 github.com/DataDog/sketches-go v1.4.7/go.mod h1:eAmQ/EBmtSO+nQp7IZMZVRPT4BQTmIc5RZQ+deGlTPM=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo=
 github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs=
 github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU=
 github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
@@ -104,8 +88,6 @@ github.com/aws/aws-sdk-go-v2/credentials v1.18.19 h1:Jc1zzwkSY1QbkEcLujwqRTXOdvW
 github.com/aws/aws-sdk-go-v2/credentials v1.18.19/go.mod h1:DIfQ9fAk5H0pGtnqfqkbSIzky82qYnGvh06ASQXXg6A=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 h1:X7X4YKb+c0rkI6d4uJ5tEMxXgCZ+jZ/D6mvkno8c8Uw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11/go.mod h1:EqM6vPZQsZHYvC4Cai35UDg/f5NCEU+vp0WfbVqVcZc=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 h1:cTXRdLkpBanlDwISl+5chq5ui1d1YWg4PWMR9c3kXyw=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84/go.mod h1:kwSy5X7tfIHN39uucmjQVs2LvDdXEjQucgQQEqCggEo=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE=
@@ -156,8 +138,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
-github.com/buildkite/zstash v0.4.1-0.20251022041055-8abe44698ee9 h1:A4xF9pSy+8z0nBa4FQ8Lw0pA9/l1Z7K/628EVq8mCTU=
-github.com/buildkite/zstash v0.4.1-0.20251022041055-8abe44698ee9/go.mod h1:PiQBhgsu0RgM8mOVupX6UJrNxIL9dlcHSYDqZEMhot8=
+github.com/buildkite/zstash v0.4.1-0.20251022224615-e5fb6fcef4ee h1:L/kSSvQ+TAx9bVV0P/pU6qX5A2/n3uMYBFUSMSsx6fg=
+github.com/buildkite/zstash v0.4.1-0.20251022224615-e5fb6fcef4ee/go.mod h1:uV3f+1NmQyH1xgTJB3o2emmKqUhowkrYOaTKCskzKgw=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -165,8 +147,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
-github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls=
-github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -194,11 +174,6 @@ github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S
 github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
 github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
 github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
-github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
-github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
-github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
-github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
-github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -211,8 +186,6 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
 github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
-github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI=
-github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -242,15 +215,9 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
-github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo=
-github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
-github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
-github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
-github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
@@ -260,8 +227,6 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4=
-github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18=
 github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
 github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
 github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
@@ -397,8 +362,6 @@ github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
 github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
-github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -438,8 +401,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
-github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/collector/component v1.31.0 h1:9LzU8X1RhV3h8/QsAoTX23aFUfoJ3EUc9O/vK+hFpSI=
@@ -478,10 +439,6 @@ go.opentelemetry.io/collector/semconv v0.125.0 h1:SyRP617YGvNSWRSKMy7Lbk9RaJSR+q
 go.opentelemetry.io/collector/semconv v0.125.0/go.mod h1:te6VQ4zZJO5Lp8dM2XIhDxDiL45mwX0YAQQWRQ0Qr9U=
 go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU=
 go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0=
-go.opentelemetry.io/contrib/detectors/gcp v1.37.0 h1:B+WbN9RPsvobe6q4vP6KgM8/9plR/HNjgGBrfcOlweA=
-go.opentelemetry.io/contrib/detectors/gcp v1.37.0/go.mod h1:K5zQ3TT7p2ru9Qkzk0bKtCql0RGkPj9pRjpXgZJZ+rU=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY=
 go.opentelemetry.io/contrib/propagators/aws v1.38.0 h1:eRZ7asSbLc5dH7+TBzL6hFKb1dabz0IV51uUUwYRZts=
@@ -521,8 +478,6 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-gocloud.dev v0.43.0 h1:aW3eq4RMyehbJ54PMsh4hsp7iX8cO/98ZRzJJOzN/5M=
-gocloud.dev v0.43.0/go.mod h1:eD8rkg7LhKUHrzkEdLTZ+Ty/vgPHPCd+yMQdfelQVu4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -595,8 +550,8 @@ google.golang.org/api v0.253.0 h1:apU86Eq9Q2eQco3NsUYFpVTfy7DwemojL7LmbAj7g/I=
 google.golang.org/api v0.253.0/go.mod h1:PX09ad0r/4du83vZVAaGg7OaeyGnaUmT/CYPNvtLCbw=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
-google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 h1:Nt6z9UHqSlIdIGJdz6KhTIs2VRx/iOsA5iE8bmQNcxs=
-google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79/go.mod h1:kTmlBHMPqR5uCZPBvwa2B18mvubkjyY3CRLI0c6fj0s=
+google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
+google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
 google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 h1:8XJ4pajGwOlasW+L13MnEGA8W4115jJySQtVfS2/IBU=
 google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4/go.mod h1:NnuHhy+bxcg30o7FnVAZbXsPHUDQ9qKWAQKCD7VxFtk=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f h1:1FTH6cpXFsENbPR5Bu8NQddPSaUUE6NA2XdZdDSAJK4=

From 4de343edf9f4d2b309885a9f5b78b6c92304848e Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 23 Oct 2025 10:00:14 +1100
Subject: [PATCH 038/242] fix: ensure we capture registry slug so we can
 provide it later

---
 clicommand/cache_restore.go | 7 +++++++
 clicommand/cache_save.go    | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
index 5abfa0299b..ae1b98ce3c 100644
--- a/clicommand/cache_restore.go
+++ b/clicommand/cache_restore.go
@@ -67,6 +67,7 @@ type CacheRestoreConfig struct {
 	APIConfig
 
 	Ids             string `cli:"ids"`
+	Registry        string `cli:"registry"`
 	BucketURL       string `cli:"bucket-url"`
 	Branch          string `cli:"branch" validate:"required"`
 	Pipeline        string `cli:"pipeline" validate:"required"`
@@ -85,6 +86,12 @@ var CacheRestoreCommand = cli.Command{
 			Usage:  "Comma-separated list of cache IDs to restore (if empty, restores all caches)",
 			EnvVar: "BUILDKITE_CACHE_IDS",
 		},
+		cli.StringFlag{
+			Name:   "registry",
+			Value:  "~",
+			Usage:  "The slug of the cache registry to use, defaults to the default registry (~)",
+			EnvVar: "BUILDKITE_CACHE_REGISTRY",
+		},
 		cli.StringFlag{
 			Name:   "bucket-url",
 			Value:  "",
diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
index 474390b51d..46fad588d8 100644
--- a/clicommand/cache_save.go
+++ b/clicommand/cache_save.go
@@ -60,6 +60,7 @@ type CacheSaveConfig struct {
 	APIConfig
 
 	Ids             string `cli:"ids"`
+	Registry        string `cli:"registry"`
 	BucketURL       string `cli:"bucket-url"`
 	Branch          string `cli:"branch" validate:"required"`
 	Pipeline        string `cli:"pipeline" validate:"required"`
@@ -78,6 +79,12 @@ var CacheSaveCommand = cli.Command{
 			Usage:  "Comma-separated list of cache IDs to save (if empty, saves all caches)",
 			EnvVar: "BUILDKITE_CACHE_IDS",
 		},
+		cli.StringFlag{
+			Name:   "registry",
+			Value:  "~",
+			Usage:  "The slug of the cache registry to use, defaults to the default registry (~)",
+			EnvVar: "BUILDKITE_CACHE_REGISTRY",
+		},
 		cli.StringFlag{
 			Name:   "bucket-url",
 			Value:  "",

From 70aa3cc75c004c218cfc19067ede999d85a1da5a Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 23 Oct 2025 10:08:35 +1100
Subject: [PATCH 039/242] refactor: centralised flags and fixed config test

---
 clicommand/cache_restore.go            | 55 +-----------------------
 clicommand/cache_save.go               | 54 +----------------------
 clicommand/config_completeness_test.go |  2 +
 clicommand/global.go                   | 59 ++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 105 deletions(-)

diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
index ae1b98ce3c..3278fb888f 100644
--- a/clicommand/cache_restore.go
+++ b/clicommand/cache_restore.go
@@ -65,65 +65,14 @@ The command automatically uses the following environment variables when availabl
 type CacheRestoreConfig struct {
 	GlobalConfig
 	APIConfig
-
-	Ids             string `cli:"ids"`
-	Registry        string `cli:"registry"`
-	BucketURL       string `cli:"bucket-url"`
-	Branch          string `cli:"branch" validate:"required"`
-	Pipeline        string `cli:"pipeline" validate:"required"`
-	Organization    string `cli:"organization" validate:"required"`
-	CacheConfigFile string `cli:"cache-config-file"`
+	CacheConfig
 }
 
 var CacheRestoreCommand = cli.Command{
 	Name:        "restore",
 	Usage:       "Restores files from the cache",
 	Description: cacheRestoreHelpDescription,
-	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
-		cli.StringFlag{
-			Name:   "ids",
-			Value:  "",
-			Usage:  "Comma-separated list of cache IDs to restore (if empty, restores all caches)",
-			EnvVar: "BUILDKITE_CACHE_IDS",
-		},
-		cli.StringFlag{
-			Name:   "registry",
-			Value:  "~",
-			Usage:  "The slug of the cache registry to use, defaults to the default registry (~)",
-			EnvVar: "BUILDKITE_CACHE_REGISTRY",
-		},
-		cli.StringFlag{
-			Name:   "bucket-url",
-			Value:  "",
-			Usage:  "The URL of the bucket to retrieve caches from (e.g., s3://bucket-name)",
-			EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
-		},
-		cli.StringFlag{
-			Name:   "branch",
-			Value:  "",
-			Usage:  "Which branch should the cache be associated with",
-			EnvVar: "BUILDKITE_BRANCH",
-		},
-		cli.StringFlag{
-			Name:   "pipeline",
-			Value:  "",
-			Usage:  "The pipeline slug for this cache",
-			EnvVar: "BUILDKITE_PIPELINE_SLUG",
-		},
-		cli.StringFlag{
-			Name:   "organization",
-			Value:  "",
-			Usage:  "The organization slug for this cache",
-			EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
-		},
-		cli.StringFlag{
-			Name:   "cache-config-file",
-			Value:  ".buildkite/cache.yml",
-			Usage:  "Path to the cache configuration YAML file",
-			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
-		},
-	}),
-
+	Flags:       slices.Concat(globalFlags(), apiFlags(), cacheFlags()),
 	Action: func(c *cli.Context) error {
 		ctx := context.Background()
 		ctx, cfg, l, _, done := setupLoggerAndConfig[CacheRestoreConfig](ctx, c)
diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
index 46fad588d8..3f6d033e5a 100644
--- a/clicommand/cache_save.go
+++ b/clicommand/cache_save.go
@@ -58,64 +58,14 @@ The command automatically uses the following environment variables when availabl
 type CacheSaveConfig struct {
 	GlobalConfig
 	APIConfig
-
-	Ids             string `cli:"ids"`
-	Registry        string `cli:"registry"`
-	BucketURL       string `cli:"bucket-url"`
-	Branch          string `cli:"branch" validate:"required"`
-	Pipeline        string `cli:"pipeline" validate:"required"`
-	Organization    string `cli:"organization" validate:"required"`
-	CacheConfigFile string `cli:"cache-config-file"`
+	CacheConfig
 }
 
 var CacheSaveCommand = cli.Command{
 	Name:        "save",
 	Usage:       "Saves files to the cache",
 	Description: cacheSaveHelpDescription,
-	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
-		cli.StringFlag{
-			Name:   "ids",
-			Value:  "",
-			Usage:  "Comma-separated list of cache IDs to save (if empty, saves all caches)",
-			EnvVar: "BUILDKITE_CACHE_IDS",
-		},
-		cli.StringFlag{
-			Name:   "registry",
-			Value:  "~",
-			Usage:  "The slug of the cache registry to use, defaults to the default registry (~)",
-			EnvVar: "BUILDKITE_CACHE_REGISTRY",
-		},
-		cli.StringFlag{
-			Name:   "bucket-url",
-			Value:  "",
-			Usage:  "The URL of the bucket to store caches (e.g., s3://bucket-name)",
-			EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
-		},
-		cli.StringFlag{
-			Name:   "branch",
-			Value:  "",
-			Usage:  "Which branch should the cache be associated with",
-			EnvVar: "BUILDKITE_BRANCH",
-		},
-		cli.StringFlag{
-			Name:   "pipeline",
-			Value:  "",
-			Usage:  "The pipeline slug for this cache",
-			EnvVar: "BUILDKITE_PIPELINE_SLUG",
-		},
-		cli.StringFlag{
-			Name:   "organization",
-			Value:  "",
-			Usage:  "The organization slug for this cache",
-			EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
-		},
-		cli.StringFlag{
-			Name:   "cache-config-file",
-			Value:  ".buildkite/cache.yml",
-			Usage:  "Path to the cache configuration YAML file",
-			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
-		},
-	}),
+	Flags:       slices.Concat(globalFlags(), apiFlags(), cacheFlags()),
 	Action: func(c *cli.Context) error {
 		ctx := context.Background()
 		ctx, cfg, l, _, done := setupLoggerAndConfig[CacheSaveConfig](ctx, c)
diff --git a/clicommand/config_completeness_test.go b/clicommand/config_completeness_test.go
index 80c2d17a14..238cbb82c9 100644
--- a/clicommand/config_completeness_test.go
+++ b/clicommand/config_completeness_test.go
@@ -28,6 +28,8 @@ var commandConfigPairs = []configCommandPair{
 	{Config: ArtifactUploadConfig{}, Command: ArtifactUploadCommand},
 	{Config: BuildCancelConfig{}, Command: BuildCancelCommand},
 	{Config: BootstrapConfig{}, Command: BootstrapCommand},
+	{Config: CacheRestoreConfig{}, Command: CacheRestoreCommand},
+	{Config: CacheSaveConfig{}, Command: CacheSaveCommand},
 	{Config: EnvDumpConfig{}, Command: EnvDumpCommand},
 	{Config: EnvGetConfig{}, Command: EnvGetCommand},
 	{Config: EnvSetConfig{}, Command: EnvSetCommand},
diff --git a/clicommand/global.go b/clicommand/global.go
index c33231b421..eec4058b47 100644
--- a/clicommand/global.go
+++ b/clicommand/global.go
@@ -182,6 +182,18 @@ type APIConfig struct {
 	NoHTTP2          bool   `cli:"no-http2"`
 }
 
+// CacheConfig includes cache-related shared options for easy inclusion across
+// cache command config structs (via embedding).
+type CacheConfig struct {
+	Ids             string `cli:"ids"`
+	Registry        string `cli:"registry"`
+	BucketURL       string `cli:"bucket-url"`
+	Branch          string `cli:"branch" validate:"required"`
+	Pipeline        string `cli:"pipeline" validate:"required"`
+	Organization    string `cli:"organization" validate:"required"`
+	CacheConfigFile string `cli:"cache-config-file"`
+}
+
 func globalFlags() []cli.Flag {
 	return []cli.Flag{
 		NoColorFlag,
@@ -202,6 +214,53 @@ func apiFlags() []cli.Flag {
 	}
 }
 
+func cacheFlags() []cli.Flag {
+	return []cli.Flag{
+		cli.StringFlag{
+			Name:   "ids",
+			Value:  "",
+			Usage:  "Comma-separated list of cache IDs (if empty, processes all caches)",
+			EnvVar: "BUILDKITE_CACHE_IDS",
+		},
+		cli.StringFlag{
+			Name:   "registry",
+			Value:  "~",
+			Usage:  "The slug of the cache registry to use, defaults to the default registry (~)",
+			EnvVar: "BUILDKITE_CACHE_REGISTRY",
+		},
+		cli.StringFlag{
+			Name:   "bucket-url",
+			Value:  "",
+			Usage:  "The URL of the bucket (e.g., s3://bucket-name)",
+			EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
+		},
+		cli.StringFlag{
+			Name:   "branch",
+			Value:  "",
+			Usage:  "Which branch should the cache be associated with",
+			EnvVar: "BUILDKITE_BRANCH",
+		},
+		cli.StringFlag{
+			Name:   "pipeline",
+			Value:  "",
+			Usage:  "The pipeline slug for this cache",
+			EnvVar: "BUILDKITE_PIPELINE_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "organization",
+			Value:  "",
+			Usage:  "The organization slug for this cache",
+			EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "cache-config-file",
+			Value:  ".buildkite/cache.yml",
+			Usage:  "Path to the cache configuration YAML file",
+			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
+		},
+	}
+}
+
 func CreateLogger(cfg any) logger.Logger {
 	var l logger.Logger
 	logFormat := "text"

From 22e9124e8de7e76742c6623afc433cd8f9ed0fe5 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 23 Oct 2025 13:00:09 +1100
Subject: [PATCH 040/242] fix: adjust logging and fixes for testing

---
 clicommand/cache_restore.go | 49 +++++++++++++++++--------------------
 clicommand/cache_save.go    | 40 ++++++++++++++++++------------
 clicommand/global.go        |  2 +-
 3 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
index 3278fb888f..2b327618a4 100644
--- a/clicommand/cache_restore.go
+++ b/clicommand/cache_restore.go
@@ -6,6 +6,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/version"
 	"github.com/buildkite/zstash"
 	"github.com/buildkite/zstash/api"
@@ -90,6 +91,11 @@ var CacheRestoreCommand = cli.Command{
 			return fmt.Errorf("failed to load cache configuration: %w", err)
 		}
 
+		if len(caches) == 0 {
+			l.Info("No caches defined in the cache configuration file, nothing to save")
+			return nil
+		}
+
 		cacheClient, err := zstash.NewCache(zstash.Config{
 			Client:       client,
 			BucketURL:    cfg.BucketURL,
@@ -115,41 +121,30 @@ var CacheRestoreCommand = cli.Command{
 		}
 
 		for _, cacheID := range cacheIDs {
+			l.Info("Restoring cache: %s", cacheID)
 			result, err := cacheClient.Restore(ctx, cacheID)
 			if err != nil {
 				return fmt.Errorf("failed to restore cache %q: %w", cacheID, err)
 			}
 
 			switch {
-			case result.CacheHit:
-				// Cache was found and restored
-				l.Info("Cache restored", map[string]interface{}{
-					"cache_id":          cacheID,
-					"cache_key":         result.Key,
-					"archive_size":      humanize.Bytes(uint64(result.Archive.Size)),
-					"written_bytes":     humanize.Bytes(uint64(result.Archive.WrittenBytes)),
-					"written_entries":   fmt.Sprintf("%d", result.Archive.WrittenEntries),
-					"compression_ratio": fmt.Sprintf("%.2f", result.Archive.CompressionRatio),
-					"transfer_speed":    fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed),
-				})
-			case result.FallbackUsed:
-				// Cache was not found, but a fallback was used
-				l.Info("Cache restored (fallback used)", map[string]interface{}{
-					"cache_id":          cacheID,
-					"cache_key":         result.Key,
-					"fallback_used":     result.FallbackUsed,
-					"archive_size":      humanize.Bytes(uint64(result.Archive.Size)),
-					"written_bytes":     humanize.Bytes(uint64(result.Archive.WrittenBytes)),
-					"written_entries":   fmt.Sprintf("%d", result.Archive.WrittenEntries),
-					"compression_ratio": fmt.Sprintf("%.2f", result.Archive.CompressionRatio),
-					"transfer_speed":    fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed),
-				})
+			case result.CacheHit, result.FallbackUsed:
+				l.WithFields(
+					logger.StringField("cache_id", cacheID),
+					logger.StringField("cache_key", result.Key),
+					logger.StringField("fallback_used", fmt.Sprintf("%t", result.FallbackUsed)),
+					logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
+					logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
+					logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
+					logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
+					logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+				).Info("Cache restored")
 			default:
 				// Cache was not found
-				l.Info("Cache not restored (not found)", map[string]interface{}{
-					"cache_id":  cacheID,
-					"cache_key": result.Key,
-				})
+				l.WithFields(
+					logger.StringField("cache_id", cacheID),
+					logger.StringField("cache_key", result.Key),
+				).Info("Cache not restored (not found)")
 			}
 		}
 
diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
index 3f6d033e5a..bd60d7c57e 100644
--- a/clicommand/cache_save.go
+++ b/clicommand/cache_save.go
@@ -6,6 +6,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/version"
 	"github.com/buildkite/zstash"
 	"github.com/buildkite/zstash/api"
@@ -75,6 +76,10 @@ var CacheSaveCommand = cli.Command{
 
 		apiCfg := loadAPIClientConfig(cfg, "AgentAccessToken")
 
+		if apiCfg.Token == "" {
+			return fmt.Errorf("an API token must be provided to save caches")
+		}
+
 		// we are using the zstash api package here which has a different client constructor but uses the same values
 		client := api.NewClient(ctx, version.Version(), apiCfg.Endpoint, apiCfg.Token)
 
@@ -83,6 +88,11 @@ var CacheSaveCommand = cli.Command{
 			return fmt.Errorf("failed to load cache configuration: %w", err)
 		}
 
+		if len(caches) == 0 {
+			l.Info("No caches defined in the cache configuration file, nothing to save")
+			return nil
+		}
+
 		cacheClient, err := zstash.NewCache(zstash.Config{
 			Client:       client,
 			BucketURL:    cfg.BucketURL,
@@ -101,13 +111,13 @@ var CacheSaveCommand = cli.Command{
 
 		if len(cacheIDs) == 0 {
 			// Save all caches configured in the client
-			caches := cacheClient.ListCaches()
-			for _, cache := range caches {
+			for _, cache := range cacheClient.ListCaches() {
 				cacheIDs = append(cacheIDs, cache.ID)
 			}
 		}
 
 		for _, cacheID := range cacheIDs {
+			l.Info("Saving cache: %s", cacheID)
 			result, err := cacheClient.Save(ctx, cacheID)
 			if err != nil {
 				return fmt.Errorf("failed to save cache %q: %w", cacheID, err)
@@ -115,20 +125,20 @@ var CacheSaveCommand = cli.Command{
 
 			switch {
 			case result.CacheCreated:
-				l.Info("Cache created", map[string]interface{}{
-					"cache_id":          cacheID,
-					"cache_key":         result.Key,
-					"archive_size":      humanize.Bytes(uint64(result.Archive.Size)),
-					"written_bytes":     humanize.Bytes(uint64(result.Archive.WrittenBytes)),
-					"written_entries":   fmt.Sprintf("%d", result.Archive.WrittenEntries),
-					"compression_ratio": fmt.Sprintf("%.2f", result.Archive.CompressionRatio),
-					"transfer_speed":    fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed),
-				})
+				l.WithFields(
+					logger.StringField("cache_id", cacheID),
+					logger.StringField("cache_key", result.Key),
+					logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
+					logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
+					logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
+					logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
+					logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+				).Info("Cache created")
 			default:
-				l.Info("Cache already exists, not saving", map[string]interface{}{
-					"cache_id":  cacheID,
-					"cache_key": result.Key,
-				})
+				l.WithFields(
+					logger.StringField("cache_id", cacheID),
+					logger.StringField("cache_key", result.Key),
+				).Info("Cache already exists, not saving")
 			}
 		}
 
diff --git a/clicommand/global.go b/clicommand/global.go
index eec4058b47..ce8490cdd4 100644
--- a/clicommand/global.go
+++ b/clicommand/global.go
@@ -187,7 +187,7 @@ type APIConfig struct {
 type CacheConfig struct {
 	Ids             string `cli:"ids"`
 	Registry        string `cli:"registry"`
-	BucketURL       string `cli:"bucket-url"`
+	BucketURL       string `cli:"bucket-url" validate:"required"`
 	Branch          string `cli:"branch" validate:"required"`
 	Pipeline        string `cli:"pipeline" validate:"required"`
 	Organization    string `cli:"organization" validate:"required"`

From da52bdf073573b5c7bf443992a804a77b21bdd82 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 23 Oct 2025 15:55:08 +1100
Subject: [PATCH 041/242] refactor: moved the cache logic into it's own package
 and added tests

---
 clicommand/cache.go          |  24 ---
 clicommand/cache_restore.go  |  84 ++-------
 clicommand/cache_save.go     |  80 ++------
 internal/cache/cache.go      | 191 +++++++++++++++++++
 internal/cache/cache_test.go | 348 +++++++++++++++++++++++++++++++++++
 5 files changed, 565 insertions(+), 162 deletions(-)
 delete mode 100644 clicommand/cache.go
 create mode 100644 internal/cache/cache.go
 create mode 100644 internal/cache/cache_test.go

diff --git a/clicommand/cache.go b/clicommand/cache.go
deleted file mode 100644
index 6c1640001f..0000000000
--- a/clicommand/cache.go
+++ /dev/null
@@ -1,24 +0,0 @@
-package clicommand
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/buildkite/zstash/cache"
-	"github.com/stretchr/testify/assert/yaml"
-)
-
-// use yaml to load cache configuration from file
-func loadCacheConfiguration(cacheConfigFile string) ([]cache.Cache, error) {
-	data, err := os.ReadFile(cacheConfigFile)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read cache config file: %w", err)
-	}
-
-	var caches []cache.Cache
-	if err := yaml.Unmarshal(data, &caches); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal cache config file: %w", err)
-	}
-
-	return caches, nil
-}
diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
index 2b327618a4..e05a2a2d50 100644
--- a/clicommand/cache_restore.go
+++ b/clicommand/cache_restore.go
@@ -2,15 +2,9 @@ package clicommand
 
 import (
 	"context"
-	"fmt"
 	"slices"
-	"strings"
 
-	"github.com/buildkite/agent/v3/logger"
-	"github.com/buildkite/agent/v3/version"
-	"github.com/buildkite/zstash"
-	"github.com/buildkite/zstash/api"
-	"github.com/dustin/go-humanize"
+	"github.com/buildkite/agent/v3/internal/cache"
 	"github.com/urfave/cli"
 )
 
@@ -83,71 +77,19 @@ var CacheRestoreCommand = cli.Command{
 
 		apiCfg := loadAPIClientConfig(cfg, "AgentAccessToken")
 
-		// we are using the zstash api package here which has a different client constructor but uses the same values
-		client := api.NewClient(ctx, version.Version(), apiCfg.Endpoint, apiCfg.Token)
-
-		caches, err := loadCacheConfiguration(cfg.CacheConfigFile)
-		if err != nil {
-			return fmt.Errorf("failed to load cache configuration: %w", err)
-		}
-
-		if len(caches) == 0 {
-			l.Info("No caches defined in the cache configuration file, nothing to save")
-			return nil
-		}
-
-		cacheClient, err := zstash.NewCache(zstash.Config{
-			Client:       client,
-			BucketURL:    cfg.BucketURL,
-			Format:       "zip",
-			Branch:       cfg.Branch,
-			Pipeline:     cfg.Pipeline,
-			Organization: cfg.Organization,
-			Caches:       caches,
-		})
-		if err != nil {
-			return fmt.Errorf("failed to create cache client: %w", err)
-		}
-
-		// split the ids by comma
-		cacheIDs := strings.Split(cfg.Ids, ",")
-
-		if len(cacheIDs) == 0 {
-			// Save all caches configured in the client
-			caches := cacheClient.ListCaches()
-			for _, cache := range caches {
-				cacheIDs = append(cacheIDs, cache.ID)
-			}
-		}
-
-		for _, cacheID := range cacheIDs {
-			l.Info("Restoring cache: %s", cacheID)
-			result, err := cacheClient.Restore(ctx, cacheID)
-			if err != nil {
-				return fmt.Errorf("failed to restore cache %q: %w", cacheID, err)
-			}
-
-			switch {
-			case result.CacheHit, result.FallbackUsed:
-				l.WithFields(
-					logger.StringField("cache_id", cacheID),
-					logger.StringField("cache_key", result.Key),
-					logger.StringField("fallback_used", fmt.Sprintf("%t", result.FallbackUsed)),
-					logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
-					logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
-					logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
-					logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
-					logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
-				).Info("Cache restored")
-			default:
-				// Cache was not found
-				l.WithFields(
-					logger.StringField("cache_id", cacheID),
-					logger.StringField("cache_key", result.Key),
-				).Info("Cache not restored (not found)")
-			}
+		// Build cache configuration
+		cacheCfg := cache.Config{
+			BucketURL:       cfg.BucketURL,
+			Branch:          cfg.Branch,
+			Pipeline:        cfg.Pipeline,
+			Organization:    cfg.Organization,
+			CacheConfigFile: cfg.CacheConfigFile,
+			Ids:             cfg.Ids,
+			APIEndpoint:     apiCfg.Endpoint,
+			APIToken:        apiCfg.Token,
 		}
 
-		return nil
+		// Perform cache restore (logging happens inside)
+		return cache.Restore(ctx, l, cacheCfg)
 	},
 }
diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
index bd60d7c57e..2b6377b3bc 100644
--- a/clicommand/cache_save.go
+++ b/clicommand/cache_save.go
@@ -4,13 +4,8 @@ import (
 	"context"
 	"fmt"
 	"slices"
-	"strings"
 
-	"github.com/buildkite/agent/v3/logger"
-	"github.com/buildkite/agent/v3/version"
-	"github.com/buildkite/zstash"
-	"github.com/buildkite/zstash/api"
-	"github.com/dustin/go-humanize"
+	"github.com/buildkite/agent/v3/internal/cache"
 	"github.com/urfave/cli"
 )
 
@@ -80,68 +75,19 @@ var CacheSaveCommand = cli.Command{
 			return fmt.Errorf("an API token must be provided to save caches")
 		}
 
-		// we are using the zstash api package here which has a different client constructor but uses the same values
-		client := api.NewClient(ctx, version.Version(), apiCfg.Endpoint, apiCfg.Token)
-
-		caches, err := loadCacheConfiguration(cfg.CacheConfigFile)
-		if err != nil {
-			return fmt.Errorf("failed to load cache configuration: %w", err)
-		}
-
-		if len(caches) == 0 {
-			l.Info("No caches defined in the cache configuration file, nothing to save")
-			return nil
-		}
-
-		cacheClient, err := zstash.NewCache(zstash.Config{
-			Client:       client,
-			BucketURL:    cfg.BucketURL,
-			Format:       "zip",
-			Branch:       cfg.Branch,
-			Pipeline:     cfg.Pipeline,
-			Organization: cfg.Organization,
-			Caches:       caches,
-		})
-		if err != nil {
-			return fmt.Errorf("failed to create cache client: %w", err)
-		}
-
-		// split the ids by comma
-		cacheIDs := strings.Split(cfg.Ids, ",")
-
-		if len(cacheIDs) == 0 {
-			// Save all caches configured in the client
-			for _, cache := range cacheClient.ListCaches() {
-				cacheIDs = append(cacheIDs, cache.ID)
-			}
-		}
-
-		for _, cacheID := range cacheIDs {
-			l.Info("Saving cache: %s", cacheID)
-			result, err := cacheClient.Save(ctx, cacheID)
-			if err != nil {
-				return fmt.Errorf("failed to save cache %q: %w", cacheID, err)
-			}
-
-			switch {
-			case result.CacheCreated:
-				l.WithFields(
-					logger.StringField("cache_id", cacheID),
-					logger.StringField("cache_key", result.Key),
-					logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
-					logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
-					logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
-					logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
-					logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
-				).Info("Cache created")
-			default:
-				l.WithFields(
-					logger.StringField("cache_id", cacheID),
-					logger.StringField("cache_key", result.Key),
-				).Info("Cache already exists, not saving")
-			}
+		// Build cache configuration
+		cacheCfg := cache.Config{
+			BucketURL:       cfg.BucketURL,
+			Branch:          cfg.Branch,
+			Pipeline:        cfg.Pipeline,
+			Organization:    cfg.Organization,
+			CacheConfigFile: cfg.CacheConfigFile,
+			Ids:             cfg.Ids,
+			APIEndpoint:     apiCfg.Endpoint,
+			APIToken:        apiCfg.Token,
 		}
 
-		return nil
+		// Perform cache save (logging happens inside)
+		return cache.Save(ctx, l, cacheCfg)
 	},
 }
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
new file mode 100644
index 0000000000..fa9ea1d2b9
--- /dev/null
+++ b/internal/cache/cache.go
@@ -0,0 +1,191 @@
+package cache
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/buildkite/agent/v3/logger"
+	"github.com/buildkite/agent/v3/version"
+	"github.com/buildkite/zstash"
+	"github.com/buildkite/zstash/api"
+	"github.com/buildkite/zstash/cache"
+	"github.com/dustin/go-humanize"
+	"gopkg.in/yaml.v3"
+)
+
+// Config holds the configuration for cache operations
+type Config struct {
+	// BucketURL is the URL of the bucket (e.g., s3://bucket-name)
+	BucketURL string
+	// Branch is the branch associated with the cache
+	Branch string
+	// Pipeline is the pipeline slug for this cache
+	Pipeline string
+	// Organization is the organization slug for this cache
+	Organization string
+	// CacheConfigFile is the path to the cache configuration YAML file
+	CacheConfigFile string
+	// Ids is a comma-separated list of cache IDs (if empty, processes all caches)
+	Ids string
+	// APIEndpoint is the Agent API endpoint
+	APIEndpoint string
+	// APIToken is the access token used to authenticate
+	APIToken string
+}
+
+// CacheClient defines the interface for cache operations
+type CacheClient interface {
+	Save(ctx context.Context, cacheID string) (zstash.SaveResult, error)
+	Restore(ctx context.Context, cacheID string) (zstash.RestoreResult, error)
+	ListCaches() []cache.Cache
+}
+
+// Save saves caches based on the provided configuration and logs results as each cache is processed
+func Save(ctx context.Context, l logger.Logger, cfg Config) error {
+	cacheClient, cacheIDs, err := setupCacheClient(ctx, cfg)
+	if err != nil {
+		return err
+	}
+
+	if cacheClient == nil {
+		l.Info("No caches defined in the cache configuration file, nothing to save")
+		return nil
+	}
+
+	return saveWithClient(ctx, l, cacheClient, cacheIDs)
+}
+
+// Restore restores caches based on the provided configuration and logs results as each cache is processed
+func Restore(ctx context.Context, l logger.Logger, cfg Config) error {
+	cacheClient, cacheIDs, err := setupCacheClient(ctx, cfg)
+	if err != nil {
+		return err
+	}
+
+	if cacheClient == nil {
+		l.Info("No caches defined in the cache configuration file, nothing to restore")
+		return nil
+	}
+
+	return restoreWithClient(ctx, l, cacheClient, cacheIDs)
+}
+
+// loadCacheConfiguration loads cache configuration from a YAML file
+func loadCacheConfiguration(cacheConfigFile string) ([]cache.Cache, error) {
+	data, err := os.ReadFile(cacheConfigFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read cache config file: %w", err)
+	}
+
+	var caches []cache.Cache
+	if err := yaml.Unmarshal(data, &caches); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal cache config file: %w", err)
+	}
+
+	return caches, nil
+}
+
+// setupCacheClient creates a cache client and determines which cache IDs to process
+func setupCacheClient(ctx context.Context, cfg Config) (*zstash.Cache, []string, error) {
+	client := api.NewClient(ctx, version.Version(), cfg.APIEndpoint, cfg.APIToken)
+
+	caches, err := loadCacheConfiguration(cfg.CacheConfigFile)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to load cache configuration: %w", err)
+	}
+
+	if len(caches) == 0 {
+		return nil, nil, nil
+	}
+
+	cacheClient, err := zstash.NewCache(zstash.Config{
+		Client:       client,
+		BucketURL:    cfg.BucketURL,
+		Format:       "zip",
+		Branch:       cfg.Branch,
+		Pipeline:     cfg.Pipeline,
+		Organization: cfg.Organization,
+		Caches:       caches,
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create cache client: %w", err)
+	}
+
+	// Determine which cache IDs to process
+	var cacheIDs []string
+	if cfg.Ids != "" {
+		cacheIDs = strings.Split(cfg.Ids, ",")
+	} else {
+		// Process all caches configured in the client
+		for _, cache := range cacheClient.ListCaches() {
+			cacheIDs = append(cacheIDs, cache.ID)
+		}
+	}
+
+	return cacheClient, cacheIDs, nil
+}
+
+// restoreWithClient performs the restore operation for the given cache IDs using the provided client
+func restoreWithClient(ctx context.Context, l logger.Logger, client CacheClient, cacheIDs []string) error {
+	for _, cacheID := range cacheIDs {
+		l.Info("Restoring cache: %s", cacheID)
+		result, err := client.Restore(ctx, cacheID)
+		if err != nil {
+			return fmt.Errorf("failed to restore cache %q: %w", cacheID, err)
+		}
+
+		switch {
+		case result.CacheHit, result.FallbackUsed:
+			l.WithFields(
+				logger.StringField("cache_id", cacheID),
+				logger.StringField("cache_key", result.Key),
+				logger.StringField("fallback_used", fmt.Sprintf("%t", result.FallbackUsed)),
+				logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
+				logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
+				logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
+				logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
+				logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+			).Info("Cache restored")
+		default:
+			l.WithFields(
+				logger.StringField("cache_id", cacheID),
+				logger.StringField("cache_key", result.Key),
+			).Info("Cache not restored (not found)")
+		}
+	}
+
+	return nil
+}
+
+// saveWithClient performs the save operation for the given cache IDs using the provided client
+func saveWithClient(ctx context.Context, l logger.Logger, client CacheClient, cacheIDs []string) error {
+	for _, cacheID := range cacheIDs {
+		l.Info("Saving cache: %s", cacheID)
+		result, err := client.Save(ctx, cacheID)
+		if err != nil {
+			return fmt.Errorf("failed to save cache %q: %w", cacheID, err)
+		}
+
+		switch {
+		case result.CacheCreated:
+			l.WithFields(
+				logger.StringField("cache_id", cacheID),
+				logger.StringField("cache_key", result.Key),
+				logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
+				logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
+				logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
+				logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
+				logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+			).Info("Cache created")
+		default:
+			l.WithFields(
+				logger.StringField("cache_id", cacheID),
+				logger.StringField("cache_key", result.Key),
+			).Info("Cache already exists, not saving")
+		}
+	}
+
+	return nil
+}
diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go
new file mode 100644
index 0000000000..a1a00d491e
--- /dev/null
+++ b/internal/cache/cache_test.go
@@ -0,0 +1,348 @@
+package cache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/buildkite/agent/v3/logger"
+	"github.com/buildkite/zstash"
+	"github.com/buildkite/zstash/cache"
+	"github.com/stretchr/testify/require"
+)
+
+// mockCacheClient is a mock implementation of the CacheClient interface for testing
+type mockCacheClient struct {
+	saveFunc    func(ctx context.Context, cacheID string) (zstash.SaveResult, error)
+	restoreFunc func(ctx context.Context, cacheID string) (zstash.RestoreResult, error)
+	listFunc    func() []cache.Cache
+}
+
+func (m *mockCacheClient) Save(ctx context.Context, cacheID string) (zstash.SaveResult, error) {
+	if m.saveFunc != nil {
+		return m.saveFunc(ctx, cacheID)
+	}
+	return zstash.SaveResult{}, nil
+}
+
+func (m *mockCacheClient) Restore(ctx context.Context, cacheID string) (zstash.RestoreResult, error) {
+	if m.restoreFunc != nil {
+		return m.restoreFunc(ctx, cacheID)
+	}
+	return zstash.RestoreResult{}, nil
+}
+
+func (m *mockCacheClient) ListCaches() []cache.Cache {
+	if m.listFunc != nil {
+		return m.listFunc()
+	}
+	return nil
+}
+
+// Test helpers
+
+func createTempCacheConfig(t *testing.T, content string) string {
+	t.Helper()
+	tmpDir := t.TempDir()
+	configFile := filepath.Join(tmpDir, "cache.yml")
+	err := os.WriteFile(configFile, []byte(content), 0o600)
+	require.NoError(t, err)
+	return configFile
+}
+
+// Tests for saveWithClient
+
+func TestSaveWithClient_CacheCreated(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	mock := &mockCacheClient{
+		saveFunc: func(ctx context.Context, cacheID string) (zstash.SaveResult, error) {
+			return zstash.SaveResult{
+				CacheCreated: true,
+				Key:          "test-key-v1",
+				Archive: zstash.ArchiveMetrics{
+					Size:             1024,
+					WrittenBytes:     1024,
+					WrittenEntries:   10,
+					CompressionRatio: 2.5,
+				},
+				Transfer: &zstash.TransferMetrics{
+					TransferSpeed: 5.5,
+				},
+			}, nil
+		},
+	}
+
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	require.NoError(t, err)
+}
+
+func TestSaveWithClient_CacheAlreadyExists(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	mock := &mockCacheClient{
+		saveFunc: func(ctx context.Context, cacheID string) (zstash.SaveResult, error) {
+			return zstash.SaveResult{
+				CacheCreated: false,
+				Key:          "test-key-v1",
+			}, nil
+		},
+	}
+
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	require.NoError(t, err)
+}
+
+func TestSaveWithClient_MultipleCaches(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	callCount := 0
+	mock := &mockCacheClient{
+		saveFunc: func(ctx context.Context, cacheID string) (zstash.SaveResult, error) {
+			callCount++
+			return zstash.SaveResult{
+				CacheCreated: true,
+				Key:          fmt.Sprintf("key-%s", cacheID),
+				Archive: zstash.ArchiveMetrics{
+					Size:             100,
+					WrittenBytes:     100,
+					WrittenEntries:   1,
+					CompressionRatio: 1.0,
+				},
+				Transfer: &zstash.TransferMetrics{
+					TransferSpeed: 1.0,
+				},
+			}, nil
+		},
+	}
+
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1", "cache2", "cache3"})
+	require.NoError(t, err)
+	require.Equal(t, 3, callCount, "Expected Save to be called 3 times")
+}
+
+func TestSaveWithClient_Error(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	expectedErr := errors.New("save failed")
+	mock := &mockCacheClient{
+		saveFunc: func(ctx context.Context, cacheID string) (zstash.SaveResult, error) {
+			return zstash.SaveResult{}, expectedErr
+		},
+	}
+
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	require.Error(t, err)
+	require.ErrorContains(t, err, "failed to save cache")
+	require.ErrorContains(t, err, "save failed")
+}
+
+func TestSaveWithClient_EmptyCacheIDs(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	mock := &mockCacheClient{
+		saveFunc: func(ctx context.Context, cacheID string) (zstash.SaveResult, error) {
+			t.Fatal("Save should not be called with empty cache IDs")
+			return zstash.SaveResult{}, nil
+		},
+	}
+
+	err := saveWithClient(ctx, logger.Discard, mock, []string{})
+	require.NoError(t, err)
+}
+
+// Tests for restoreWithClient
+
+func TestRestoreWithClient_CacheHit(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	mock := &mockCacheClient{
+		restoreFunc: func(ctx context.Context, cacheID string) (zstash.RestoreResult, error) {
+			return zstash.RestoreResult{
+				CacheHit:      true,
+				CacheRestored: true,
+				FallbackUsed:  false,
+				Key:           "test-key-v1",
+				Archive: zstash.ArchiveMetrics{
+					Size:             1024,
+					WrittenBytes:     1024,
+					WrittenEntries:   10,
+					CompressionRatio: 2.5,
+				},
+				Transfer: zstash.TransferMetrics{
+					TransferSpeed: 5.5,
+				},
+			}, nil
+		},
+	}
+
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	require.NoError(t, err)
+}
+
+func TestRestoreWithClient_FallbackUsed(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	mock := &mockCacheClient{
+		restoreFunc: func(ctx context.Context, cacheID string) (zstash.RestoreResult, error) {
+			return zstash.RestoreResult{
+				CacheHit:      false,
+				CacheRestored: true,
+				FallbackUsed:  true,
+				Key:           "test-key-fallback",
+				Archive: zstash.ArchiveMetrics{
+					Size:             512,
+					WrittenBytes:     512,
+					WrittenEntries:   5,
+					CompressionRatio: 2.0,
+				},
+				Transfer: zstash.TransferMetrics{
+					TransferSpeed: 3.5,
+				},
+			}, nil
+		},
+	}
+
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	require.NoError(t, err)
+}
+
+func TestRestoreWithClient_CacheMiss(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	mock := &mockCacheClient{
+		restoreFunc: func(ctx context.Context, cacheID string) (zstash.RestoreResult, error) {
+			return zstash.RestoreResult{
+				CacheHit:      false,
+				CacheRestored: false,
+				FallbackUsed:  false,
+				Key:           "test-key-v1",
+			}, nil
+		},
+	}
+
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	require.NoError(t, err)
+}
+
+func TestRestoreWithClient_MultipleCaches(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	callCount := 0
+	mock := &mockCacheClient{
+		restoreFunc: func(ctx context.Context, cacheID string) (zstash.RestoreResult, error) {
+			callCount++
+			return zstash.RestoreResult{
+				CacheHit:      true,
+				CacheRestored: true,
+				Key:           fmt.Sprintf("key-%s", cacheID),
+			}, nil
+		},
+	}
+
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1", "cache2", "cache3"})
+	require.NoError(t, err)
+	require.Equal(t, 3, callCount, "Expected Restore to be called 3 times")
+}
+
+func TestRestoreWithClient_Error(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	expectedErr := errors.New("restore failed")
+	mock := &mockCacheClient{
+		restoreFunc: func(ctx context.Context, cacheID string) (zstash.RestoreResult, error) {
+			return zstash.RestoreResult{}, expectedErr
+		},
+	}
+
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	require.Error(t, err)
+	require.ErrorContains(t, err, "failed to restore cache")
+	require.ErrorContains(t, err, "restore failed")
+}
+
+func TestRestoreWithClient_EmptyCacheIDs(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	mock := &mockCacheClient{
+		restoreFunc: func(ctx context.Context, cacheID string) (zstash.RestoreResult, error) {
+			t.Fatal("Restore should not be called with empty cache IDs")
+			return zstash.RestoreResult{}, nil
+		},
+	}
+
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{})
+	require.NoError(t, err)
+}
+
+// Tests for loadCacheConfiguration
+
+func TestLoadCacheConfiguration_Valid(t *testing.T) {
+	t.Parallel()
+
+	config := `- id: node
+  key: 'node-{{ checksum "package-lock.json" }}'
+  paths:
+    - node_modules
+- id: ruby
+  key: 'ruby-{{ checksum "Gemfile.lock" }}'
+  paths:
+    - vendor/bundle
+`
+	configFile := createTempCacheConfig(t, config)
+
+	caches, err := loadCacheConfiguration(configFile)
+	require.NoError(t, err)
+	require.Len(t, caches, 2)
+	require.Equal(t, "node", caches[0].ID)
+	require.Equal(t, "ruby", caches[1].ID)
+}
+
+func TestLoadCacheConfiguration_InvalidYAML(t *testing.T) {
+	t.Parallel()
+
+	config := `---
+- id: node
+  key: test
+  paths
+    - invalid indentation here
+  : wrong syntax
+`
+	configFile := createTempCacheConfig(t, config)
+
+	_, err := loadCacheConfiguration(configFile)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "failed to unmarshal cache config file")
+}
+
+func TestLoadCacheConfiguration_FileNotFound(t *testing.T) {
+	t.Parallel()
+
+	_, err := loadCacheConfiguration("/nonexistent/path/to/cache.yml")
+	require.Error(t, err)
+	require.ErrorContains(t, err, "failed to read cache config file")
+}
+
+func TestLoadCacheConfiguration_EmptyFile(t *testing.T) {
+	t.Parallel()
+
+	configFile := createTempCacheConfig(t, "")
+
+	caches, err := loadCacheConfiguration(configFile)
+	require.NoError(t, err)
+	require.Empty(t, caches)
+}

From a3f639186c43c4996f3676e443e76f5320ea05c7 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 23 Oct 2025 16:01:14 +1100
Subject: [PATCH 042/242] feat: add progress updates during save or restore

---
 internal/cache/cache.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index fa9ea1d2b9..ce63b960ea 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -108,6 +108,14 @@ func setupCacheClient(ctx context.Context, cfg Config) (*zstash.Cache, []string,
 		Pipeline:     cfg.Pipeline,
 		Organization: cfg.Organization,
 		Caches:       caches,
+		OnProgress: func(stage, message string, current, total int) {
+			l.WithFields(
+				logger.StringField("stage", stage),
+				logger.StringField("message", message),
+				logger.IntField("current", current),
+				logger.IntField("total", total),
+			).Info("Cache progress")
+		},
 	})
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to create cache client: %w", err)

From 0fb60cce5d3bc91a196db5fd4c3dadf6fa2b5f86 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 23 Oct 2025 16:25:18 +1100
Subject: [PATCH 043/242] fix: updated to latest zstash and added progress
 logging

---
 go.mod                  |  3 +--
 go.sum                  | 14 ++------------
 internal/cache/cache.go |  6 +++---
 3 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/go.mod b/go.mod
index 8432d0de3e..a09e9d674d 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
-	github.com/buildkite/zstash v0.4.1-0.20251022224615-e5fb6fcef4ee
+	github.com/buildkite/zstash v0.4.1-0.20251023050154-79acaf6ed38f
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
@@ -161,7 +161,6 @@ require (
 	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/qri-io/jsonpointer v0.1.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/rs/zerolog v1.34.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect
diff --git a/go.sum b/go.sum
index 0841378969..537b9b75b2 100644
--- a/go.sum
+++ b/go.sum
@@ -138,8 +138,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
-github.com/buildkite/zstash v0.4.1-0.20251022224615-e5fb6fcef4ee h1:L/kSSvQ+TAx9bVV0P/pU6qX5A2/n3uMYBFUSMSsx6fg=
-github.com/buildkite/zstash v0.4.1-0.20251022224615-e5fb6fcef4ee/go.mod h1:uV3f+1NmQyH1xgTJB3o2emmKqUhowkrYOaTKCskzKgw=
+github.com/buildkite/zstash v0.4.1-0.20251023050154-79acaf6ed38f h1:0mNPlld0dDlBniRkS7FRMPgWoatOpO4JQL0aJ5FaT2o=
+github.com/buildkite/zstash v0.4.1-0.20251023050154-79acaf6ed38f/go.mod h1:h70JfAEa2Ys1GDQQ6CNoKIMfMgJ0LZkNmQnzK710PHQ=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -147,7 +147,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
-github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.19 h1:tUN6H7LWqNx4hQVxomd0CVsDwaDr9gaRQaI4GpSmrsA=
@@ -198,7 +197,6 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
 github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -277,11 +275,8 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB
 github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc=
 github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -337,9 +332,6 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
-github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
-github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 h1:8zyjtFyKi5NJySVOJRiHmSN1vl6qugQ5n9C4X7WyY3U=
@@ -516,10 +508,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
 golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index ce63b960ea..99591d4562 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -44,7 +44,7 @@ type CacheClient interface {
 
 // Save saves caches based on the provided configuration and logs results as each cache is processed
 func Save(ctx context.Context, l logger.Logger, cfg Config) error {
-	cacheClient, cacheIDs, err := setupCacheClient(ctx, cfg)
+	cacheClient, cacheIDs, err := setupCacheClient(ctx, l, cfg)
 	if err != nil {
 		return err
 	}
@@ -59,7 +59,7 @@ func Save(ctx context.Context, l logger.Logger, cfg Config) error {
 
 // Restore restores caches based on the provided configuration and logs results as each cache is processed
 func Restore(ctx context.Context, l logger.Logger, cfg Config) error {
-	cacheClient, cacheIDs, err := setupCacheClient(ctx, cfg)
+	cacheClient, cacheIDs, err := setupCacheClient(ctx, l, cfg)
 	if err != nil {
 		return err
 	}
@@ -88,7 +88,7 @@ func loadCacheConfiguration(cacheConfigFile string) ([]cache.Cache, error) {
 }
 
 // setupCacheClient creates a cache client and determines which cache IDs to process
-func setupCacheClient(ctx context.Context, cfg Config) (*zstash.Cache, []string, error) {
+func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash.Cache, []string, error) {
 	client := api.NewClient(ctx, version.Version(), cfg.APIEndpoint, cfg.APIToken)
 
 	caches, err := loadCacheConfiguration(cfg.CacheConfigFile)

From 04ceb3d2fff23d77bd1efd56fe0e38d9eaf49d69 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Fri, 24 Oct 2025 11:11:43 +1100
Subject: [PATCH 044/242] chore: I have hidden the cache sub command added
 notes to help

---
 clicommand/cache_restore.go | 3 +++
 clicommand/cache_save.go    | 3 +++
 clicommand/commands.go      | 1 +
 3 files changed, 7 insertions(+)

diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
index e05a2a2d50..fa9bfbadae 100644
--- a/clicommand/cache_restore.go
+++ b/clicommand/cache_restore.go
@@ -22,6 +22,9 @@ and their associated cache keys. Caches are scoped by organization, pipeline, an
 branch. If an exact cache match is not found, the command will attempt to use
 fallback keys if defined in your cache configuration.
 
+Note: This feature is currently in development and subject to change. It is not
+yet available to all customers.
+
 Example:
 
     $ buildkite-agent cache restore
diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
index 2b6377b3bc..43964310db 100644
--- a/clicommand/cache_save.go
+++ b/clicommand/cache_save.go
@@ -22,6 +22,9 @@ The cache configuration file defines which files or directories should be cached
 and their associated cache keys. Caches are scoped by organization, pipeline, and
 branch.
 
+Note: This feature is currently in development and subject to change. It is not
+yet available to all customers.
+
 Example:
 
     $ buildkite-agent cache save
diff --git a/clicommand/commands.go b/clicommand/commands.go
index d092d60b09..a11b6bfb0c 100644
--- a/clicommand/commands.go
+++ b/clicommand/commands.go
@@ -47,6 +47,7 @@ var BuildkiteAgentCommands = []cli.Command{
 		Name:     "cache",
 		Category: categoryJobCommands,
 		Usage:    "Manage build caches",
+		Hidden:   true, // currently in experimental phase
 		Subcommands: []cli.Command{
 			CacheSaveCommand,
 			CacheRestoreCommand,

From fa8728acecef157a255b04be19723f4e9a8b29e8 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Fri, 24 Oct 2025 16:22:07 +1100
Subject: [PATCH 045/242] chore: bump zstash to v0.5.0 which includes quite a
 few fixes

---
 go.mod | 14 +++++++-------
 go.sum | 28 ++++++++++++++--------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/go.mod b/go.mod
index a09e9d674d..cec1726f5d 100644
--- a/go.mod
+++ b/go.mod
@@ -15,15 +15,15 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.39.4
 	github.com/aws/aws-sdk-go-v2/config v1.31.15
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1
-	github.com/aws/aws-sdk-go-v2/service/kms v1.46.2
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2
+	github.com/aws/aws-sdk-go-v2/service/kms v1.46.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
 	github.com/buildkite/go-pipeline v0.16.0
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
-	github.com/buildkite/zstash v0.4.1-0.20251023050154-79acaf6ed38f
+	github.com/buildkite/zstash v0.5.0
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
@@ -96,12 +96,12 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect
diff --git a/go.sum b/go.sum
index 537b9b75b2..29645883e1 100644
--- a/go.sum
+++ b/go.sum
@@ -94,22 +94,22 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11/go.mod h1:7bUb2sSr2MZ3M/N+VyETLTQtInemHXb/Fl3s8CLzm0Y=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 h1:FHw90xCTsofzk6vjU808TSuDtDfOOKPNdz5Weyc3tUI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10/go.mod h1:n8jdIE/8F3UYkg8O4IGkQpn2qUmapg/1K1yl29/uf/c=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1 h1:D8cBaI1TsIF+cbB8qPmiZWsMqGsbs1/e7qYQ0NMDscY=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1/go.mod h1:DT0XByGaNaOff3CtLVmj3jKcMeVDfOj5DkLD39UPJY0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11 h1:bKgSxk1TW//00PGQqYmrq83c+2myGidEclp+t9pPqVI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11/go.mod h1:vrPYCQ6rFHL8jzQA8ppu3gWX18zxjLIDGTeqDxkBmSI=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2 h1:D8MCemFa8rt09x7o6Fkm2T7ThVbRPrD91R+LKhVEnVU=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2/go.mod h1:Q/kZ++hvhasMpQU37I7daQh07ZqTa++isjj1aPi4zvM=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 h1:ne+eepnDB2Wh5lHKzELgEncIqeVlQ1rSF9fEa4r5I+A=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1/go.mod h1:u0Jkg0L+dcG1ozUq21uFElmpbmjBnhHR5DELHIme4wg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2 h1:DGFpGybmutVsCuF6vSuLZ25Vh55E3VmsnJmFfjeBx4M=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2/go.mod h1:hm/wU1HDvXCFEDzOLorQnZZ/CVvPXvWEmHMSmqgQRuA=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 h1:GpMf3z2KJa4RnJ0ew3Hac+hRFYLZ9DDjfgXjuW+pB54=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11/go.mod h1:6MZP3ZI4QQsgUCFTwMZA2V0sEriNQ8k2hmoHF3qjimQ=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 h1:DA+Hl5adieRyFvE7pCvBWm3VOZTRexGVkXw33SUqNoY=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10/go.mod h1:L+A89dH3/gr8L4ecrdzuXUYd1znoko6myzndVGZx/DA=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.2 h1:hz2rJseQXnVQtVbByFpeSCNJBBU7oFN+yenW4biJtvs=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.2/go.mod h1:E4ink1KCQgqIe2pHFD9E+b5CNXovm50rQbWFuh0cM+I=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5 h1:FlGScxzCGNzT+2AvHT1ZGMvxTwAMa6gsooFb1pO/AiM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5/go.mod h1:N/iojY+8bW3MYol9NUMuKimpSbPEur75cuI1SmtonFM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 h1:weapBOuuFIBEQ9OX/NVW3tFQCvSutyjZYk/ga5jDLPo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11/go.mod h1:3C1gN4FmIVLwYSh8etngUS+f1viY6nLCDVtZmrFbDy0=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.0 h1:vSXYridw+tT3AHuK1PWdJto2qEc30/wG/fm8dmCHHis=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.0/go.mod h1:YXPskkMuiMgp6qUG96NSTl7UpideOQT/Kx0u9Y1MKn0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 h1:Wer3W0GuaedWT7dv/PiWNZGSQFSTcBY2rZpbiUp5xcA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7/go.mod h1:UHKgcRSx8PVtvsc1Poxb/Co3PD3wL7P+f49P0+cWtuY=
 github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU=
 github.com/aws/aws-sdk-go-v2/service/sso v1.29.8/go.mod h1:mbef/pgKhtKRwrigPPs7SSSKZgytzP8PQ6P6JAAdqyM=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 h1:S5GuJZpYxE0lKeMHKn+BRTz6PTFpgThyJ+5mYfux7BM=
@@ -138,8 +138,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
-github.com/buildkite/zstash v0.4.1-0.20251023050154-79acaf6ed38f h1:0mNPlld0dDlBniRkS7FRMPgWoatOpO4JQL0aJ5FaT2o=
-github.com/buildkite/zstash v0.4.1-0.20251023050154-79acaf6ed38f/go.mod h1:h70JfAEa2Ys1GDQQ6CNoKIMfMgJ0LZkNmQnzK710PHQ=
+github.com/buildkite/zstash v0.5.0 h1:e70mf8U2EjEB1eixXR78s6bsLgfo6bWLisVlRv58wCI=
+github.com/buildkite/zstash v0.5.0/go.mod h1:h70JfAEa2Ys1GDQQ6CNoKIMfMgJ0LZkNmQnzK710PHQ=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

From 5f0e982bd50e408e86de5ad8d7ae183334902b9e Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Mon, 27 Oct 2025 08:07:30 +1100
Subject: [PATCH 046/242] fix: validate cache ids and return list of invalid
 ones in error

---
 internal/cache/cache.go      | 17 +++++++
 internal/cache/cache_test.go | 99 ++++++++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)

diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index 99591d4562..9d09f35b1e 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -125,6 +125,23 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 	var cacheIDs []string
 	if cfg.Ids != "" {
 		cacheIDs = strings.Split(cfg.Ids, ",")
+		
+		// Validate that specified cache IDs exist
+		validIDs := make(map[string]bool)
+		for _, cache := range cacheClient.ListCaches() {
+			validIDs[cache.ID] = true
+		}
+		
+		var invalidIDs []string
+		for _, id := range cacheIDs {
+			if !validIDs[id] {
+				invalidIDs = append(invalidIDs, id)
+			}
+		}
+		
+		if len(invalidIDs) > 0 {
+			return nil, nil, fmt.Errorf("cache IDs not found in configuration: %s", strings.Join(invalidIDs, ", "))
+		}
 	} else {
 		// Process all caches configured in the client
 		for _, cache := range cacheClient.ListCaches() {
diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go
index a1a00d491e..2d32bf0a70 100644
--- a/internal/cache/cache_test.go
+++ b/internal/cache/cache_test.go
@@ -346,3 +346,102 @@ func TestLoadCacheConfiguration_EmptyFile(t *testing.T) {
 	require.NoError(t, err)
 	require.Empty(t, caches)
 }
+
+// Tests for setupCacheClient
+
+func TestSetupCacheClient_InvalidCacheIDs(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	config := `- id: cache1
+  key: 'test-key-1'
+  paths:
+    - path1
+- id: cache2
+  key: 'test-key-2'
+  paths:
+    - path2
+`
+	configFile := createTempCacheConfig(t, config)
+
+	cfg := Config{
+		CacheConfigFile: configFile,
+		Ids:             "cache1,invalid1,cache2,invalid2",
+		BucketURL:       "s3://test-bucket",
+		Branch:          "main",
+		Pipeline:        "test-pipeline",
+		Organization:    "test-org",
+		APIEndpoint:     "https://api.buildkite.com/v3",
+		APIToken:        "test-token",
+	}
+
+	_, _, err := setupCacheClient(ctx, logger.Discard, cfg)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "cache IDs not found in configuration")
+	require.ErrorContains(t, err, "invalid1")
+	require.ErrorContains(t, err, "invalid2")
+}
+
+func TestSetupCacheClient_ValidCacheIDs(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	config := `- id: cache1
+  key: 'test-key-1'
+  paths:
+    - path1
+- id: cache2
+  key: 'test-key-2'
+  paths:
+    - path2
+`
+	configFile := createTempCacheConfig(t, config)
+
+	cfg := Config{
+		CacheConfigFile: configFile,
+		Ids:             "cache1,cache2",
+		BucketURL:       "s3://test-bucket",
+		Branch:          "main",
+		Pipeline:        "test-pipeline",
+		Organization:    "test-org",
+		APIEndpoint:     "https://api.buildkite.com/v3",
+		APIToken:        "test-token",
+	}
+
+	client, cacheIDs, err := setupCacheClient(ctx, logger.Discard, cfg)
+	require.NoError(t, err)
+	require.NotNil(t, client)
+	require.Equal(t, []string{"cache1", "cache2"}, cacheIDs)
+}
+
+func TestSetupCacheClient_AllCaches(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	config := `- id: cache1
+  key: 'test-key-1'
+  paths:
+    - path1
+- id: cache2
+  key: 'test-key-2'
+  paths:
+    - path2
+`
+	configFile := createTempCacheConfig(t, config)
+
+	cfg := Config{
+		CacheConfigFile: configFile,
+		Ids:             "",
+		BucketURL:       "s3://test-bucket",
+		Branch:          "main",
+		Pipeline:        "test-pipeline",
+		Organization:    "test-org",
+		APIEndpoint:     "https://api.buildkite.com/v3",
+		APIToken:        "test-token",
+	}
+
+	client, cacheIDs, err := setupCacheClient(ctx, logger.Discard, cfg)
+	require.NoError(t, err)
+	require.NotNil(t, client)
+	require.ElementsMatch(t, []string{"cache1", "cache2"}, cacheIDs)
+}

From db27790a7f73dcd5f0d1c85b5c5dc1c87567fc35 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Mon, 27 Oct 2025 08:10:48 +1100
Subject: [PATCH 047/242] fix: corrected description build>job

---
 clicommand/cache_restore.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
index fa9bfbadae..5cf3629521 100644
--- a/clicommand/cache_restore.go
+++ b/clicommand/cache_restore.go
@@ -14,7 +14,7 @@ const cacheRestoreHelpDescription = `Usage:
 
 Description:
 
-Restores files from the cache for the current build based on the cache configuration
+Restores files from the cache for the current job based on the cache configuration
 defined in your cache config file (defaults to .buildkite/cache.yml).
 
 The cache configuration file defines which files or directories should be restored

From 527e40cb22591c6bab4b4e714c0a0e68db155562 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Mon, 27 Oct 2025 08:21:15 +1100
Subject: [PATCH 048/242] feat: added a top level configuration key for deps to
 enable extension

This structure allows additional options at the top level of the configuration.
---
 clicommand/cache_restore.go  | 13 ++---
 clicommand/cache_save.go     | 13 ++---
 internal/cache/cache.go      | 26 ++++++----
 internal/cache/cache_test.go | 92 +++++++++++++++++++-----------------
 4 files changed, 78 insertions(+), 66 deletions(-)

diff --git a/clicommand/cache_restore.go b/clicommand/cache_restore.go
index 5cf3629521..261b5e4323 100644
--- a/clicommand/cache_restore.go
+++ b/clicommand/cache_restore.go
@@ -41,12 +41,13 @@ Configuration File Format:
 
 The cache configuration file should be in YAML format:
 
-    - id: node
-      key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
-	  fallback_keys:
-		- '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
-      paths:
-        - node_modules
+    dependencies:
+      - id: node
+        key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
+        fallback_keys:
+          - '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
+        paths:
+          - node_modules
 
 Cache Restoration Results:
 
diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
index 43964310db..8da4254d40 100644
--- a/clicommand/cache_save.go
+++ b/clicommand/cache_save.go
@@ -42,12 +42,13 @@ Configuration File Format:
 
 The cache configuration file should be in YAML format:
 
-    - id: node
-      key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
-	  fallback_keys:
-		- '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
-      paths:
-        - node_modules
+    dependencies:
+      - id: node
+        key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
+        fallback_keys:
+          - '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
+        paths:
+          - node_modules
 
 The command automatically uses the following environment variables when available:
   - BUILDKITE_BRANCH (for branch scoping)
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index 9d09f35b1e..fbd350f795 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -35,6 +35,12 @@ type Config struct {
 	APIToken string
 }
 
+// FileConfig represents the structure of the cache configuration YAML file
+type FileConfig struct {
+	// Dependencies is the list of dependency caches to restore/save
+	Dependencies []cache.Cache `yaml:"dependencies"`
+}
+
 // CacheClient defines the interface for cache operations
 type CacheClient interface {
 	Save(ctx context.Context, cacheID string) (zstash.SaveResult, error)
@@ -73,30 +79,30 @@ func Restore(ctx context.Context, l logger.Logger, cfg Config) error {
 }
 
 // loadCacheConfiguration loads cache configuration from a YAML file
-func loadCacheConfiguration(cacheConfigFile string) ([]cache.Cache, error) {
+func loadCacheConfiguration(cacheConfigFile string) (*FileConfig, error) {
 	data, err := os.ReadFile(cacheConfigFile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read cache config file: %w", err)
 	}
 
-	var caches []cache.Cache
-	if err := yaml.Unmarshal(data, &caches); err != nil {
+	var config FileConfig
+	if err := yaml.Unmarshal(data, &config); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal cache config file: %w", err)
 	}
 
-	return caches, nil
+	return &config, nil
 }
 
 // setupCacheClient creates a cache client and determines which cache IDs to process
 func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash.Cache, []string, error) {
 	client := api.NewClient(ctx, version.Version(), cfg.APIEndpoint, cfg.APIToken)
 
-	caches, err := loadCacheConfiguration(cfg.CacheConfigFile)
+	fileConfig, err := loadCacheConfiguration(cfg.CacheConfigFile)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to load cache configuration: %w", err)
 	}
 
-	if len(caches) == 0 {
+	if len(fileConfig.Dependencies) == 0 {
 		return nil, nil, nil
 	}
 
@@ -107,7 +113,7 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 		Branch:       cfg.Branch,
 		Pipeline:     cfg.Pipeline,
 		Organization: cfg.Organization,
-		Caches:       caches,
+		Caches:       fileConfig.Dependencies,
 		OnProgress: func(stage, message string, current, total int) {
 			l.WithFields(
 				logger.StringField("stage", stage),
@@ -125,20 +131,20 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 	var cacheIDs []string
 	if cfg.Ids != "" {
 		cacheIDs = strings.Split(cfg.Ids, ",")
-		
+
 		// Validate that specified cache IDs exist
 		validIDs := make(map[string]bool)
 		for _, cache := range cacheClient.ListCaches() {
 			validIDs[cache.ID] = true
 		}
-		
+
 		var invalidIDs []string
 		for _, id := range cacheIDs {
 			if !validIDs[id] {
 				invalidIDs = append(invalidIDs, id)
 			}
 		}
-		
+
 		if len(invalidIDs) > 0 {
 			return nil, nil, fmt.Errorf("cache IDs not found in configuration: %s", strings.Join(invalidIDs, ", "))
 		}
diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go
index 2d32bf0a70..ef494b8499 100644
--- a/internal/cache/cache_test.go
+++ b/internal/cache/cache_test.go
@@ -294,33 +294,34 @@ func TestRestoreWithClient_EmptyCacheIDs(t *testing.T) {
 func TestLoadCacheConfiguration_Valid(t *testing.T) {
 	t.Parallel()
 
-	config := `- id: node
-  key: 'node-{{ checksum "package-lock.json" }}'
-  paths:
-    - node_modules
-- id: ruby
-  key: 'ruby-{{ checksum "Gemfile.lock" }}'
-  paths:
-    - vendor/bundle
+	config := `dependencies:
+  - id: node
+    key: 'node-{{ checksum "package-lock.json" }}'
+    paths:
+      - node_modules
+  - id: ruby
+    key: 'ruby-{{ checksum "Gemfile.lock" }}'
+    paths:
+      - vendor/bundle
 `
 	configFile := createTempCacheConfig(t, config)
 
-	caches, err := loadCacheConfiguration(configFile)
+	fileConfig, err := loadCacheConfiguration(configFile)
 	require.NoError(t, err)
-	require.Len(t, caches, 2)
-	require.Equal(t, "node", caches[0].ID)
-	require.Equal(t, "ruby", caches[1].ID)
+	require.Len(t, fileConfig.Dependencies, 2)
+	require.Equal(t, "node", fileConfig.Dependencies[0].ID)
+	require.Equal(t, "ruby", fileConfig.Dependencies[1].ID)
 }
 
 func TestLoadCacheConfiguration_InvalidYAML(t *testing.T) {
 	t.Parallel()
 
-	config := `---
-- id: node
-  key: test
-  paths
-    - invalid indentation here
-  : wrong syntax
+	config := `dependencies:
+  - id: node
+    key: test
+    paths
+      - invalid indentation here
+    : wrong syntax
 `
 	configFile := createTempCacheConfig(t, config)
 
@@ -342,9 +343,9 @@ func TestLoadCacheConfiguration_EmptyFile(t *testing.T) {
 
 	configFile := createTempCacheConfig(t, "")
 
-	caches, err := loadCacheConfiguration(configFile)
+	fileConfig, err := loadCacheConfiguration(configFile)
 	require.NoError(t, err)
-	require.Empty(t, caches)
+	require.Empty(t, fileConfig.Dependencies)
 }
 
 // Tests for setupCacheClient
@@ -353,14 +354,15 @@ func TestSetupCacheClient_InvalidCacheIDs(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()
 
-	config := `- id: cache1
-  key: 'test-key-1'
-  paths:
-    - path1
-- id: cache2
-  key: 'test-key-2'
-  paths:
-    - path2
+	config := `dependencies:
+  - id: cache1
+    key: 'test-key-1'
+    paths:
+      - path1
+  - id: cache2
+    key: 'test-key-2'
+    paths:
+      - path2
 `
 	configFile := createTempCacheConfig(t, config)
 
@@ -386,14 +388,15 @@ func TestSetupCacheClient_ValidCacheIDs(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()
 
-	config := `- id: cache1
-  key: 'test-key-1'
-  paths:
-    - path1
-- id: cache2
-  key: 'test-key-2'
-  paths:
-    - path2
+	config := `dependencies:
+  - id: cache1
+    key: 'test-key-1'
+    paths:
+      - path1
+  - id: cache2
+    key: 'test-key-2'
+    paths:
+      - path2
 `
 	configFile := createTempCacheConfig(t, config)
 
@@ -418,14 +421,15 @@ func TestSetupCacheClient_AllCaches(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()
 
-	config := `- id: cache1
-  key: 'test-key-1'
-  paths:
-    - path1
-- id: cache2
-  key: 'test-key-2'
-  paths:
-    - path2
+	config := `dependencies:
+  - id: cache1
+    key: 'test-key-1'
+    paths:
+      - path1
+  - id: cache2
+    key: 'test-key-2'
+    paths:
+      - path2
 `
 	configFile := createTempCacheConfig(t, config)
 

From 04c0aa1aeadb33180c81967e50b123d9db6e9254 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Mon, 27 Oct 2025 11:30:06 +1100
Subject: [PATCH 049/242] fix: as suggested switched to using StringSlice and
 cache_shared.go

---
 clicommand/cache_shared.go   | 62 ++++++++++++++++++++++++++++++++++++
 clicommand/global.go         | 59 ----------------------------------
 internal/cache/cache.go      | 25 +++++++--------
 internal/cache/cache_test.go |  6 ++--
 4 files changed, 77 insertions(+), 75 deletions(-)
 create mode 100644 clicommand/cache_shared.go

diff --git a/clicommand/cache_shared.go b/clicommand/cache_shared.go
new file mode 100644
index 0000000000..5d6b817201
--- /dev/null
+++ b/clicommand/cache_shared.go
@@ -0,0 +1,62 @@
+package clicommand
+
+import "github.com/urfave/cli"
+
+// CacheConfig includes cache-related shared options for easy inclusion across
+// cache command config structs (via embedding).
+type CacheConfig struct {
+	Ids             []string `cli:"ids"`
+	Registry        string   `cli:"registry"`
+	BucketURL       string   `cli:"bucket-url" validate:"required"`
+	Branch          string   `cli:"branch" validate:"required"`
+	Pipeline        string   `cli:"pipeline" validate:"required"`
+	Organization    string   `cli:"organization" validate:"required"`
+	CacheConfigFile string   `cli:"cache-config-file"`
+}
+
+func cacheFlags() []cli.Flag {
+	return []cli.Flag{
+		cli.StringSliceFlag{
+			Name:   "ids",
+			Value:  &cli.StringSlice{},
+			Usage:  "Comma-separated list of cache IDs (if empty, processes all caches)",
+			EnvVar: "BUILDKITE_CACHE_IDS",
+		},
+		cli.StringFlag{
+			Name:   "registry",
+			Value:  "~",
+			Usage:  "The slug of the cache registry to use, defaults to the default registry (~)",
+			EnvVar: "BUILDKITE_CACHE_REGISTRY",
+		},
+		cli.StringFlag{
+			Name:   "bucket-url",
+			Value:  "",
+			Usage:  "The URL of the bucket (e.g., s3://bucket-name)",
+			EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
+		},
+		cli.StringFlag{
+			Name:   "branch",
+			Value:  "",
+			Usage:  "Which branch should the cache be associated with",
+			EnvVar: "BUILDKITE_BRANCH",
+		},
+		cli.StringFlag{
+			Name:   "pipeline",
+			Value:  "",
+			Usage:  "The pipeline slug for this cache",
+			EnvVar: "BUILDKITE_PIPELINE_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "organization",
+			Value:  "",
+			Usage:  "The organization slug for this cache",
+			EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
+		},
+		cli.StringFlag{
+			Name:   "cache-config-file",
+			Value:  ".buildkite/cache.yml",
+			Usage:  "Path to the cache configuration YAML file",
+			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
+		},
+	}
+}
diff --git a/clicommand/global.go b/clicommand/global.go
index ce8490cdd4..c33231b421 100644
--- a/clicommand/global.go
+++ b/clicommand/global.go
@@ -182,18 +182,6 @@ type APIConfig struct {
 	NoHTTP2          bool   `cli:"no-http2"`
 }
 
-// CacheConfig includes cache-related shared options for easy inclusion across
-// cache command config structs (via embedding).
-type CacheConfig struct {
-	Ids             string `cli:"ids"`
-	Registry        string `cli:"registry"`
-	BucketURL       string `cli:"bucket-url" validate:"required"`
-	Branch          string `cli:"branch" validate:"required"`
-	Pipeline        string `cli:"pipeline" validate:"required"`
-	Organization    string `cli:"organization" validate:"required"`
-	CacheConfigFile string `cli:"cache-config-file"`
-}
-
 func globalFlags() []cli.Flag {
 	return []cli.Flag{
 		NoColorFlag,
@@ -214,53 +202,6 @@ func apiFlags() []cli.Flag {
 	}
 }
 
-func cacheFlags() []cli.Flag {
-	return []cli.Flag{
-		cli.StringFlag{
-			Name:   "ids",
-			Value:  "",
-			Usage:  "Comma-separated list of cache IDs (if empty, processes all caches)",
-			EnvVar: "BUILDKITE_CACHE_IDS",
-		},
-		cli.StringFlag{
-			Name:   "registry",
-			Value:  "~",
-			Usage:  "The slug of the cache registry to use, defaults to the default registry (~)",
-			EnvVar: "BUILDKITE_CACHE_REGISTRY",
-		},
-		cli.StringFlag{
-			Name:   "bucket-url",
-			Value:  "",
-			Usage:  "The URL of the bucket (e.g., s3://bucket-name)",
-			EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
-		},
-		cli.StringFlag{
-			Name:   "branch",
-			Value:  "",
-			Usage:  "Which branch should the cache be associated with",
-			EnvVar: "BUILDKITE_BRANCH",
-		},
-		cli.StringFlag{
-			Name:   "pipeline",
-			Value:  "",
-			Usage:  "The pipeline slug for this cache",
-			EnvVar: "BUILDKITE_PIPELINE_SLUG",
-		},
-		cli.StringFlag{
-			Name:   "organization",
-			Value:  "",
-			Usage:  "The organization slug for this cache",
-			EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
-		},
-		cli.StringFlag{
-			Name:   "cache-config-file",
-			Value:  ".buildkite/cache.yml",
-			Usage:  "Path to the cache configuration YAML file",
-			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
-		},
-	}
-}
-
 func CreateLogger(cfg any) logger.Logger {
 	var l logger.Logger
 	logFormat := "text"
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index fbd350f795..a53aa2d520 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -27,8 +27,8 @@ type Config struct {
 	Organization string
 	// CacheConfigFile is the path to the cache configuration YAML file
 	CacheConfigFile string
-	// Ids is a comma-separated list of cache IDs (if empty, processes all caches)
-	Ids string
+	// Ids is a list of cache IDs (if empty, processes all caches)
+	Ids []string
 	// APIEndpoint is the Agent API endpoint
 	APIEndpoint string
 	// APIToken is the access token used to authenticate
@@ -128,10 +128,7 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 	}
 
 	// Determine which cache IDs to process
-	var cacheIDs []string
-	if cfg.Ids != "" {
-		cacheIDs = strings.Split(cfg.Ids, ",")
-
+	if len(cfg.Ids) > 0 {
 		// Validate that specified cache IDs exist
 		validIDs := make(map[string]bool)
 		for _, cache := range cacheClient.ListCaches() {
@@ -139,7 +136,7 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 		}
 
 		var invalidIDs []string
-		for _, id := range cacheIDs {
+		for _, id := range cfg.Ids {
 			if !validIDs[id] {
 				invalidIDs = append(invalidIDs, id)
 			}
@@ -148,14 +145,16 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 		if len(invalidIDs) > 0 {
 			return nil, nil, fmt.Errorf("cache IDs not found in configuration: %s", strings.Join(invalidIDs, ", "))
 		}
-	} else {
-		// Process all caches configured in the client
-		for _, cache := range cacheClient.ListCaches() {
-			cacheIDs = append(cacheIDs, cache.ID)
-		}
+
+		return cacheClient, cfg.Ids, nil
+	}
+
+	var cacheDs []string
+	for _, cache := range cacheClient.ListCaches() {
+		cacheDs = append(cacheDs, cache.ID)
 	}
 
-	return cacheClient, cacheIDs, nil
+	return cacheClient, cacheDs, nil
 }
 
 // restoreWithClient performs the restore operation for the given cache IDs using the provided client
diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go
index ef494b8499..94fccc16ab 100644
--- a/internal/cache/cache_test.go
+++ b/internal/cache/cache_test.go
@@ -368,7 +368,7 @@ func TestSetupCacheClient_InvalidCacheIDs(t *testing.T) {
 
 	cfg := Config{
 		CacheConfigFile: configFile,
-		Ids:             "cache1,invalid1,cache2,invalid2",
+		Ids:             []string{"cache1", "invalid1", "cache2", "invalid2"},
 		BucketURL:       "s3://test-bucket",
 		Branch:          "main",
 		Pipeline:        "test-pipeline",
@@ -402,7 +402,7 @@ func TestSetupCacheClient_ValidCacheIDs(t *testing.T) {
 
 	cfg := Config{
 		CacheConfigFile: configFile,
-		Ids:             "cache1,cache2",
+		Ids:             []string{"cache1", "cache2"},
 		BucketURL:       "s3://test-bucket",
 		Branch:          "main",
 		Pipeline:        "test-pipeline",
@@ -435,7 +435,7 @@ func TestSetupCacheClient_AllCaches(t *testing.T) {
 
 	cfg := Config{
 		CacheConfigFile: configFile,
-		Ids:             "",
+		Ids:             []string{},
 		BucketURL:       "s3://test-bucket",
 		Branch:          "main",
 		Pipeline:        "test-pipeline",

From 58880856ef9bc035c1ee1433e2d906ea318ebbd5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Nov 2025 16:21:09 +0000
Subject: [PATCH 050/242] build(deps): bump golangci/golangci-lint

Bumps the container-images group with 1 update in the /.buildkite directory: golangci/golangci-lint.


Updates `golangci/golangci-lint` from v2.5-alpine to v2.6-alpine

---
updated-dependencies:
- dependency-name: golangci/golangci-lint
  dependency-version: v2.6-alpine
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-lint | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/Dockerfile-lint b/.buildkite/Dockerfile-lint
index 9f245a8121..b01dea690e 100644
--- a/.buildkite/Dockerfile-lint
+++ b/.buildkite/Dockerfile-lint
@@ -1 +1 @@
-FROM golangci/golangci-lint:v2.5-alpine@sha256:ac072ef3a8a6aa52c04630c68a7514e06be6f634d09d5975be60f2d53b484106
\ No newline at end of file
+FROM golangci/golangci-lint:v2.6-alpine@sha256:1e8c410818ea9f1f4176b89dd2d95776f07184a7d4a8bf88d25e553b04c1995a
\ No newline at end of file

From 2e9eef8e4c986d1dd72a84575310656d48a16bf5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Nov 2025 16:23:18 +0000
Subject: [PATCH 051/242] build(deps): bump the container-images group across 4
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `1df9b22` to `5922070`

Updates `buildkite/agent-base` from `3532569` to `b4d0e49`

Updates `buildkite/agent-base` from `7427ebc` to `03dbce9`

Updates `buildkite/agent-base` from `cac895b` to `82ce9ff`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 991e5cea95..fcfac1e707 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:3532569cd996e8f3f8985896e15f8d360c27424c56bfad985b7d26652f2bc15f
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:b4d0e49d234e37574c12225c579406f9eacbb8d0600489085bc1ccf97300a7a3
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 9c0641d274..ed5ca505d7 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:1df9b2216279b31581a50f160d5acd23f10bb38607b1a0725db4f506ec44beda
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:592207055fd1a54ea7c92f3277f352bf582c34f543ad0d9ebb4115d5a9ca5ad6
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 7a5f84aa26..91b8432c5f 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:7427ebca3c4ab49bc98614c336708ad3bc6340e43d12e382eafa0e5fdc07b56d
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:03dbce9ca376b7dc88550c9e8f8b1b865c2127757950ff4b04e1544ce893ceed
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index 5b7ae16645..67d601f549 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:cac895b78d2eb6fc8df165d19104cdfa0a95cd51b6f7df3d74cb8ae9517823f0
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:82ce9ff54c5a4a3bd196daa80fe4396b47b63a64c655ccf632df1707f74870bd
 ARG TARGETOS
 ARG TARGETARCH
 

From 2a95448832f3fa4c2eb08dabb69db06ff01782fb Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 27 Oct 2025 16:27:48 +1100
Subject: [PATCH 052/242] Upgrade to AWS Go SDK v2

---
 .gitignore                                  |   2 +-
 agent/ec2_tags.go                           |   9 +-
 go.mod                                      |  11 +-
 go.sum                                      |  17 +-
 internal/artifact/artifactory_downloader.go |   5 +-
 internal/artifact/download.go               |  16 +-
 internal/artifact/downloader.go             |  12 +-
 internal/artifact/s3.go                     | 223 +++++++++-----------
 internal/artifact/s3_downloader.go          |  18 +-
 internal/artifact/s3_test.go                | 121 +++++++++++
 internal/artifact/s3_uploader.go            |  53 +++--
 internal/artifact/s3_uploader_test.go       | 129 +++++++----
 internal/awslib/BUILD.bazel                 |   1 -
 internal/awslib/aws.go                      |  58 -----
 internal/awslib/awsv2.go                    |   3 +-
 15 files changed, 382 insertions(+), 296 deletions(-)
 create mode 100644 internal/artifact/s3_test.go
 delete mode 100644 internal/awslib/aws.go

diff --git a/.gitignore b/.gitignore
index 8c366472b7..3a567bc7a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-*.DS_STORE
+.DS_Store
 
 /tmp
 /pkg
diff --git a/agent/ec2_tags.go b/agent/ec2_tags.go
index 12bff3817e..b74be6fe77 100644
--- a/agent/ec2_tags.go
+++ b/agent/ec2_tags.go
@@ -5,8 +5,9 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/buildkite/agent/v3/internal/awslib"
+
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
@@ -15,7 +16,7 @@ import (
 type EC2Tags struct{}
 
 func (e EC2Tags) Get(ctx context.Context) (map[string]string, error) {
-	cfg, err := config.LoadDefaultConfig(ctx)
+	cfg, err := awslib.GetConfigV2(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("loading default AWS config: %w", err)
 	}
@@ -27,7 +28,7 @@ func (e EC2Tags) Get(ctx context.Context) (map[string]string, error) {
 		Path: "instance-id",
 	})
 	if err != nil {
-		return nil, fmt.Errorf("fetching metadata from IMDS: %v", err)
+		return nil, fmt.Errorf("fetching metadata from IMDS: %w", err)
 	}
 
 	instanceID, err := io.ReadAll(mdOut.Content)
@@ -51,7 +52,7 @@ func (e EC2Tags) Get(ctx context.Context) (map[string]string, error) {
 	}
 
 	// Collect the tags
-	tags := make(map[string]string)
+	tags := make(map[string]string, len(resp.Tags))
 	for _, tag := range resp.Tags {
 		tags[*tag.Key] = *tag.Value
 	}
diff --git a/go.mod b/go.mod
index cec1726f5d..bdd8b4dad5 100644
--- a/go.mod
+++ b/go.mod
@@ -11,12 +11,14 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
-	github.com/aws/aws-sdk-go v1.55.8
 	github.com/aws/aws-sdk-go-v2 v1.39.4
 	github.com/aws/aws-sdk-go-v2/config v1.31.15
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2
-	github.com/aws/aws-sdk-go-v2/service/kms v1.46.0
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.15
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1
+	github.com/aws/aws-sdk-go-v2/service/kms v1.46.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7
+	github.com/aws/smithy-go v1.23.1
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
 	github.com/buildkite/go-pipeline v0.16.0
@@ -101,11 +103,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect
-	github.com/aws/smithy-go v1.23.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/buildkite/test-engine-client v1.6.0 // indirect
@@ -134,7 +134,6 @@ require (
 	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.18.1 // indirect
diff --git a/go.sum b/go.sum
index 29645883e1..ce940ca23c 100644
--- a/go.sum
+++ b/go.sum
@@ -76,8 +76,6 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
-github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
 github.com/aws/aws-sdk-go-v2 v1.39.4 h1:qTsQKcdQPHnfGYBBs+Btl8QwxJeoWcOcPcixK90mRhg=
 github.com/aws/aws-sdk-go-v2 v1.39.4/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
@@ -88,6 +86,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.18.19 h1:Jc1zzwkSY1QbkEcLujwqRTXOdvW
 github.com/aws/aws-sdk-go-v2/credentials v1.18.19/go.mod h1:DIfQ9fAk5H0pGtnqfqkbSIzky82qYnGvh06ASQXXg6A=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 h1:X7X4YKb+c0rkI6d4uJ5tEMxXgCZ+jZ/D6mvkno8c8Uw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11/go.mod h1:EqM6vPZQsZHYvC4Cai35UDg/f5NCEU+vp0WfbVqVcZc=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.15 h1:OsZ2Sk84YUPJfi6BemhyMQyuR8/5tWu37WBMVUl8lJk=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.15/go.mod h1:CYZDjBMY+MyT+U+QmXw81GBiq+lhgM97kIMdDAJk+hg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE=
@@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11 h1:bKgSxk1TW//00PGQqYmrq83c+2myGidEclp+t9pPqVI=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11/go.mod h1:vrPYCQ6rFHL8jzQA8ppu3gWX18zxjLIDGTeqDxkBmSI=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2 h1:D8MCemFa8rt09x7o6Fkm2T7ThVbRPrD91R+LKhVEnVU=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2/go.mod h1:Q/kZ++hvhasMpQU37I7daQh07ZqTa++isjj1aPi4zvM=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1 h1:D8cBaI1TsIF+cbB8qPmiZWsMqGsbs1/e7qYQ0NMDscY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1/go.mod h1:DT0XByGaNaOff3CtLVmj3jKcMeVDfOj5DkLD39UPJY0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2 h1:DGFpGybmutVsCuF6vSuLZ25Vh55E3VmsnJmFfjeBx4M=
@@ -106,8 +106,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 h1:GpMf3z2K
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11/go.mod h1:6MZP3ZI4QQsgUCFTwMZA2V0sEriNQ8k2hmoHF3qjimQ=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 h1:weapBOuuFIBEQ9OX/NVW3tFQCvSutyjZYk/ga5jDLPo=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11/go.mod h1:3C1gN4FmIVLwYSh8etngUS+f1viY6nLCDVtZmrFbDy0=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.0 h1:vSXYridw+tT3AHuK1PWdJto2qEc30/wG/fm8dmCHHis=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.0/go.mod h1:YXPskkMuiMgp6qUG96NSTl7UpideOQT/Kx0u9Y1MKn0=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.2 h1:hz2rJseQXnVQtVbByFpeSCNJBBU7oFN+yenW4biJtvs=
+github.com/aws/aws-sdk-go-v2/service/kms v1.46.2/go.mod h1:E4ink1KCQgqIe2pHFD9E+b5CNXovm50rQbWFuh0cM+I=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 h1:Wer3W0GuaedWT7dv/PiWNZGSQFSTcBY2rZpbiUp5xcA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7/go.mod h1:UHKgcRSx8PVtvsc1Poxb/Co3PD3wL7P+f49P0+cWtuY=
 github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU=
@@ -241,10 +241,6 @@ github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
@@ -557,7 +553,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/artifact/artifactory_downloader.go b/internal/artifact/artifactory_downloader.go
index 2f1c93e664..f84b62a6f6 100644
--- a/internal/artifact/artifactory_downloader.go
+++ b/internal/artifact/artifactory_downloader.go
@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -66,8 +67,8 @@ func (d ArtifactoryDownloader) Start(ctx context.Context) error {
 	)
 
 	// create headers map
-	headers := map[string]string{
-		"Authorization": fmt.Sprintf("Basic %s", getBasicAuthHeader(username, password)),
+	headers := http.Header{
+		"Authorization": []string{fmt.Sprintf("Basic %s", getBasicAuthHeader(username, password))},
 	}
 
 	client := agenthttp.NewClient(
diff --git a/internal/artifact/download.go b/internal/artifact/download.go
index 1af8ac55c5..8fffca0dbf 100644
--- a/internal/artifact/download.go
+++ b/internal/artifact/download.go
@@ -1,6 +1,7 @@
 package artifact
 
 import (
+	"cmp"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -35,7 +36,10 @@ type DownloadConfig struct {
 	Destination string
 
 	// Optional Headers to append to the request
-	Headers map[string]string
+	Headers http.Header
+
+	// HTTP method to use (default is GET)
+	Method string
 
 	// The relative path that should be preserved in the download folder
 	Path string
@@ -130,7 +134,9 @@ func (d Download) try(ctx context.Context) error {
 	// Show a nice message that we're starting to download the file
 	d.logger.Debug("Downloading %s to %s", d.conf.URL, targetPath)
 
-	request, err := http.NewRequestWithContext(ctx, "GET", d.conf.URL, nil)
+	method := cmp.Or(d.conf.Method, http.MethodGet)
+
+	request, err := http.NewRequestWithContext(ctx, method, d.conf.URL, nil)
 	if err != nil {
 		return err
 	}
@@ -140,8 +146,10 @@ func (d Download) try(ctx context.Context) error {
 		request.Header.Add(headerUserAgent, version.UserAgent())
 	}
 
-	for k, v := range d.conf.Headers {
-		request.Header.Add(k, v)
+	for k, vs := range d.conf.Headers {
+		for _, v := range vs {
+			request.Header.Add(k, v)
+		}
 	}
 
 	// Start by downloading the file
diff --git a/internal/artifact/downloader.go b/internal/artifact/downloader.go
index f3ddfa0691..be876069ae 100644
--- a/internal/artifact/downloader.go
+++ b/internal/artifact/downloader.go
@@ -9,7 +9,7 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/buildkite/agent/v3/api"
 	"github.com/buildkite/agent/v3/internal/agenthttp"
 	"github.com/buildkite/agent/v3/logger"
@@ -85,7 +85,7 @@ func (a *Downloader) Download(ctx context.Context) error {
 
 	p := pool.New(pool.MaxConcurrencyLimit)
 	errors := []error{}
-	s3Clients, err := a.generateS3Clients(artifacts)
+	s3Clients, err := a.generateS3Clients(ctx, artifacts)
 	if err != nil {
 		return fmt.Errorf("failed to generate S3 clients for artifact upload: %w", err)
 	}
@@ -126,8 +126,8 @@ func (a *Downloader) Download(ctx context.Context) error {
 // We want to have as few S3 clients as possible, as creating them is kind of an expensive operation
 // But it's also theoretically possible that we'll have multiple artifacts with different S3 buckets, and each
 // S3Client only applies to one bucket, so we need to store the S3 clients in a map, one for each bucket
-func (a *Downloader) generateS3Clients(artifacts []*api.Artifact) (map[string]*s3.S3, error) {
-	s3Clients := map[string]*s3.S3{}
+func (a *Downloader) generateS3Clients(ctx context.Context, artifacts []*api.Artifact) (map[string]*s3.Client, error) {
+	s3Clients := map[string]*s3.Client{}
 
 	for _, artifact := range artifacts {
 		if !strings.HasPrefix(artifact.UploadDestination, "s3://") {
@@ -136,7 +136,7 @@ func (a *Downloader) generateS3Clients(artifacts []*api.Artifact) (map[string]*s
 
 		bucketName, _ := ParseS3Destination(artifact.UploadDestination)
 		if _, has := s3Clients[bucketName]; !has {
-			client, err := NewS3Client(a.logger, bucketName)
+			client, err := NewS3Client(ctx, a.logger, bucketName)
 			if err != nil {
 				return nil, fmt.Errorf("failed to create S3 client for bucket %s: %w", bucketName, err)
 			}
@@ -152,7 +152,7 @@ type downloader interface {
 	Start(context.Context) error
 }
 
-func (a *Downloader) createDownloader(artifact *api.Artifact, path, destination string, s3Clients map[string]*s3.S3) downloader {
+func (a *Downloader) createDownloader(artifact *api.Artifact, path, destination string, s3Clients map[string]*s3.Client) downloader {
 	// Handle downloading from S3, GS, RT, or Azure
 	switch {
 	case strings.HasPrefix(artifact.UploadDestination, "s3://"):
diff --git a/internal/artifact/s3.go b/internal/artifact/s3.go
index 5d58a24cca..978d5045a9 100644
--- a/internal/artifact/s3.go
+++ b/internal/artifact/s3.go
@@ -1,21 +1,22 @@
 package artifact
 
 import (
+	"cmp"
+	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
-	"github.com/aws/aws-sdk-go/aws/defaults"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/buildkite/agent/v3/internal/awslib"
 	"github.com/buildkite/agent/v3/logger"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	awshttp "github.com/aws/smithy-go/transport/http"
 )
 
 const (
@@ -24,168 +25,138 @@ const (
 )
 
 type buildkiteEnvProvider struct {
-	retrieved bool
+	next aws.CredentialsProvider
 }
 
-func (e *buildkiteEnvProvider) Retrieve() (credentials.Value, error) {
-	creds := credentials.Value{}
-
-	e.retrieved = false
-
-	creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY_ID")
-	if creds.AccessKeyID == "" {
-		creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY")
+func (p buildkiteEnvProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	creds := aws.Credentials{
+		CanExpire:       false,
+		AccessKeyID:     cmp.Or(os.Getenv("BUILDKITE_S3_ACCESS_KEY_ID"), os.Getenv("BUILDKITE_S3_ACCESS_KEY")),
+		SecretAccessKey: cmp.Or(os.Getenv("BUILDKITE_S3_SECRET_ACCESS_KEY"), os.Getenv("BUILDKITE_S3_SECRET_KEY")),
+		SessionToken:    os.Getenv("BUILDKITE_S3_SESSION_TOKEN"),
+		Source:          "buildkiteEnvProvider",
 	}
 
-	creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_ACCESS_KEY")
-	if creds.SecretAccessKey == "" {
-		creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_KEY")
+	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
+		// Fall back to the default provider.
+		return p.next.Retrieve(ctx)
 	}
 
-	creds.SessionToken = os.Getenv("BUILDKITE_S3_SESSION_TOKEN")
-
-	if creds.AccessKeyID == "" {
-		return credentials.Value{}, errors.New("BUILDKITE_S3_ACCESS_KEY_ID or BUILDKITE_S3_ACCESS_KEY not found in environment")
-	}
-
-	if creds.SecretAccessKey == "" {
-		return credentials.Value{}, errors.New("BUILDKITE_S3_SECRET_ACCESS_KEY or BUILDKITE_S3_SECRET_KEY not found in environment")
-	}
-
-	e.retrieved = true
 	return creds, nil
 }
 
-func (e *buildkiteEnvProvider) IsExpired() bool {
-	return !e.retrieved
-}
+func awsS3Config(ctx context.Context, region string) (aws.Config, error) {
+	profile := cmp.Or(os.Getenv("BUILDKITE_S3_PROFILE"), os.Getenv("AWS_PROFILE"))
 
-func awsS3Session(region string, l logger.Logger) (*session.Session, error) {
-	// Chicken and egg... but this is kinda how they do it in the sdk
-	sess, err := session.NewSession()
+	cfg, err := awslib.GetConfigV2(ctx,
+		config.WithRegion(region),
+		config.WithSharedConfigProfile(profile),
+	)
 	if err != nil {
-		return nil, err
+		return aws.Config{}, err
 	}
 
-	sess.Config.Region = aws.String(region)
-
-	sess.Config.Credentials = credentials.NewChainCredentials(
-		[]credentials.Provider{
-			&buildkiteEnvProvider{},
-			&credentials.EnvProvider{},
-			sharedCredentialsProvider(),
-			webIdentityRoleProvider(sess),
-			// EC2 and ECS meta-data providers
-			defaults.RemoteCredProvider(*sess.Config, sess.Handlers),
-		},
-	)
-
-	// An optional endpoint URL (hostname only or fully qualified URI)
-	// that overrides the default generated endpoint for a client.
-	// This is useful for S3-compatible servers like MinIO.
-	if endpoint := os.Getenv(s3EndpointEnvVar); endpoint != "" {
-		l.Debug("S3 session Endpoint from %s: %q", s3EndpointEnvVar, endpoint)
-		sess.Config.Endpoint = aws.String(endpoint)
+	// Wrap the default credential provider in buildkiteEnvProvider
+	// (Buildkite env vars get first bite of the AWS config cherry).
+	cfg.Credentials = buildkiteEnvProvider{next: cfg.Credentials}
 
-		// Configure the S3 client to use path-style addressing instead of the
-		// default DNS-style “virtual hosted bucket addressing”. See:
-		// - https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config.WithS3ForcePathStyle
-		// - https://github.com/aws/aws-sdk-go/blob/v1.44.181/aws/config.go#L118-L127
-		// This is useful for S3-compatible servers like MinIO when they're deployed
-		// without subdomain support.
-
-		// AWS CLI does this by default when a custom endpoint is specified [1] so
-		// we will too.
-		// [1]: https://github.com/aws/aws-cli/blob/2.9.18/awscli/botocore/args.py#L414-L417
-		l.Debug("S3 session S3ForcePathStyle=true because custom Endpoint specified")
-		sess.Config.S3ForcePathStyle = aws.Bool(true)
-	}
-
-	return sess, nil
+	return cfg, nil
 }
 
-func sharedCredentialsProvider() credentials.Provider {
-	// If empty SDK will default to environment variable "AWS_PROFILE"
-	// or "default" if environment variable is also not set.
-	awsProfile := os.Getenv("BUILDKITE_S3_PROFILE")
-
-	return &credentials.SharedCredentialsProvider{Profile: awsProfile}
-}
-
-func webIdentityRoleProvider(sess *session.Session) *stscreds.WebIdentityRoleProvider {
-	return stscreds.NewWebIdentityRoleProviderWithOptions(
-		sts.New(sess),
-		os.Getenv("AWS_ROLE_ARN"),
-		os.Getenv("AWS_ROLE_SESSION_NAME"),
-		stscreds.FetchTokenPath(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")),
-	)
-}
-
-func NewS3Client(l logger.Logger, bucket string) (*s3.S3, error) {
-	var sess *session.Session
+func NewS3Client(ctx context.Context, l logger.Logger, bucket string) (*s3.Client, error) {
+	var cfg aws.Config
 
 	regionHint := os.Getenv(regionHintEnvVar)
 	if regionHint != "" {
 		l.Debug("Using bucket region %q from environment variable %q", regionHint, regionHintEnvVar)
 		// If there is a region hint provided, we use it unconditionally
-		session, err := awsS3Session(regionHint, l)
+		tempCfg, err := awsS3Config(ctx, regionHint)
 		if err != nil {
-			return nil, fmt.Errorf("Could not load the AWS SDK config (%w)", err)
+			return nil, fmt.Errorf("could not load the AWS SDK config: %w", err)
 		}
-
-		sess = session
+		cfg = tempCfg
 	} else {
-		// Otherwise, use the current region (or a guess) to dynamically find
-		// where the bucket lives.
-		region, err := awslib.Region()
+		// Two-stage method. First, create a client for the current/default
+		// region, then use that to ask S3 where the bucket is.
+		tempCfg, err := awsS3Config(ctx, "")
 		if err != nil {
-			region = "us-east-1"
+			return nil, fmt.Errorf("could not load the AWS SDK config: %w", err)
 		}
+		cfg = tempCfg
 
-		l.Debug("Discovered current region as %q", region)
+		l.Debug("Discovered current region as %q", cfg.Region)
 
-		// Using the guess region, construct a session and ask that region where the
-		// bucket lives
-		session, err := awsS3Session(region, l)
-		if err != nil {
-			return nil, fmt.Errorf("Could not load the AWS SDK config (%w)", err)
-		}
+		client := s3.NewFromConfig(cfg)
 
-		bucketRegion, bucketRegionErr := s3manager.GetBucketRegion(aws.BackgroundContext(), session, bucket, region)
-		if bucketRegionErr == nil && bucketRegion != "" {
-			l.Debug("Discovered %q bucket region as %q", bucket, bucketRegion)
-			session.Config.Region = &bucketRegion
+		bucketRegion, err := manager.GetBucketRegion(ctx, client, bucket)
+		if err != nil || bucketRegion == "" {
+			l.Error(
+				"Could not discover region for bucket %q. Using the %q region as a fallback, if this is not correct configure a bucket region using the %q environment variable. (%v)",
+				bucket, cfg.Region, regionHintEnvVar, err,
+			)
 		} else {
-			l.Error("Could not discover region for bucket %q. Using the %q region as a fallback, if this is not correct configure a bucket region using the %q environment variable. (%v)", bucket, *session.Config.Region, regionHintEnvVar, err)
+			l.Debug("Discovered %q bucket region as %q", bucket, bucketRegion)
+			cfg.Region = bucketRegion
 		}
+	}
+
+	// An optional endpoint URL (hostname only or fully qualified URI)
+	// that overrides the default generated endpoint for a client.
+	// This is useful for S3-compatible servers like MinIO.
+	usePathStyle := false
+	if endpoint := os.Getenv(s3EndpointEnvVar); endpoint != "" {
+		l.Debug("S3 session Endpoint from %s: %q", s3EndpointEnvVar, endpoint)
+		cfg.BaseEndpoint = aws.String(endpoint)
+
+		// Configure the S3 client to use path-style addressing instead of the
+		// default DNS-style “virtual hosted bucket addressing”. See:
+		// - https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config.WithS3ForcePathStyle
+		// - https://github.com/aws/aws-sdk-go/blob/v1.44.181/aws/config.go#L118-L127
+		// This is useful for S3-compatible servers like MinIO when they're deployed
+		// without subdomain support.
 
-		sess = session
+		// AWS CLI does this by default when a custom endpoint is specified [1] so
+		// we will too.
+		// [1]: https://github.com/aws/aws-cli/blob/2.9.18/awscli/botocore/args.py#L414-L417
+		l.Debug("S3 UsePathStyle=true because custom Endpoint specified")
+		usePathStyle = true
 	}
 
-	l.Debug("Testing AWS S3 credentials for bucket %q in region %q...", bucket, *sess.Config.Region)
+	s3client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = usePathStyle
+	})
 
-	s3client := s3.New(sess)
+	l.Debug("Testing AWS S3 credentials for bucket %q in region %q...", bucket, cfg.Region)
 
 	// Test the authentication by trying to list the first 0 objects in the bucket.
-	_, err := s3client.ListObjects(&s3.ListObjectsInput{
+	_, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
 		Bucket:  aws.String(bucket),
-		MaxKeys: aws.Int64(0),
+		MaxKeys: aws.Int32(0),
 	})
-	if err != nil {
-		if errors.Is(err, credentials.ErrNoValidProvidersFoundInChain) {
-			hasProxy := os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != ""
-			hasNoProxyIdmsException := strings.Contains(os.Getenv("NO_PROXY"), "169.254.169.254")
 
-			errorTitle := "Could not authenticate with AWS S3 using any of the included credential providers."
+	if isAWSAuthFailure(err) {
+		hasProxy := os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != ""
+		hasNoProxyIdmsException := strings.Contains(os.Getenv("NO_PROXY"), "169.254.169.254")
 
-			if hasProxy && !hasNoProxyIdmsException {
-				return nil, fmt.Errorf("%s Your HTTP proxy settings do not grant a NO_PROXY=169.254.169.254 exemption for the instance metadata service, instance profile credentials may not be retrievable via your HTTP proxy.", errorTitle)
-			}
+		const errorTitle = "could not authenticate to AWS S3 using any of the included credential providers."
 
-			return nil, fmt.Errorf("%s You can authenticate by setting Buildkite environment variables (BUILDKITE_S3_ACCESS_KEY_ID, BUILDKITE_S3_SECRET_ACCESS_KEY, BUILDKITE_S3_PROFILE), AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_PROFILE), Web Identity environment variables (AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_WEB_IDENTITY_TOKEN_FILE), or if running on AWS EC2 ensuring network access to the EC2 Instance Metadata Service to use an instance profile’s IAM Role credentials.", errorTitle)
+		if hasProxy && !hasNoProxyIdmsException {
+			return nil, fmt.Errorf("%s Your HTTP proxy settings do not grant a NO_PROXY=169.254.169.254 exemption for the instance metadata service, instance profile credentials may not be retrievable via your HTTP proxy.", errorTitle)
 		}
-		return nil, fmt.Errorf("Could not s3:ListObjects in your AWS S3 bucket %q in region %q: (%s)", bucket, *sess.Config.Region, err.Error())
+
+		return nil, fmt.Errorf("%s You can authenticate by setting Buildkite environment variables (BUILDKITE_S3_ACCESS_KEY_ID, BUILDKITE_S3_SECRET_ACCESS_KEY, BUILDKITE_S3_PROFILE), AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_PROFILE), Web Identity environment variables (AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_WEB_IDENTITY_TOKEN_FILE), or if running on AWS EC2 ensuring network access to the EC2 Instance Metadata Service to use an instance profile’s IAM Role credentials.", errorTitle)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("could not s3:ListObjects in your AWS S3 bucket %q in region %q: %w", bucket, cfg.Region, err)
 	}
 
 	return s3client, nil
 }
+
+func isAWSAuthFailure(err error) bool {
+	var respErr *awshttp.ResponseError
+	if errors.As(err, &respErr) {
+		return respErr.HTTPStatusCode() == http.StatusForbidden
+	}
+	return false
+}
diff --git a/internal/artifact/s3_downloader.go b/internal/artifact/s3_downloader.go
index 828b0f52a5..cda362e2e9 100644
--- a/internal/artifact/s3_downloader.go
+++ b/internal/artifact/s3_downloader.go
@@ -6,15 +6,15 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/buildkite/agent/v3/internal/agenthttp"
 	"github.com/buildkite/agent/v3/logger"
 )
 
 type S3DownloaderConfig struct {
 	// The client for interacting with S3
-	S3Client *s3.S3
+	S3Client *s3.Client
 
 	// The S3 bucket name and the path, for example, s3://my-bucket-name/foo/bar
 	S3Path string
@@ -55,12 +55,14 @@ func (d S3Downloader) Start(ctx context.Context) error {
 		return fmt.Errorf("S3Downloader for %s: S3Client is nil", d.conf.S3Path)
 	}
 
-	req, _ := d.conf.S3Client.GetObjectRequest(&s3.GetObjectInput{
+	presigner := s3.NewPresignClient(d.conf.S3Client)
+
+	req, err := presigner.PresignGetObject(ctx, &s3.GetObjectInput{
 		Bucket: aws.String(d.BucketName()),
 		Key:    aws.String(d.BucketFileLocation()),
+	}, func(opts *s3.PresignOptions) {
+		opts.Expires = time.Duration(time.Hour)
 	})
-
-	signedURL, err := req.Presign(time.Hour)
 	if err != nil {
 		return fmt.Errorf("error pre-signing request: %w", err)
 	}
@@ -71,7 +73,9 @@ func (d S3Downloader) Start(ctx context.Context) error {
 		agenthttp.WithNoTimeout,
 	)
 	return NewDownload(d.logger, client, DownloadConfig{
-		URL:         signedURL,
+		URL:         req.URL,
+		Headers:     req.SignedHeader,
+		Method:      req.Method,
 		Path:        d.conf.Path,
 		Destination: d.conf.Destination,
 		Retries:     d.conf.Retries,
diff --git a/internal/artifact/s3_test.go b/internal/artifact/s3_test.go
new file mode 100644
index 0000000000..6cef607279
--- /dev/null
+++ b/internal/artifact/s3_test.go
@@ -0,0 +1,121 @@
+package artifact
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/google/go-cmp/cmp"
+)
+
+var fakeProviderCreds = aws.Credentials{
+	AccessKeyID:     "fakeProvider.AccessKeyID",
+	SecretAccessKey: "fakeProvider.SecretAccessKey",
+	SessionToken:    "fakeProvider.SessionToken",
+	Source:          "fakeProvider",
+	AccountID:       "fakeProvider.AccountID",
+}
+
+type fakeProvider struct{}
+
+func (f fakeProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	return fakeProviderCreds, nil
+}
+
+func TestBuildkiteEnvProvider(t *testing.T) {
+	// Manipulates env vars, so no parallel.
+	tests := []struct {
+		name string
+		env  map[string]string
+		want aws.Credentials
+	}{
+		{
+			name: "no env",
+			env:  map[string]string{},
+			want: fakeProviderCreds,
+		},
+		{
+			name: "bk vars 1",
+			env: map[string]string{
+				"BUILDKITE_S3_ACCESS_KEY_ID":     "buildkite s3 access key id",
+				"BUILDKITE_S3_SECRET_ACCESS_KEY": "buildkite s3 secret access key",
+				"BUILDKITE_S3_SESSION_TOKEN":     "buildkite s3 session token",
+			},
+			want: aws.Credentials{
+				CanExpire:       false,
+				AccessKeyID:     "buildkite s3 access key id",
+				SecretAccessKey: "buildkite s3 secret access key",
+				SessionToken:    "buildkite s3 session token",
+				Source:          "buildkiteEnvProvider",
+			},
+		},
+		{
+			name: "bk vars 2",
+			env: map[string]string{
+				"BUILDKITE_S3_ACCESS_KEY":    "buildkite s3 access key",
+				"BUILDKITE_S3_SECRET_KEY":    "buildkite s3 secret key",
+				"BUILDKITE_S3_SESSION_TOKEN": "buildkite s3 session token",
+			},
+			want: aws.Credentials{
+				CanExpire:       false,
+				AccessKeyID:     "buildkite s3 access key",
+				SecretAccessKey: "buildkite s3 secret key",
+				SessionToken:    "buildkite s3 session token",
+				Source:          "buildkiteEnvProvider",
+			},
+		},
+		{
+			name: "session token missing is OK",
+			env: map[string]string{
+				"BUILDKITE_S3_ACCESS_KEY_ID":     "buildkite s3 access key id",
+				"BUILDKITE_S3_SECRET_ACCESS_KEY": "buildkite s3 secret access key",
+			},
+			want: aws.Credentials{
+				CanExpire:       false,
+				AccessKeyID:     "buildkite s3 access key id",
+				SecretAccessKey: "buildkite s3 secret access key",
+				Source:          "buildkiteEnvProvider",
+			},
+		},
+		{
+			name: "access key ID missing fallback",
+			env: map[string]string{
+				"BUILDKITE_S3_SECRET_ACCESS_KEY": "buildkite s3 secret access key",
+				"BUILDKITE_S3_SESSION_TOKEN":     "buildkite s3 session token",
+			},
+			want: fakeProviderCreds,
+		},
+		{
+			name: "secret access key missing fallback",
+			env: map[string]string{
+				"BUILDKITE_S3_ACCESS_KEY_ID": "buildkite s3 access key id",
+				"BUILDKITE_S3_SESSION_TOKEN": "buildkite s3 session token",
+			},
+			want: fakeProviderCreds,
+		},
+	}
+
+	ctx := t.Context()
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for n, v := range test.env {
+				if err := os.Setenv(n, v); err != nil {
+					t.Fatalf("os.Setenv(%q, %q) = %v", n, v, err)
+				}
+				t.Cleanup(func() {
+					os.Unsetenv(n) //nolint:errcheck // Best-effort cleanup
+				})
+			}
+
+			got, err := buildkiteEnvProvider{next: fakeProvider{}}.Retrieve(ctx)
+			if err != nil {
+				t.Errorf("env=%v buildkiteEnvProvider{}.Retrieve(ctx) error = %v", test.env, err)
+			}
+			if diff := cmp.Diff(got, test.want); diff != "" {
+				t.Errorf("env=%v buildkiteEnvProvider{}.Retrieve(ctx) diff (-got +want):\n%s", test.env, diff)
+			}
+		})
+	}
+}
diff --git a/internal/artifact/s3_uploader.go b/internal/artifact/s3_uploader.go
index ac5ccdf9a2..34ebf78d4f 100644
--- a/internal/artifact/s3_uploader.go
+++ b/internal/artifact/s3_uploader.go
@@ -5,12 +5,14 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/buildkite/agent/v3/api"
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/roko"
@@ -30,7 +32,7 @@ type S3Uploader struct {
 	BucketName string
 
 	// The s3 client to use
-	client *s3.S3
+	client *s3.Client
 
 	// The configuration
 	conf S3UploaderConfig
@@ -48,9 +50,9 @@ func NewS3Uploader(ctx context.Context, l logger.Logger, c S3UploaderConfig) (*S
 		roko.WithJitter(),
 	)
 
-	s3Client, err := roko.DoFunc(ctx, r, func(*roko.Retrier) (*s3.S3, error) {
+	s3Client, err := roko.DoFunc(ctx, r, func(*roko.Retrier) (*s3.Client, error) {
 		// Initialize the s3 client, and authenticate it
-		return NewS3Client(l, bucketName)
+		return NewS3Client(ctx, l, bucketName)
 	})
 
 	if err != nil {
@@ -107,14 +109,14 @@ func (u *s3UploaderWork) Description() string {
 	return singleUnitDescription(u.artifact)
 }
 
-func (u *s3UploaderWork) DoWork(context.Context) (*api.ArtifactPartETag, error) {
+func (u *s3UploaderWork) DoWork(ctx context.Context) (*api.ArtifactPartETag, error) {
 	permission, err := u.resolvePermission()
 	if err != nil {
 		return nil, err
 	}
 
 	// Create an uploader with the session and default options
-	uploader := s3manager.NewUploaderWithClient(u.client)
+	uploader := manager.NewUploader(u.client)
 
 	// Open file from filesystem
 	u.logger.Debug("Reading file %q", u.artifact.AbsolutePath)
@@ -126,19 +128,19 @@ func (u *s3UploaderWork) DoWork(context.Context) (*api.ArtifactPartETag, error)
 	// Upload the file to S3.
 	u.logger.Debug("Uploading %q to bucket with permission %q", u.artifactPath(u.artifact), permission)
 
-	params := &s3manager.UploadInput{
+	params := &s3.PutObjectInput{
 		Bucket:      aws.String(u.BucketName),
 		Key:         aws.String(u.artifactPath(u.artifact)),
 		ContentType: aws.String(u.artifact.ContentType),
-		ACL:         aws.String(permission),
+		ACL:         permission,
 		Body:        f,
 	}
 	// if enabled we assign the sse configuration
 	if u.serverSideEncryptionEnabled() {
-		params.ServerSideEncryption = aws.String("AES256")
+		params.ServerSideEncryption = types.ServerSideEncryptionAes256
 	}
 
-	_, err = uploader.Upload(params)
+	_, err = uploader.Upload(ctx, params)
 	return nil, err
 }
 
@@ -148,20 +150,19 @@ func (u *S3Uploader) artifactPath(artifact *api.Artifact) string {
 	return strings.Join(parts, "/")
 }
 
-func (u *S3Uploader) resolvePermission() (string, error) {
-	permission := "public-read"
-	if os.Getenv("BUILDKITE_S3_ACL") != "" {
-		permission = os.Getenv("BUILDKITE_S3_ACL")
-	} else if os.Getenv("AWS_S3_ACL") != "" {
-		permission = os.Getenv("AWS_S3_ACL")
+func (u *S3Uploader) resolvePermission() (types.ObjectCannedACL, error) {
+	permission := types.ObjectCannedACLPublicRead
+	switch {
+	case os.Getenv("BUILDKITE_S3_ACL") != "":
+		permission = types.ObjectCannedACL(os.Getenv("BUILDKITE_S3_ACL"))
+	case os.Getenv("AWS_S3_ACL") != "":
+		permission = types.ObjectCannedACL(os.Getenv("AWS_S3_ACL"))
 	}
 
-	switch permission {
-	case "private", "public-read", "public-read-write", "authenticated-read", "bucket-owner-read", "bucket-owner-full-control":
-		return permission, nil
-	default:
-		return "", fmt.Errorf("Invalid S3 ACL value: `%s`", permission)
+	if !slices.Contains(permission.Values(), permission) {
+		return "", invalidACLError(permission)
 	}
+	return permission, nil
 }
 
 // is encryption at rest enabled for artifacts to satisfy basic security requirements
@@ -174,3 +175,9 @@ func (u *S3Uploader) serverSideEncryptionEnabled() bool {
 		return false
 	}
 }
+
+type invalidACLError string
+
+func (e invalidACLError) Error() string {
+	return fmt.Sprintf("invalid S3 ACL value: %q", string(e))
+}
diff --git a/internal/artifact/s3_uploader_test.go b/internal/artifact/s3_uploader_test.go
index 9c97446d25..b04f275a39 100644
--- a/internal/artifact/s3_uploader_test.go
+++ b/internal/artifact/s3_uploader_test.go
@@ -4,7 +4,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
 func TestParseS3Destination(t *testing.T) {
@@ -35,64 +35,101 @@ func TestParseS3Destination(t *testing.T) {
 }
 
 func TestResolveServerSideEncryptionConfig(t *testing.T) {
-
-	assert := require.New(t)
-
-	for _, tc := range []struct {
-		ServerSideEncryptionConfig string
-		ExpectedResult             bool
+	// Test manipulates env vars, so not parallel.
+	tests := []struct {
+		config string
+		want   bool
 	}{
-		{"True", true},
-		{"falsE", false},
-		{"lol", false},
-	} {
+		{config: "True", want: true},
+		{config: "falsE", want: false},
+		{config: "lol", want: false},
+	}
+
+	for _, test := range tests {
 		uploader := &S3Uploader{}
-		if err := os.Setenv("BUILDKITE_S3_SSE_ENABLED", tc.ServerSideEncryptionConfig); err != nil {
-			t.Fatalf("os.Setenv(BUILDKITE_S3_SSE_ENABLED, tc.ServerSideEncryptionConfig) = %v", err)
+		if err := os.Setenv("BUILDKITE_S3_SSE_ENABLED", test.config); err != nil {
+			t.Fatalf("os.Setenv(BUILDKITE_S3_SSE_ENABLED, %q) = %v", test.config, err)
 		}
 		t.Cleanup(func() {
 			os.Unsetenv("BUILDKITE_S3_SSE_ENABLED") //nolint:errcheck // Best-effort cleanup.
 		})
 
-		config := uploader.serverSideEncryptionEnabled()
-
-		assert.Equal(tc.ExpectedResult, config)
-
+		if got := uploader.serverSideEncryptionEnabled(); got != test.want {
+			t.Errorf("BUILDKITE_S3_SSE_ENABLED=%q uploader.serverSideEncryptionEnabled() = %t, want %t", test.config, got, test.want)
+		}
 	}
 }
 
 func TestResolvePermission(t *testing.T) {
-
-	assert := require.New(t)
-	for _, tc := range []struct {
-		Permission     string
-		ExpectedResult string
-		ShouldErr      bool
+	// Test manipulates env vars, so not parallel.
+	tests := []struct {
+		permission string
+		want       types.ObjectCannedACL
 	}{
-		{"", "public-read", false},
-		{"private", "private", false},
-		{"public-read", "public-read", false},
-		{"public-read-write", "public-read-write", false},
-		{"authenticated-read", "authenticated-read", false},
-		{"bucket-owner-read", "bucket-owner-read", false},
-		{"bucket-owner-full-control", "bucket-owner-full-control", false},
-		{"foo", "", true},
-	} {
-		uploader := &S3Uploader{}
-		if err := os.Setenv("BUILDKITE_S3_ACL", tc.Permission); err != nil {
-			t.Fatalf("os.Setenv(BUILDKITE_S3_ACL, tc.Permission) = %v", err)
-		}
-		t.Cleanup(func() {
-			os.Unsetenv("BUILDKITE_S3_ACL") //nolint:errcheck // Best-effort cleanup.
+		{
+			permission: "",
+			want:       types.ObjectCannedACLPublicRead,
+		},
+		{
+			permission: "private",
+			want:       types.ObjectCannedACLPrivate,
+		},
+		{
+			permission: "public-read",
+			want:       types.ObjectCannedACLPublicRead,
+		},
+		{
+			permission: "public-read-write",
+			want:       types.ObjectCannedACLPublicReadWrite,
+		},
+		{
+			permission: "authenticated-read",
+			want:       types.ObjectCannedACLAuthenticatedRead,
+		},
+		{
+			permission: "bucket-owner-read",
+			want:       types.ObjectCannedACLBucketOwnerRead,
+		},
+		{
+			permission: "bucket-owner-full-control",
+			want:       types.ObjectCannedACLBucketOwnerFullControl,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.permission, func(t *testing.T) {
+			// Test manipulates env vars, so not parallel.
+			uploader := &S3Uploader{}
+			if err := os.Setenv("BUILDKITE_S3_ACL", test.permission); err != nil {
+				t.Fatalf("os.Setenv(BUILDKITE_S3_ACL, %q) = %v", test.permission, err)
+			}
+			t.Cleanup(func() {
+				os.Unsetenv("BUILDKITE_S3_ACL") //nolint:errcheck // Best-effort cleanup.
+			})
+			got, err := uploader.resolvePermission()
+			if err != nil {
+				t.Fatalf("BUILDKITE_S3_ACL=%q uploader.resolvePermission() error = %v", test.permission, err)
+			}
+			if got != test.want {
+				t.Errorf("BUILDKITE_S3_ACL=%q uploader.resolvePermission() = %v, want %v", test.permission, got, test.want)
+			}
 		})
-		config, err := uploader.resolvePermission()
+	}
+}
 
-		// if it should error we just look at the error
-		if tc.ShouldErr {
-			assert.Error(err)
-		} else {
-			assert.Nil(err)
-			assert.Equal(tc.ExpectedResult, config)
-		}
+func TestResolvePermission_InvalidPermission(t *testing.T) {
+	// Test manipulates env vars, so not parallel.
+	permission := "foo"
+	uploader := &S3Uploader{}
+	if err := os.Setenv("BUILDKITE_S3_ACL", permission); err != nil {
+		t.Fatalf("os.Setenv(BUILDKITE_S3_ACL, %q) = %v", permission, err)
+	}
+	t.Cleanup(func() {
+		os.Unsetenv("BUILDKITE_S3_ACL") //nolint:errcheck // Best-effort cleanup.
+	})
+	wantErr := invalidACLError(permission)
+	_, err := uploader.resolvePermission()
+	if err != wantErr {
+		t.Errorf("BUILDKITE_S3_ACL=%q uploader.resolvePermission() error = %v, want %v", permission, err, wantErr)
 	}
 }
diff --git a/internal/awslib/BUILD.bazel b/internal/awslib/BUILD.bazel
index 03953cfa91..a0e095acd1 100644
--- a/internal/awslib/BUILD.bazel
+++ b/internal/awslib/BUILD.bazel
@@ -3,7 +3,6 @@ load("@rules_go//go:def.bzl", "go_library")
 go_library(
     name = "awslib",
     srcs = [
-        "aws.go",
         "awsv2.go",
     ],
     importpath = "github.com/buildkite/agent/v3/internal/awslib",
diff --git a/internal/awslib/aws.go b/internal/awslib/aws.go
deleted file mode 100644
index 343ce71b75..0000000000
--- a/internal/awslib/aws.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package awslib
-
-import (
-	"os"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
-	"github.com/aws/aws-sdk-go/aws/session"
-)
-
-var awsSess *session.Session
-
-// Region detects the current AWS region where the agent might be running
-// using the env but also the local instance metadata if available.
-func Region() (string, error) {
-	if r := os.Getenv("AWS_REGION"); r != "" {
-		return r, nil
-	}
-
-	if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" {
-		return r, nil
-	}
-
-	// The metadata service seems to want a session
-	sess, err := session.NewSession(&aws.Config{
-		Region: aws.String("us-east-1"),
-	})
-	if err != nil {
-		return "", err
-	}
-
-	meta := ec2metadata.New(sess)
-	if meta.Available() {
-		return meta.Region()
-	}
-
-	return "", aws.ErrMissingRegion
-}
-
-// Session returns a singleton Session, creating a new Session for the
-// current region if not created previously.
-func Session() (*session.Session, error) {
-	region, err := Region()
-	if err != nil {
-		return nil, err
-	}
-
-	if awsSess == nil {
-		awsSess, err = session.NewSession(&aws.Config{
-			Region: aws.String(region),
-		})
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return awsSess, nil
-}
diff --git a/internal/awslib/awsv2.go b/internal/awslib/awsv2.go
index 2eec722df3..3e8ebf9b8b 100644
--- a/internal/awslib/awsv2.go
+++ b/internal/awslib/awsv2.go
@@ -9,7 +9,8 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 )
 
-// GetConfigV2 creates a new AWS SDK v2 config.
+// GetConfigV2 creates a new AWS SDK v2 config that uses the current region from
+// IMDS, if not otherwise provided.
 func GetConfigV2(ctx context.Context, optFns ...func(*config.LoadOptions) error) (cfg aws.Config, err error) {
 	cfg, err = config.LoadDefaultConfig(ctx, optFns...)
 	if err != nil {

From 6893d38c44d8b0acc7c9f2939443995db53cdbe9 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Tue, 4 Nov 2025 16:06:45 +1100
Subject: [PATCH 053/242] Update error message for hashing failures

---
 internal/artifact/uploader.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/internal/artifact/uploader.go b/internal/artifact/uploader.go
index d54e95afea..283c3fd063 100644
--- a/internal/artifact/uploader.go
+++ b/internal/artifact/uploader.go
@@ -356,11 +356,10 @@ func (a *Uploader) build(path string, absolutePath string) (*api.Artifact, error
 	defer file.Close() //nolint:errcheck // File is only open for read.
 
 	// Generate a SHA-1 and SHA-256 checksums for the file.
-	// Writing to hashes never errors, but reading from the file might.
 	hash1, hash256 := sha1.New(), sha256.New()
 	size, err := io.Copy(io.MultiWriter(hash1, hash256), file)
 	if err != nil {
-		return nil, fmt.Errorf("reading contents of %s: %w", absolutePath, err)
+		return nil, fmt.Errorf("hashing artifact file %s: %w", absolutePath, err)
 	}
 	sha1sum := fmt.Sprintf("%040x", hash1.Sum(nil))
 	sha256sum := fmt.Sprintf("%064x", hash256.Sum(nil))

From 1e7deb3144190f1625bdf9ac3e27cbc8b23143bd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 4 Nov 2025 23:32:19 +0000
Subject: [PATCH 054/242] build(deps): bump the cloud-providers group across 1
 directory with 9 updates

Bumps the cloud-providers group with 6 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.39.4` | `1.39.6` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.15` | `1.31.17` |
| [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.19.15` | `1.20.3` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.258.1` | `1.261.1` |
| [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) | `1.46.2` | `1.47.1` |
| [google.golang.org/api](https://github.com/googleapis/google-api-go-client) | `0.253.0` | `0.255.0` |



Updates `github.com/aws/aws-sdk-go-v2` from 1.39.4 to 1.39.6
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.39.4...v1.39.6)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.15 to 1.31.17
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.15...config/v1.31.17)

Updates `github.com/aws/aws-sdk-go-v2/feature/ec2/imds` from 1.18.11 to 1.18.13
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.13/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.11...config/v1.18.13)

Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.19.15 to 1.20.3
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/sqs/v1.19.15...v1.20.3)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.258.1 to 1.261.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.258.1...service/ec2/v1.261.1)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.46.2 to 1.47.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/rds/v1.46.2...service/s3/v1.47.1)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.88.7 to 1.89.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.88.7...service/s3/v1.89.2)

Updates `github.com/aws/smithy-go` from 1.23.1 to 1.23.2
- [Release notes](https://github.com/aws/smithy-go/releases)
- [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/aws/smithy-go/compare/v1.23.1...v1.23.2)

Updates `google.golang.org/api` from 0.253.0 to 0.255.0
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.253.0...v0.255.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2
  dependency-version: 1.39.6
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.31.17
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/ec2/imds
  dependency-version: 1.18.13
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-version: 1.20.3
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.261.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.47.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-version: 1.89.2
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/smithy-go
  dependency-version: 1.23.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: google.golang.org/api
  dependency-version: 0.255.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 44 ++++++++++++++---------------
 go.sum | 88 +++++++++++++++++++++++++++++-----------------------------
 2 files changed, 66 insertions(+), 66 deletions(-)

diff --git a/go.mod b/go.mod
index bdd8b4dad5..ea092d7244 100644
--- a/go.mod
+++ b/go.mod
@@ -11,14 +11,14 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
-	github.com/aws/aws-sdk-go-v2 v1.39.4
-	github.com/aws/aws-sdk-go-v2/config v1.31.15
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.15
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1
-	github.com/aws/aws-sdk-go-v2/service/kms v1.46.2
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7
-	github.com/aws/smithy-go v1.23.1
+	github.com/aws/aws-sdk-go-v2 v1.39.6
+	github.com/aws/aws-sdk-go-v2/config v1.31.17
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.261.1
+	github.com/aws/aws-sdk-go-v2/service/kms v1.47.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.89.2
+	github.com/aws/smithy-go v1.23.2
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
 	github.com/buildkite/go-pipeline v0.16.0
@@ -60,7 +60,7 @@ require (
 	golang.org/x/sync v0.17.0
 	golang.org/x/sys v0.37.0
 	golang.org/x/term v0.36.0
-	google.golang.org/api v0.253.0
+	google.golang.org/api v0.255.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
@@ -93,19 +93,19 @@ require (
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/buildkite/test-engine-client v1.6.0 // indirect
@@ -194,7 +194,7 @@ require (
 	golang.org/x/tools v0.37.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
 	google.golang.org/grpc v1.76.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/go.sum b/go.sum
index ce940ca23c..7969af33fc 100644
--- a/go.sum
+++ b/go.sum
@@ -76,48 +76,48 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/aws/aws-sdk-go-v2 v1.39.4 h1:qTsQKcdQPHnfGYBBs+Btl8QwxJeoWcOcPcixK90mRhg=
-github.com/aws/aws-sdk-go-v2 v1.39.4/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
-github.com/aws/aws-sdk-go-v2/config v1.31.15 h1:gE3M4xuNXfC/9bG4hyowGm/35uQTi7bUKeYs5e/6uvU=
-github.com/aws/aws-sdk-go-v2/config v1.31.15/go.mod h1:HvnvGJoE2I95KAIW8kkWVPJ4XhdrlvwJpV6pEzFQa8o=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.19 h1:Jc1zzwkSY1QbkEcLujwqRTXOdvW8ppND3jRBb/VhBQc=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.19/go.mod h1:DIfQ9fAk5H0pGtnqfqkbSIzky82qYnGvh06ASQXXg6A=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 h1:X7X4YKb+c0rkI6d4uJ5tEMxXgCZ+jZ/D6mvkno8c8Uw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11/go.mod h1:EqM6vPZQsZHYvC4Cai35UDg/f5NCEU+vp0WfbVqVcZc=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.15 h1:OsZ2Sk84YUPJfi6BemhyMQyuR8/5tWu37WBMVUl8lJk=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.15/go.mod h1:CYZDjBMY+MyT+U+QmXw81GBiq+lhgM97kIMdDAJk+hg=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11/go.mod h1:7bUb2sSr2MZ3M/N+VyETLTQtInemHXb/Fl3s8CLzm0Y=
+github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk=
+github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
+github.com/aws/aws-sdk-go-v2/config v1.31.17 h1:QFl8lL6RgakNK86vusim14P2k8BFSxjvUkcWLDjgz9Y=
+github.com/aws/aws-sdk-go-v2/config v1.31.17/go.mod h1:V8P7ILjp/Uef/aX8TjGk6OHZN6IKPM5YW6S78QnRD5c=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.21 h1:56HGpsgnmD+2/KpG0ikvvR8+3v3COCwaF4r+oWwOeNA=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.21/go.mod h1:3YELwedmQbw7cXNaII2Wywd+YY58AmLPwX4LzARgmmA=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 h1:4GNV1lhyELGjMz5ILMRxDvxvOaeo3Ux9Z69S1EgVMMQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3/go.mod h1:br7KA6edAAqDGUYJ+zVVPAyMrPhnN+zdt17yTUT6FPw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11 h1:bKgSxk1TW//00PGQqYmrq83c+2myGidEclp+t9pPqVI=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11/go.mod h1:vrPYCQ6rFHL8jzQA8ppu3gWX18zxjLIDGTeqDxkBmSI=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1 h1:D8cBaI1TsIF+cbB8qPmiZWsMqGsbs1/e7qYQ0NMDscY=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1/go.mod h1:DT0XByGaNaOff3CtLVmj3jKcMeVDfOj5DkLD39UPJY0=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2 h1:DGFpGybmutVsCuF6vSuLZ25Vh55E3VmsnJmFfjeBx4M=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2/go.mod h1:hm/wU1HDvXCFEDzOLorQnZZ/CVvPXvWEmHMSmqgQRuA=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 h1:GpMf3z2KJa4RnJ0ew3Hac+hRFYLZ9DDjfgXjuW+pB54=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11/go.mod h1:6MZP3ZI4QQsgUCFTwMZA2V0sEriNQ8k2hmoHF3qjimQ=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 h1:weapBOuuFIBEQ9OX/NVW3tFQCvSutyjZYk/ga5jDLPo=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11/go.mod h1:3C1gN4FmIVLwYSh8etngUS+f1viY6nLCDVtZmrFbDy0=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.2 h1:hz2rJseQXnVQtVbByFpeSCNJBBU7oFN+yenW4biJtvs=
-github.com/aws/aws-sdk-go-v2/service/kms v1.46.2/go.mod h1:E4ink1KCQgqIe2pHFD9E+b5CNXovm50rQbWFuh0cM+I=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 h1:Wer3W0GuaedWT7dv/PiWNZGSQFSTcBY2rZpbiUp5xcA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7/go.mod h1:UHKgcRSx8PVtvsc1Poxb/Co3PD3wL7P+f49P0+cWtuY=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.29.8/go.mod h1:mbef/pgKhtKRwrigPPs7SSSKZgytzP8PQ6P6JAAdqyM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 h1:S5GuJZpYxE0lKeMHKn+BRTz6PTFpgThyJ+5mYfux7BM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3/go.mod h1:X4OF+BTd7HIb3L+tc4UlWHVrpgwZZIVENU15pRDVTI0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 h1:Ekml5vGg6sHSZLZJQJagefnVe6PmqC2oiRkBq4F7fU0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.38.9/go.mod h1:/e15V+o1zFHWdH3u7lpI3rVBcxszktIKuHKCY2/py+k=
-github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
-github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.261.1 h1:1U9Is5mKIIgi8xlIZi5Apuq/SHnQrKUBmmRm4tmgLYM=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.261.1/go.mod h1:NDdDLLW5PtLLXN661gKcvJvqAH5OBXsfhMlmKVu1/pY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
+github.com/aws/aws-sdk-go-v2/service/kms v1.47.1 h1:6+C0RoGF4HJQALrsecOXN7cm/l5rgNHCw2xbcvFgpH4=
+github.com/aws/aws-sdk-go-v2/service/kms v1.47.1/go.mod h1:VJcNH6BLr+3VJwinRKdotLOMglHO8mIKlD3ea5c7hbw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.89.2 h1:xgBWsgaeUESl8A8k80p6yBdexMWDVeiDmJ/pkjohJ7c=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.89.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 h1:mLlUgHn02ue8whiR4BmxxGJLR2gwU6s6ZzJ5wDamBUs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.39.1/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
+github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
+github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE=
 github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY=
 github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
@@ -532,16 +532,16 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.253.0 h1:apU86Eq9Q2eQco3NsUYFpVTfy7DwemojL7LmbAj7g/I=
-google.golang.org/api v0.253.0/go.mod h1:PX09ad0r/4du83vZVAaGg7OaeyGnaUmT/CYPNvtLCbw=
+google.golang.org/api v0.255.0 h1:OaF+IbRwOottVCYV2wZan7KUq7UeNUQn1BcPc4K7lE4=
+google.golang.org/api v0.255.0/go.mod h1:d1/EtvCLdtiWEV4rAEHDHGh2bCnqsWhw+M8y2ECN4a8=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
 google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 h1:8XJ4pajGwOlasW+L13MnEGA8W4115jJySQtVfS2/IBU=
 google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4/go.mod h1:NnuHhy+bxcg30o7FnVAZbXsPHUDQ9qKWAQKCD7VxFtk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f h1:1FTH6cpXFsENbPR5Bu8NQddPSaUUE6NA2XdZdDSAJK4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
 google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
 google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=

From e8f587ac5025468b267741821e0d003a2c7d2a4b Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 5 Nov 2025 13:32:03 +1100
Subject: [PATCH 055/242] Bump version and changelog for v3.111.0

---
 CHANGELOG.md    | 42 ++++++++++++++++++++++++++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b41d8dd5a2..72bcf9ac7a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,48 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.111.0](https://github.com/buildkite/agent/tree/v3.111.0) (2025-11-05)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.110.0...v3.111.0)
+
+> [!WARNING]
+> If you use a custom S3 bucket for artifacts, this applies to you.
+>
+> As part of updating to AWS Go SDK v2, the "credential chain" for providing
+> authentication credentials to access artifacts in custom S3 buckets, is now
+> more standard. The existing `BUILDKITE_S3_` env vars are still available and
+> take precedence, but when these are not set, the AWS-default mechanisms are
+> used as provided by the SDK, with as few customisations as possible.
+>
+> This means additional ways to pass credentials to the AWS S3 client may be
+> accepted, and where multiple credentials are available, the precedence may
+> have changed (to match what the AWS SDK expects by default).
+>
+> Because of this, and the number of combinations of different ways to provide
+> credentials, this change may inadvertently break pipelines using custom S3
+> buckets for artifacts. Please reach out to support@buildkite.com or raise
+> issues in GitHub if this impacts you!
+
+### Added
+- Add cache save and restore using github.com/buildkite/zstash [#3551](https://github.com/buildkite/agent/pull/3551) (@wolfeidau)
+
+### Changed
+- Upgrade to AWS Go SDK v2 [#3554](https://github.com/buildkite/agent/pull/3554) (@DrJosh9000)
+- Catch all 'ignored' vars [#3502](https://github.com/buildkite/agent/pull/3502) (@DrJosh9000)
+
+### Internal
+- chore: go modernize to do a bit of a tidy up and remove some junk [#3560](https://github.com/buildkite/agent/pull/3560) (@wolfeidau)
+- Enforce that command descriptions indent using spaces, not tabs [#3553](https://github.com/buildkite/agent/pull/3553) (@moskyb)
+
+### Dependency updates
+- build(deps): bump the cloud-providers group across 1 directory with 9 updates [#3566](https://github.com/buildkite/agent/pull/3566) (@dependabot[bot])
+- build(deps): bump golangci/golangci-lint from v2.5-alpine to v2.6-alpine in /.buildkite in the container-images group across 1 directory [#3563](https://github.com/buildkite/agent/pull/3563) (@dependabot[bot])
+- build(deps): bump the container-images group across 4 directories with 1 update [#3564](https://github.com/buildkite/agent/pull/3564) (@dependabot[bot])
+- build(deps): bump gopkg.in/DataDog/dd-trace-go.v1 from 1.74.7 to 1.74.8 [#3555](https://github.com/buildkite/agent/pull/3555) (@dependabot[bot])
+- build(deps): bump the cloud-providers group with 6 updates [#3556](https://github.com/buildkite/agent/pull/3556) (@dependabot[bot])
+- build(deps): bump the container-images group across 4 directories with 1 update [#3557](https://github.com/buildkite/agent/pull/3557) (@dependabot[bot])
+- build(deps): bump docker/library/golang from `02ce1d7` to `5034fa4` in /.buildkite in the container-images group across 1 directory [#3558](https://github.com/buildkite/agent/pull/3558) (@dependabot[bot])
+
+
 ## [v3.110.0](https://github.com/buildkite/agent/tree/v3.110.0) (2025-10-22)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.109.1...v3.110.0)
 
diff --git a/version/VERSION b/version/VERSION
index a23b9aae41..7d8eb99498 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.110.0
+3.111.0

From 6f4eb5e182d0f4fa6c69ecca6eabd3e5995ab259 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Fri, 7 Nov 2025 14:22:59 +1100
Subject: [PATCH 056/242] Add --scope flag to `buildkite-agent annotate`
 command

The idea is that users can choose to either annotate the build (the default) or the job, which will affect where the annotation is displayed in the buildkite UI
---
 api/annotations.go     | 1 +
 clicommand/annotate.go | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/api/annotations.go b/api/annotations.go
index 90f5649e57..fdf109dc17 100644
--- a/api/annotations.go
+++ b/api/annotations.go
@@ -12,6 +12,7 @@ type Annotation struct {
 	Style    string `json:"style,omitempty"`
 	Append   bool   `json:"append,omitempty"`
 	Priority int    `json:"priority,omitempty"`
+	Scope    string `json:"scope,omitempty"`
 }
 
 // Annotate a build in the Buildkite UI
diff --git a/clicommand/annotate.go b/clicommand/annotate.go
index 63290a782c..7f8a58d558 100644
--- a/clicommand/annotate.go
+++ b/clicommand/annotate.go
@@ -66,6 +66,7 @@ type AnnotateConfig struct {
 	Priority     int      `cli:"priority"`
 	Job          string   `cli:"job" validate:"required"`
 	RedactedVars []string `cli:"redacted-vars" normalize:"list"`
+	Scope        string   `cli:"scope"`
 }
 
 var AnnotateCommand = cli.Command{
@@ -101,6 +102,12 @@ var AnnotateCommand = cli.Command{
 			Usage:  "Which job should the annotation come from",
 			EnvVar: "BUILDKITE_JOB_ID",
 		},
+		cli.StringFlag{
+			Name:   "scope",
+			Value:  "build",
+			Usage:  "The scope of the annotation, which will control where the annotation is displayed in the Buildkite UI. One of 'build', 'job'",
+			EnvVar: "BUILDKITE_ANNOTATION_SCOPE",
+		},
 
 		RedactedVars,
 	}),
@@ -159,6 +166,7 @@ func annotate(ctx context.Context, cfg AnnotateConfig, l logger.Logger) error {
 		Context:  cfg.Context,
 		Append:   cfg.Append,
 		Priority: cfg.Priority,
+		Scope:    cfg.Scope,
 	}
 
 	// Retry the annotation a few times before giving up

From d1281ded52ff4a269c8a28b8ce1275306663f64a Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Fri, 7 Nov 2025 14:23:19 +1100
Subject: [PATCH 057/242] Allow removal of annotations of differing scopes

---
 agent/api.go                    |  2 +-
 api/annotations.go              | 11 ++++++++++-
 clicommand/annotation_remove.go |  9 ++++++++-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/agent/api.go b/agent/api.go
index bca6d2f407..9534e1922d 100644
--- a/agent/api.go
+++ b/agent/api.go
@@ -13,7 +13,7 @@ type APIClient interface {
 	AcceptJob(context.Context, *api.Job) (*api.Job, *api.Response, error)
 	AcquireJob(context.Context, string, ...api.Header) (*api.Job, *api.Response, error)
 	Annotate(context.Context, string, *api.Annotation) (*api.Response, error)
-	AnnotationRemove(context.Context, string, string) (*api.Response, error)
+	AnnotationRemove(context.Context, string, string, string) (*api.Response, error)
 	CancelBuild(context.Context, string) (*api.Build, *api.Response, error)
 	Config() api.Config
 	Connect(context.Context) (*api.Response, error)
diff --git a/api/annotations.go b/api/annotations.go
index fdf109dc17..ae8157c8e4 100644
--- a/api/annotations.go
+++ b/api/annotations.go
@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"fmt"
+	"net/url"
 )
 
 // Annotation represents a Buildkite Agent API Annotation
@@ -28,7 +29,7 @@ func (c *Client) Annotate(ctx context.Context, jobId string, annotation *Annotat
 }
 
 // Remove an annotation from a build
-func (c *Client) AnnotationRemove(ctx context.Context, jobId string, context string) (*Response, error) {
+func (c *Client) AnnotationRemove(ctx context.Context, jobId string, context, scope string) (*Response, error) {
 	u := fmt.Sprintf("jobs/%s/annotations/%s", railsPathEscape(jobId), railsPathEscape(context))
 
 	req, err := c.newRequest(ctx, "DELETE", u, nil)
@@ -36,5 +37,13 @@ func (c *Client) AnnotationRemove(ctx context.Context, jobId string, context str
 		return nil, err
 	}
 
+	q, err := url.ParseQuery(req.URL.RawQuery)
+	if err != nil {
+		return nil, fmt.Errorf("decoding query string: %w", err)
+	}
+
+	q.Set("scope", scope)
+	req.URL.RawQuery = q.Encode()
+
 	return c.doRequest(req, nil)
 }
diff --git a/clicommand/annotation_remove.go b/clicommand/annotation_remove.go
index 73609d17f1..baf60307a2 100644
--- a/clicommand/annotation_remove.go
+++ b/clicommand/annotation_remove.go
@@ -32,6 +32,7 @@ type AnnotationRemoveConfig struct {
 	APIConfig
 
 	Context string `cli:"context" validate:"required"`
+	Scope   string `cli:"scope"`
 	Job     string `cli:"job" validate:"required"`
 }
 
@@ -46,6 +47,12 @@ var AnnotationRemoveCommand = cli.Command{
 			Usage:  "The context of the annotation used to differentiate this annotation from others",
 			EnvVar: "BUILDKITE_ANNOTATION_CONTEXT",
 		},
+		cli.StringFlag{
+			Name:   "scope",
+			Value:  "build",
+			Usage:  "The scope of the annotation to remove. One of either 'build' or 'job'",
+			EnvVar: "BUILDKITE_ANNOTATION_SCOPE",
+		},
 		cli.StringFlag{
 			Name:   "job",
 			Value:  "",
@@ -68,7 +75,7 @@ var AnnotationRemoveCommand = cli.Command{
 			roko.WithJitter(),
 		).DoWithContext(ctx, func(r *roko.Retrier) error {
 			// Attempt to remove the annotation
-			resp, err := client.AnnotationRemove(ctx, cfg.Job, cfg.Context)
+			resp, err := client.AnnotationRemove(ctx, cfg.Job, cfg.Context, cfg.Scope)
 
 			// Don't bother retrying if the response was one of these statuses
 			if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 404 || resp.StatusCode == 400 || resp.StatusCode == 410) {

From a84fd08c4be198a365e644584315e587c6ec3d8f Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 10 Nov 2025 11:10:04 +1100
Subject: [PATCH 058/242] IdleMonitor-related fixes

---
 agent/agent_configuration.go |  4 +-
 agent/agent_pool.go          | 19 +++----
 agent/agent_worker.go        | 56 ++++++++-------------
 agent/agent_worker_test.go   | 20 ++++----
 agent/idle_monitor.go        | 98 ++++++++++++++++++++++++++----------
 clicommand/agent_start.go    |  8 +--
 6 files changed, 119 insertions(+), 86 deletions(-)

diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go
index 1556e94c82..b467bc51b6 100644
--- a/agent/agent_configuration.go
+++ b/agent/agent_configuration.go
@@ -49,8 +49,8 @@ type AgentConfiguration struct {
 	TimestampLines               bool
 	HealthCheckAddr              string
 	DisconnectAfterJob           bool
-	DisconnectAfterIdleTimeout   int
-	DisconnectAfterUptime        int
+	DisconnectAfterIdleTimeout   time.Duration
+	DisconnectAfterUptime        time.Duration
 	CancelGracePeriod            int
 	SignalGracePeriod            time.Duration
 	EnableJobLogTmpfile          bool
diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index f31b8153eb..c43a306e46 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -13,17 +13,15 @@ import (
 	"github.com/buildkite/agent/v3/status"
 )
 
-// AgentPool manages multiple parallel AgentWorkers
+// AgentPool manages multiple parallel AgentWorkers.
 type AgentPool struct {
-	workers     []*AgentWorker
-	idleMonitor *IdleMonitor
+	workers []*AgentWorker
 }
 
-// NewAgentPool returns a new AgentPool
+// NewAgentPool returns a new AgentPool.
 func NewAgentPool(workers []*AgentWorker) *AgentPool {
 	return &AgentPool{
-		workers:     workers,
-		idleMonitor: NewIdleMonitor(len(workers)),
+		workers: workers,
 	}
 }
 
@@ -57,12 +55,15 @@ func (r *AgentPool) Start(ctx context.Context) error {
 	defer done()
 	setStat("🏃 Spawning workers...")
 
+	idleMon := newIdleMonitor(len(r.workers))
+
 	errCh := make(chan error)
 
 	// Spawn each worker "in parallel" (in its own goroutine)
 	for _, worker := range r.workers {
 		go func() {
-			errCh <- r.runWorker(ctx, worker)
+			defer idleMon.markDead(worker)
+			errCh <- r.runWorker(ctx, worker, idleMon)
 		}()
 	}
 
@@ -76,7 +77,7 @@ func (r *AgentPool) Start(ctx context.Context) error {
 	return errors.Join(errs...) // nil if all errs are nil
 }
 
-func (r *AgentPool) runWorker(ctx context.Context, worker *AgentWorker) error {
+func (r *AgentPool) runWorker(ctx context.Context, worker *AgentWorker, idleMon *idleMonitor) error {
 	// Connect the worker to the API
 	if err := worker.Connect(ctx); err != nil {
 		return err
@@ -85,7 +86,7 @@ func (r *AgentPool) runWorker(ctx context.Context, worker *AgentWorker) error {
 	defer worker.Disconnect(ctx) //nolint:errcheck // Error is logged within core/client
 
 	// Starts the agent worker and wait for it to finish.
-	return worker.Start(ctx, r.idleMonitor)
+	return worker.Start(ctx, idleMon)
 }
 
 // StopGracefully stops all workers in the pool gracefully.
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 2e0901d199..755cb2a2fa 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -217,7 +217,7 @@ func (a *AgentWorker) statusCallback(context.Context) (any, error) {
 }
 
 // Starts the agent worker
-func (a *AgentWorker) Start(ctx context.Context, idleMonitor *IdleMonitor) (startErr error) {
+func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr error) {
 	// Record the start time for max agent lifetime tracking
 	a.startTime = time.Now()
 
@@ -248,7 +248,7 @@ func (a *AgentWorker) Start(ctx context.Context, idleMonitor *IdleMonitor) (star
 		// there's really no point in letting the idle monitor know
 		// we're busy, but it's probably a good thing to do for good
 		// measure.
-		idleMonitor.MarkBusy(a.agent.UUID)
+		idleMon.markBusy(a)
 
 		if err := a.AcquireAndRunJob(ctx, a.agentConfiguration.AcquireJob); err != nil {
 			// If the job acquisition was rejected, we can exit with an error
@@ -270,7 +270,7 @@ func (a *AgentWorker) Start(ctx context.Context, idleMonitor *IdleMonitor) (star
 		}
 	}
 
-	return a.runPingLoop(ctx, idleMonitor)
+	return a.runPingLoop(ctx, idleMon)
 }
 
 func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) {
@@ -311,13 +311,14 @@ func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) {
 	}
 }
 
-func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor) error {
+// runPingLoop runs the loop that pings Buildkite for work. It does all critical
+// agent things.
+// The lifetime of an agent is the lifetime of the ping loop.
+func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) error {
 	ctx, setStat, _ := status.AddSimpleItem(ctx, "Ping loop")
 	defer setStat("🛑 Ping loop stopped!")
 	setStat("🏃 Starting...")
 
-	disconnectAfterIdleTimeout := time.Second * time.Duration(a.agentConfiguration.DisconnectAfterIdleTimeout)
-
 	// Create the ticker
 	pingInterval := time.Second * time.Duration(a.agent.PingInterval)
 	pingTicker := time.NewTicker(pingInterval)
@@ -334,7 +335,6 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 	first := make(chan struct{}, 1)
 	first <- struct{}{}
 
-	lastActionTime := time.Now()
 	a.logger.Info("Waiting for instructions...")
 
 	ranJob := false
@@ -430,8 +430,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 		}
 
 		// Exit after disconnect-after-uptime is exceeded.
-		if a.agentConfiguration.DisconnectAfterUptime > 0 {
-			maxUptime := time.Second * time.Duration(a.agentConfiguration.DisconnectAfterUptime)
+		if maxUptime := a.agentConfiguration.DisconnectAfterUptime; maxUptime > 0 {
 			if time.Since(a.startTime) >= maxUptime {
 				if job != nil {
 					a.logger.Error("Agent ping dispatched a job (id %q) but agent has exceeded max uptime of %v!", job.ID, maxUptime)
@@ -443,40 +442,26 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 
 		// Note that Ping only returns a job if err == nil.
 		if job == nil {
-			if disconnectAfterIdleTimeout == 0 {
+			idleTimeout := a.agentConfiguration.DisconnectAfterIdleTimeout
+			if idleTimeout == 0 {
 				// No job and no idle timeout.
 				continue
 			}
 
-			// Handle disconnect after idle timeout (and deprecated disconnect-after-job-timeout).
-			// Only do this check if we weren't just dispatched a job.
-			// (If we were dispatched a job, we're not idle.)
-			idleDeadline := lastActionTime.Add(disconnectAfterIdleTimeout)
-			if time.Now().After(idleDeadline) {
-				// Let other agents know this agent is now idle and termination
-				// is possible
-				idleMonitor.MarkIdle(a.agent.UUID)
-
-				// But only terminate if everyone else is also idle
-				if idleMonitor.Idle() {
-					a.logger.Info("All agents have been idle for %v. Disconnecting...", disconnectAfterIdleTimeout)
-					return nil
-				}
-				a.logger.Debug("Agent has been idle for %.f seconds, but other agents haven't",
-					time.Since(lastActionTime).Seconds())
+			// Exit if every agent has been idle for at least the timeout.
+			if idleMon.shouldExit(idleTimeout) {
+				a.logger.Info("All agents have been idle for at least %v. Disconnecting...", idleTimeout)
+				return nil
 			}
-			// Not idle enough yet. Wait and ping again.
+
+			// Not idle enough to exit. Wait and ping again.
 			continue
 		}
 
-		// Let other agents know this agent is now busy and
-		// not to idle terminate
-		idleMonitor.MarkBusy(a.agent.UUID)
-
 		setStat("💼 Accepting job")
 
 		// Runs the job, only errors if something goes wrong
-		if err := a.AcceptAndRunJob(ctx, job); err != nil {
+		if err := a.AcceptAndRunJob(ctx, job, idleMon); err != nil {
 			a.logger.Error("%v", err)
 			setStat(fmt.Sprintf("✅ Finished job with error: %v", err))
 			continue
@@ -489,7 +474,6 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMonitor *IdleMonitor)
 			pingTicker.Reset(pingInterval)
 			continue
 		}
-		lastActionTime = time.Now()
 
 		// Observation: jobs are rarely the last within a pipeline,
 		// thus if this worker just completed a job,
@@ -700,9 +684,13 @@ func (a *AgentWorker) AcquireAndRunJob(ctx context.Context, jobId string) error
 }
 
 // Accepts a job and runs it, only returns an error if something goes wrong
-func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, job *api.Job) error {
+func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, job *api.Job, idleMon *idleMonitor) error {
 	a.logger.Info("Assigned job %s. Accepting...", job.ID)
 
+	// An agent is busy during a job, and idle when the job is done.
+	idleMon.markBusy(a)
+	defer idleMon.markIdle(a)
+
 	// Accept the job. We'll retry on connection related issues, but if
 	// Buildkite returns a 422 or 500 for example, we'll just bail out,
 	// re-ping, and try the whole process again.
diff --git a/agent/agent_worker_test.go b/agent/agent_worker_test.go
index 36d78bffa2..6fed4f6f1b 100644
--- a/agent/agent_worker_test.go
+++ b/agent/agent_worker_test.go
@@ -317,7 +317,7 @@ func TestAgentWorker_Start_AcquireJob_JobAcquisitionRejected(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := NewIdleMonitor(1)
+	idleMonitor := newIdleMonitor(1)
 
 	// we expect the worker to try to acquire the job, but fail with ErrJobAcquisitionRejected
 	// because the server returns a 422 Unprocessable Entity.
@@ -401,7 +401,7 @@ func TestAgentWorker_Start_AcquireJob_Pause_Unpause(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := NewIdleMonitor(1)
+	idleMonitor := newIdleMonitor(1)
 
 	if err := worker.Start(ctx, idleMonitor); err != nil {
 		t.Errorf("worker.Start() = %v", err)
@@ -495,7 +495,7 @@ func TestAgentWorker_DisconnectAfterJob_Start_Pause_Unpause(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := NewIdleMonitor(1)
+	idleMonitor := newIdleMonitor(1)
 
 	if err := worker.Start(ctx, idleMonitor); err != nil {
 		t.Errorf("worker.Start() = %v", err)
@@ -576,13 +576,13 @@ func TestAgentWorker_DisconnectAfterUptime(t *testing.T) {
 				BootstrapScript:       dummyBootstrap,
 				BuildPath:             buildPath,
 				HooksPath:             hooksPath,
-				DisconnectAfterUptime: 1, // 1 second max uptime
+				DisconnectAfterUptime: 1 * time.Second, // max uptime
 			},
 		},
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := NewIdleMonitor(1)
+	idleMonitor := newIdleMonitor(1)
 
 	// Record start time
 	startTime := time.Now()
@@ -666,7 +666,7 @@ func TestAgentWorker_SetEndpointDuringRegistration(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, NewIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -755,7 +755,7 @@ func TestAgentWorker_UpdateEndpointDuringPing(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, NewIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -835,7 +835,7 @@ func TestAgentWorker_UpdateEndpointDuringPing_FailAndRevert(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, NewIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -904,7 +904,7 @@ func TestAgentWorker_SetRequestHeadersDuringRegistration(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, NewIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -992,7 +992,7 @@ func TestAgentWorker_UpdateRequestHeadersDuringPing(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, NewIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
diff --git a/agent/idle_monitor.go b/agent/idle_monitor.go
index 7d40a5c096..d9a4557118 100644
--- a/agent/idle_monitor.go
+++ b/agent/idle_monitor.go
@@ -1,44 +1,88 @@
 package agent
 
-import "sync"
+import (
+	"sync"
+	"time"
+)
 
-// This monitor has a 3rd implicit state we will call "initializing" that all agents start in
-// Agents can transition to busy and/or idle but always start in the "initializing" state
+// idleMonitor tracks agent idleness, needed for "disconnect-after-idle" type
+// logic.
+//
+// In addition to "busy", "idle", and "dead", idleMonitor has an implicit
+// "initial" state. Agents always start in the "initial" state
 /*
-//                -> Busy
-//              /     ^
-// Initializing       |
-//              \     v
-//                -> Idle
+//                -> Busy --
+//              /     ^      \
+//      Initial ------+--------> Dead
+//              \     v      /
+//                -> Idle --
 */
-// This (intentionally?) ensures the DisconnectAfterIdleTimeout doesn't fire before agents have had a chance to run a job
-type IdleMonitor struct {
-	sync.Mutex
+// This (intentionally) ensures the DisconnectAfterIdleTimeout doesn't fire
+// before agents have had a chance to run a job.
+type idleMonitor struct {
+	mu          sync.Mutex
+	exiting     bool
 	totalAgents int
-	idle        map[string]struct{}
+	idleAt      map[*AgentWorker]time.Time
 }
 
-func NewIdleMonitor(totalAgents int) *IdleMonitor {
-	return &IdleMonitor{
+// newIdleMonitor creates a new IdleMonitor.
+func newIdleMonitor(totalAgents int) *idleMonitor {
+	return &idleMonitor{
 		totalAgents: totalAgents,
-		idle:        map[string]struct{}{},
+		idleAt:      make(map[*AgentWorker]time.Time),
 	}
 }
 
-func (i *IdleMonitor) Idle() bool {
-	i.Lock()
-	defer i.Unlock()
-	return len(i.idle) == i.totalAgents
+// shouldExit reports whether all agents are dead or have been idle for at least
+// minIdle.  If shouldExit returns true, it will return true on all subsequent
+// calls.
+func (i *idleMonitor) shouldExit(minIdle time.Duration) bool {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	// Once the idle monitor decides we're exiting, we're exiting.
+	if i.exiting {
+		return true
+	}
+
+	// Are all alive agents dead or idle for long enough?
+	idle := 0
+	for _, t := range i.idleAt {
+		if !t.IsZero() && time.Since(t) < minIdle {
+			return false
+		}
+		idle++
+	}
+	if idle < i.totalAgents {
+		return false
+	}
+	i.exiting = true
+	return true
+}
+
+// markIdle marks an agent as idle.
+func (i *idleMonitor) markIdle(agent *AgentWorker) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+	// Allow MarkIdle to be called multiple times without updating the idleAt
+	// timestamp.
+	if _, alreadyIdle := i.idleAt[agent]; alreadyIdle {
+		return
+	}
+	i.idleAt[agent] = time.Now()
 }
 
-func (i *IdleMonitor) MarkIdle(agentUUID string) {
-	i.Lock()
-	defer i.Unlock()
-	i.idle[agentUUID] = struct{}{}
+// markDead marks an agent as dead.
+func (i *idleMonitor) markDead(agent *AgentWorker) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+	i.idleAt[agent] = time.Time{}
 }
 
-func (i *IdleMonitor) MarkBusy(agentUUID string) {
-	i.Lock()
-	defer i.Unlock()
-	delete(i.idle, agentUUID)
+// markBusy marks an agent as busy.
+func (i *idleMonitor) markBusy(agent *AgentWorker) {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+	delete(i.idleAt, agent)
 }
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index c839414ce9..0ea9a2f2d8 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1037,8 +1037,8 @@ var AgentStartCommand = cli.Command{
 			ANSITimestamps:               !cfg.NoANSITimestamps,
 			TimestampLines:               cfg.TimestampLines,
 			DisconnectAfterJob:           cfg.DisconnectAfterJob,
-			DisconnectAfterIdleTimeout:   cfg.DisconnectAfterIdleTimeout,
-			DisconnectAfterUptime:        cfg.DisconnectAfterUptime,
+			DisconnectAfterIdleTimeout:   time.Duration(cfg.DisconnectAfterIdleTimeout) * time.Second,
+			DisconnectAfterUptime:        time.Duration(cfg.DisconnectAfterUptime) * time.Second,
 			CancelGracePeriod:            cfg.CancelGracePeriod,
 			SignalGracePeriod:            signalGracePeriod,
 			EnableJobLogTmpfile:          cfg.EnableJobLogTmpfile,
@@ -1131,11 +1131,11 @@ var AgentStartCommand = cli.Command{
 		}
 
 		if agentConf.DisconnectAfterIdleTimeout > 0 {
-			l.Info("Agents will disconnect after %d seconds of inactivity", agentConf.DisconnectAfterIdleTimeout)
+			l.Info("Agents will disconnect after %v of inactivity", agentConf.DisconnectAfterIdleTimeout)
 		}
 
 		if agentConf.DisconnectAfterUptime > 0 {
-			l.Info("Agents will disconnect after %d seconds of uptime and shut down after any running jobs complete", agentConf.DisconnectAfterUptime)
+			l.Info("Agents will disconnect after %v of uptime and shut down after any running jobs complete", agentConf.DisconnectAfterUptime)
 		}
 
 		if len(cfg.AllowedRepositories) > 0 {

From 7501aff5510374940cdec1e085ccf0c6567f9050 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 10 Nov 2025 16:30:49 +1100
Subject: [PATCH 059/242] Skip ticker after running job

---
 agent/agent_worker.go | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 755cb2a2fa..8d287a7272 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -332,8 +332,10 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		close(testTriggerCh)
 	}
 
-	first := make(chan struct{}, 1)
-	first <- struct{}{}
+	// On the first iteration, skip waiting for the pingTicker.
+	// This doesn't skip the jitter, though.
+	skipTicker := make(chan struct{}, 1)
+	skipTicker <- struct{}{}
 
 	a.logger.Info("Waiting for instructions...")
 
@@ -354,7 +356,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		select {
 		case <-testTriggerCh:
 			// instant receive from closed chan when noWaitBetweenPingsForTesting is true
-		case <-first:
+		case <-skipTicker:
 			// continue below
 		case <-pingTicker.C:
 			// continue below
@@ -468,22 +470,21 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		}
 
 		ranJob = true
-		if a.agentConfiguration.DisconnectAfterJob {
-			// Unless paused, this agent disconnects after the next ping.
-			// Do the ping immediately so we reduce the chances our agent is assigned a job
-			pingTicker.Reset(pingInterval)
-			continue
-		}
 
 		// Observation: jobs are rarely the last within a pipeline,
 		// thus if this worker just completed a job,
 		// there is likely another immediately available.
 		// Skip waiting for the ping interval until
 		// a ping without a job has occurred,
-		// but in exchange, ensure the next ping must wait a full
+		// but in exchange, ensure the next ping must wait at least a full
 		// pingInterval to avoid too much server load.
-
 		pingTicker.Reset(pingInterval)
+		select {
+		case skipTicker <- struct{}{}:
+			// Ticker will be skipped
+		default:
+			// We're already skipping the ticker, don't block.
+		}
 	}
 }
 

From ea69765f04bba8f853abda853b57a3797b6fd14e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Nov 2025 15:41:25 +0000
Subject: [PATCH 060/242] build(deps): bump the golang-x group with 3 updates

Bumps the golang-x group with 3 updates: [golang.org/x/oauth2](https://github.com/golang/oauth2), [golang.org/x/sync](https://github.com/golang/sync) and [golang.org/x/sys](https://github.com/golang/sys).


Updates `golang.org/x/oauth2` from 0.32.0 to 0.33.0
- [Commits](https://github.com/golang/oauth2/compare/v0.32.0...v0.33.0)

Updates `golang.org/x/sync` from 0.17.0 to 0.18.0
- [Commits](https://github.com/golang/sync/compare/v0.17.0...v0.18.0)

Updates `golang.org/x/sys` from 0.37.0 to 0.38.0
- [Commits](https://github.com/golang/sys/compare/v0.37.0...v0.38.0)

---
updated-dependencies:
- dependency-name: golang.org/x/oauth2
  dependency-version: 0.33.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/sync
  dependency-version: 0.18.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/sys
  dependency-version: 0.38.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod |  6 +++---
 go.sum | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index ea092d7244..be58a72fd2 100644
--- a/go.mod
+++ b/go.mod
@@ -56,9 +56,9 @@ require (
 	go.opentelemetry.io/otel/trace v1.38.0
 	golang.org/x/crypto v0.43.0
 	golang.org/x/net v0.46.0
-	golang.org/x/oauth2 v0.32.0
-	golang.org/x/sync v0.17.0
-	golang.org/x/sys v0.37.0
+	golang.org/x/oauth2 v0.33.0
+	golang.org/x/sync v0.18.0
+	golang.org/x/sys v0.38.0
 	golang.org/x/term v0.36.0
 	google.golang.org/api v0.255.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
diff --git a/go.sum b/go.sum
index 7969af33fc..10d8d05092 100644
--- a/go.sum
+++ b/go.sum
@@ -485,14 +485,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
 golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
-golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
-golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo=
+golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -506,8 +506,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
 golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=

From 6cbd33f195ff2d9293576acb361f7b072e4477ee Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Nov 2025 15:45:57 +0000
Subject: [PATCH 061/242] build(deps): bump the cloud-providers group with 4
 updates

Bumps the cloud-providers group with 4 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2).


Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.20.3 to 1.20.4
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.20.3...service/mq/v1.20.4)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.261.1 to 1.264.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.261.1...service/ec2/v1.264.0)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.47.1 to 1.48.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.47.1...service/s3/v1.48.0)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.89.2 to 1.90.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.89.2...service/s3/v1.90.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-version: 1.20.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.264.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.48.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-version: 1.90.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod |  8 ++++----
 go.sum | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/go.mod b/go.mod
index ea092d7244..b417461592 100644
--- a/go.mod
+++ b/go.mod
@@ -14,10 +14,10 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.39.6
 	github.com/aws/aws-sdk-go-v2/config v1.31.17
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.261.1
-	github.com/aws/aws-sdk-go-v2/service/kms v1.47.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.89.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.264.0
+	github.com/aws/aws-sdk-go-v2/service/kms v1.48.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0
 	github.com/aws/smithy-go v1.23.2
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
diff --git a/go.sum b/go.sum
index 7969af33fc..04578ee830 100644
--- a/go.sum
+++ b/go.sum
@@ -86,8 +86,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.18.21 h1:56HGpsgnmD+2/KpG0ikvvR8+3v3
 github.com/aws/aws-sdk-go-v2/credentials v1.18.21/go.mod h1:3YELwedmQbw7cXNaII2Wywd+YY58AmLPwX4LzARgmmA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 h1:4GNV1lhyELGjMz5ILMRxDvxvOaeo3Ux9Z69S1EgVMMQ=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3/go.mod h1:br7KA6edAAqDGUYJ+zVVPAyMrPhnN+zdt17yTUT6FPw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
@@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.261.1 h1:1U9Is5mKIIgi8xlIZi5Apuq/SHnQrKUBmmRm4tmgLYM=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.261.1/go.mod h1:NDdDLLW5PtLLXN661gKcvJvqAH5OBXsfhMlmKVu1/pY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.264.0 h1:3SsIzhGS28WMDppm5VLeTM9qxrN7vhxDRlUUi54NXRE=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.264.0/go.mod h1:NDdDLLW5PtLLXN661gKcvJvqAH5OBXsfhMlmKVu1/pY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
@@ -106,10 +106,10 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
-github.com/aws/aws-sdk-go-v2/service/kms v1.47.1 h1:6+C0RoGF4HJQALrsecOXN7cm/l5rgNHCw2xbcvFgpH4=
-github.com/aws/aws-sdk-go-v2/service/kms v1.47.1/go.mod h1:VJcNH6BLr+3VJwinRKdotLOMglHO8mIKlD3ea5c7hbw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.2 h1:xgBWsgaeUESl8A8k80p6yBdexMWDVeiDmJ/pkjohJ7c=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
+github.com/aws/aws-sdk-go-v2/service/kms v1.48.0 h1:pQgVxqqNOacqb19+xaoih/wNLil4d8tgi+FxtBi/qQY=
+github.com/aws/aws-sdk-go-v2/service/kms v1.48.0/go.mod h1:VJcNH6BLr+3VJwinRKdotLOMglHO8mIKlD3ea5c7hbw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0=

From 2cf5fe880031c96c2cf80bbc101e9ff642cdedf5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Nov 2025 16:44:26 +0000
Subject: [PATCH 062/242] build(deps): bump the container-images group across 5
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `5922070` to `d3529e9`

Updates `buildkite/agent-base` from `b4d0e49` to `0d16202`

Updates `buildkite/agent-base` from `03dbce9` to `767ff4a`

Updates `buildkite/agent-base` from `82ce9ff` to `b600458`

Updates `buildkite/agent-base` from `e60eef5` to `e33daff`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index fcfac1e707..1f5dd93e18 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:b4d0e49d234e37574c12225c579406f9eacbb8d0600489085bc1ccf97300a7a3
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:0d1620227851573d0d9ae37132d58d6c56d7264f9ec933615683f6cdea39b79f
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index ed5ca505d7..2090cc8dd7 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:592207055fd1a54ea7c92f3277f352bf582c34f543ad0d9ebb4115d5a9ca5ad6
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:d3529e9b56cce5f6d846efdee15fda4053ec7048b67c7c354ac8fc806e5cce0d
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 91b8432c5f..9127c8d07d 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:03dbce9ca376b7dc88550c9e8f8b1b865c2127757950ff4b04e1544ce893ceed
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:767ff4a1d4d1a8b94aad0646ce8c6498ed5f312019b7ec897561f318b138326f
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index 67d601f549..f1fa93dabf 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:82ce9ff54c5a4a3bd196daa80fe4396b47b63a64c655ccf632df1707f74870bd
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:b6004588a34e4e9ce919a30ad627826d39835dc656b3855bb717ed7545e17bf7
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index 13bc04281e..9899b416ae 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:e60eef5258c55ce337789385c0fb8db8488cdcae75243d7a2f3c17b7a2d3d5f8
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:e33daff4bca2e47ef4d6018c857ed479903cc5d986a4face3c6a322657dd9005
 ARG TARGETOS
 ARG TARGETARCH
 

From e0298b3654ce1c72bf4b35377643084226e18487 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Nov 2025 16:45:02 +0000
Subject: [PATCH 063/242] build(deps): bump the container-images group across 1
 directory with 2 updates

Bumps the container-images group with 2 updates in the /.buildkite directory: docker/library/golang and golangci/golangci-lint.


Updates `docker/library/golang` from 1.24.9 to 1.24.10

Updates `golangci/golangci-lint` from `1e8c410` to `a7da515`

---
updated-dependencies:
- dependency-name: docker/library/golang
  dependency-version: 1.24.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: container-images
- dependency-name: golangci/golangci-lint
  dependency-version: v2.6-alpine
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-compile | 2 +-
 .buildkite/Dockerfile-lint    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index b9e0568fa5..268182ec83 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.9@sha256:5034fa44b36163a4109b71ed75c67dbdcb52c3cd9750953befe00054315d9fd2
+FROM public.ecr.aws/docker/library/golang:1.24.10@sha256:c3ea4172c1dd39e1c90bb36a11ef95af6d0ccbb1d7cdedbb5dd14988c324d689
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest
diff --git a/.buildkite/Dockerfile-lint b/.buildkite/Dockerfile-lint
index b01dea690e..cc0ea8a671 100644
--- a/.buildkite/Dockerfile-lint
+++ b/.buildkite/Dockerfile-lint
@@ -1 +1 @@
-FROM golangci/golangci-lint:v2.6-alpine@sha256:1e8c410818ea9f1f4176b89dd2d95776f07184a7d4a8bf88d25e553b04c1995a
\ No newline at end of file
+FROM golangci/golangci-lint:v2.6-alpine@sha256:a7da5151e0bd61bd7f99e1ebd8e5e144b535b73b2762c498443ff4f6a4a538c4
\ No newline at end of file

From 8d28707f85f52e494006c914052a15350498ba8d Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Wed, 12 Nov 2025 14:04:44 +1100
Subject: [PATCH 064/242] Bump version + changelog for v3.112.0

---
 CHANGELOG.md    | 20 ++++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 72bcf9ac7a..8767ee5ba7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.112.0](https://github.com/buildkite/agent/tree/v3.112.0) (2025-11-12)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.111.0...v3.112.0)
+
+### Added
+
+The agent can now annotate jobs as well as builds! Job annotations will show up in a dedicated section of the job detail
+in the build UI. This is a great way to provide additional, richly-formatted context and information about specific jobs.
+
+See the [PR](https://github.com/buildkite/agent/pull/3569) for more details.
+
+### Changed
+- Agents will now check for new work more quickly immediately after finishing a job [#3571](https://github.com/buildkite/agent/pull/3571) (@DrJosh9000)
+
+### Fixed
+- IdleMonitor-related fixes [#3570](https://github.com/buildkite/agent/pull/3570) (@DrJosh9000)
+- Fix confusing error message when hashing artifact payloads [#3565](https://github.com/buildkite/agent/pull/3565) (@moskyb)
+
+### Internal
+- Dependency updates [#3575](https://github.com/buildkite/agent/pull/3575) [#3574](https://github.com/buildkite/agent/pull/3574) [#3573](https://github.com/buildkite/agent/pull/3573) [#3572](https://github.com/buildkite/agent/pull/3572) (@dependabot[bot])
+
 ## [v3.111.0](https://github.com/buildkite/agent/tree/v3.111.0) (2025-11-05)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.110.0...v3.111.0)
 
diff --git a/version/VERSION b/version/VERSION
index 7d8eb99498..87f3781675 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.111.0
+3.112.0

From a00135e37d1ba0b15606dbae7ea37c5377702485 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 11 Nov 2025 12:00:30 +1100
Subject: [PATCH 065/242] Add Prometheus /metrics handler and some basic
 metrics

---
 agent/agent_pool.go   |  19 +++++---
 agent/agent_worker.go |  10 ++++
 agent/job_runner.go   |  12 ++++-
 agent/metrics.go      | 105 ++++++++++++++++++++++++++++++++++++++++++
 go.mod                |   7 +++
 go.sum                |  14 ++++++
 6 files changed, 159 insertions(+), 8 deletions(-)
 create mode 100644 agent/metrics.go

diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index c43a306e46..8fccaae80f 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -11,6 +11,8 @@ import (
 
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/status"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // AgentPool manages multiple parallel AgentWorkers.
@@ -29,6 +31,7 @@ func (ap *AgentPool) StartStatusServer(ctx context.Context, l logger.Logger, add
 	mux := http.NewServeMux()
 
 	mux.HandleFunc("/", healthHandler(l))
+	mux.Handle("/metrics", promhttp.Handler())
 	mux.HandleFunc("/status", status.Handle)
 	mux.HandleFunc("/status.json", ap.statusJSONHandler(l))
 
@@ -61,10 +64,7 @@ func (r *AgentPool) Start(ctx context.Context) error {
 
 	// Spawn each worker "in parallel" (in its own goroutine)
 	for _, worker := range r.workers {
-		go func() {
-			defer idleMon.markDead(worker)
-			errCh <- r.runWorker(ctx, worker, idleMon)
-		}()
+		go runWorker(ctx, worker, idleMon, errCh)
 	}
 
 	setStat("✅ Workers spawned!")
@@ -77,16 +77,21 @@ func (r *AgentPool) Start(ctx context.Context) error {
 	return errors.Join(errs...) // nil if all errs are nil
 }
 
-func (r *AgentPool) runWorker(ctx context.Context, worker *AgentWorker, idleMon *idleMonitor) error {
+func runWorker(ctx context.Context, worker *AgentWorker, idleMon *idleMonitor, errCh chan<- error) {
+	agentWorkersStarted.Inc()
+	defer agentWorkersEnded.Inc()
+	defer idleMon.markDead(worker)
+
 	// Connect the worker to the API
 	if err := worker.Connect(ctx); err != nil {
-		return err
+		errCh <- err
+		return
 	}
 	// Ensure the worker is disconnected at the end of this function.
 	defer worker.Disconnect(ctx) //nolint:errcheck // Error is logged within core/client
 
 	// Starts the agent worker and wait for it to finish.
-	return worker.Start(ctx, idleMon)
+	errCh <- worker.Start(ctx, idleMon)
 }
 
 // StopGracefully stops all workers in the pool gracefully.
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 8d287a7272..2027b844b8 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -352,6 +352,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 	//   longer than the idle timeout, and the ping action isn't "pause".
 	// * the agent has exceeded its disconnect-after-uptime and the ping action isn't "pause".
 	for {
+		startWait := time.Now()
 		setStat("😴 Waiting until next ping interval tick")
 		select {
 		case <-testTriggerCh:
@@ -380,17 +381,23 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		case <-ctx.Done():
 			return ctx.Err()
 		}
+		pingWaitDurations.Observe(time.Since(startWait).Seconds())
 
 		setStat("📡 Pinging Buildkite for instructions")
+		pingsSent.Inc()
+		startPing := time.Now()
 		job, action, err := a.Ping(ctx)
 		if err != nil {
+			pingErrors.Inc()
 			if errors.Is(err, &errUnrecoverable{}) {
 				a.logger.Error("%v", err)
 			} else {
 				a.logger.Warn("%v", err)
 			}
 		}
+		pingDurations.Observe(time.Since(startPing).Seconds())
 
+		pingActions.WithLabelValues(action).Inc()
 		switch action {
 		case "disconnect":
 			a.StopUngracefully()
@@ -733,6 +740,9 @@ func (a *AgentWorker) RunJob(ctx context.Context, acceptResponse *api.Job, ignor
 	a.setBusy(acceptResponse.ID)
 	defer a.setIdle()
 
+	jobsStarted.Inc()
+	defer jobsEnded.Inc()
+
 	jobMetricsScope := a.metrics.With(metrics.Tags{
 		"pipeline": acceptResponse.Env["BUILDKITE_PIPELINE_SLUG"],
 		"org":      acceptResponse.Env["BUILDKITE_ORGANIZATION_SLUG"],
diff --git a/agent/job_runner.go b/agent/job_runner.go
index c91084078d..51ca52e568 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -176,7 +176,17 @@ func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, con
 	r.logStreamer = NewLogStreamer(
 		r.agentLogger,
 		func(ctx context.Context, chunk *api.Chunk) error {
-			return r.client.UploadChunk(ctx, r.conf.Job.ID, chunk)
+			startUpload := time.Now()
+			// core.Client.UploadChunk contains the retry/backoff.
+			if err := r.client.UploadChunk(ctx, r.conf.Job.ID, chunk); err != nil {
+				logChunkUploadErrors.Inc()
+				logBytesUploadErrors.Add(float64(chunk.Size))
+				return err
+			}
+			logUploadDurations.Observe(time.Since(startUpload).Seconds())
+			logChunksUploaded.Inc()
+			logBytesUploaded.Add(float64(chunk.Size))
+			return nil
 		},
 		LogStreamerConfig{
 			Concurrency:       3,
diff --git a/agent/metrics.go b/agent/metrics.go
new file mode 100644
index 0000000000..db6ae1b509
--- /dev/null
+++ b/agent/metrics.go
@@ -0,0 +1,105 @@
+package agent
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	metricsNamespace = "buildkite_agent"
+)
+
+var (
+	agentWorkersStarted = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "workers",
+		Name:      "started_total",
+		Help:      "Count of agent workers started",
+	})
+	agentWorkersEnded = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "workers",
+		Name:      "ended_total",
+		Help:      "Count of agent workers ended",
+	})
+	// Currently running = started - ended.
+
+	pingsSent = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "pings",
+		Name:      "sent_total",
+		Help:      "Count of pings sent",
+	})
+	pingErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "pings",
+		Name:      "errors_total",
+		Help:      "Count of pings that failed",
+	})
+	pingActions = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "pings",
+		Name:      "actions_total",
+		Help:      "Count of successful pings by subsequent action",
+	}, []string{"action"})
+	pingDurations = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "pings",
+		Name:      "duration_seconds_total",
+		Help:      "Time spent pinging (including errors, not including subsequent actions)",
+		Buckets:   prometheus.ExponentialBuckets(0.015625, 2, 12),
+	})
+	pingWaitDurations = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "pings",
+		Name:      "wait_duration_seconds_total",
+		Help:      "Time spent waiting between pings (ping ticker + jitter)",
+		Buckets:   prometheus.LinearBuckets(1, 1, 20),
+	})
+
+	jobsStarted = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "jobs",
+		Name:      "started_total",
+		Help:      "Count of jobs started",
+	})
+	jobsEnded = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "jobs",
+		Name:      "ended_total",
+		Help:      "Count of jobs ended (any outcome)",
+	})
+
+	logChunksUploaded = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "logs",
+		Name:      "chunks_uploaded_total",
+		Help:      "Count of log chunks uploaded",
+	})
+	logBytesUploaded = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "logs",
+		Name:      "bytes_uploaded_total",
+		Help:      "Count of log bytes uploaded",
+	})
+	logChunkUploadErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "logs",
+		Name:      "chunk_uploads_errored_total",
+		Help:      "Count of log chunks not uploaded due to error",
+	})
+	logBytesUploadErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "logs",
+		Name:      "bytes_uploads_errored_total",
+		Help:      "Count of log bytes not uploaded due to error",
+	})
+	logUploadDurations = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: metricsNamespace,
+		Subsystem: "logs",
+		Name:      "upload_duration_seconds_total",
+		Help:      "Time spent uploading log chunks",
+		// Log chunk upload can be retried for a while.
+		Buckets: prometheus.ExponentialBuckets(0.015625, 2, 16),
+	})
+)
diff --git a/go.mod b/go.mod
index 3fc5b76ca7..3f8c807019 100644
--- a/go.mod
+++ b/go.mod
@@ -41,6 +41,7 @@ require (
 	github.com/oleiade/reflections v1.1.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pborman/uuid v1.2.1
+	github.com/prometheus/client_golang v1.23.2
 	github.com/puzpuzpuz/xsync/v2 v2.5.1
 	github.com/qri-io/jsonschema v0.2.1
 	github.com/stretchr/testify v1.11.1
@@ -106,6 +107,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/buildkite/test-engine-client v1.6.0 // indirect
@@ -149,6 +151,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/outcaste-io/ristretto v0.2.3 // indirect
 	github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
@@ -157,6 +160,9 @@ require (
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/qri-io/jsonpointer v0.1.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
@@ -187,6 +193,7 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
 	golang.org/x/mod v0.28.0 // indirect
 	golang.org/x/text v0.30.0 // indirect
diff --git a/go.sum b/go.sum
index 2247bf905d..6cb6478b21 100644
--- a/go.sum
+++ b/go.sum
@@ -118,6 +118,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 h1:mLlUgHn02ue8whiR4BmxxGJLR2gw
 github.com/aws/aws-sdk-go-v2/service/sts v1.39.1/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
 github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
 github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE=
 github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY=
 github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
@@ -284,6 +286,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/oleiade/reflections v1.1.0 h1:D+I/UsXQB4esMathlt0kkZRJZdUDmhv5zGi/HOwYTWo=
 github.com/oleiade/reflections v1.1.0/go.mod h1:mCxx0QseeVCHs5Um5HhJeCKVC7AwS8kO67tky4rdisA=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -313,6 +317,14 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
 github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU=
 github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU=
 github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg=
@@ -466,6 +478,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

From af44e7974ee1173c310402f5f8056a9f9bc96897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Jendrysik?= <lukasz@buildkite.com>
Date: Thu, 13 Nov 2025 14:34:50 +0100
Subject: [PATCH 066/242] Fix idle tracking for agents that never received jobs

In #3570 idle logic has changed and `idleMon.markIdle` would get only
triggered via `AcceptAndRunJob()`.
Agents that never received a job wouldn't be marked as idle.
---
 agent/agent_worker.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 2027b844b8..9ff4bbcc54 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -457,6 +457,10 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 				continue
 			}
 
+			// This ensures agents that never receive a job are still tracked
+			// by the idle monitor and can properly trigger disconnect-after-idle-timeout.
+			idleMon.markIdle(a)
+
 			// Exit if every agent has been idle for at least the timeout.
 			if idleMon.shouldExit(idleTimeout) {
 				a.logger.Info("All agents have been idle for at least %v. Disconnecting...", idleTimeout)

From 520f5693e74a9fcf8b53076b03714e7da85d1b48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Jendrysik?= <lukasz@buildkite.com>
Date: Thu, 13 Nov 2025 14:39:30 +0100
Subject: [PATCH 067/242] go generate

---
 internal/mime/mime.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/mime/mime.go b/internal/mime/mime.go
index 2c428e89b7..659a2f720e 100644
--- a/internal/mime/mime.go
+++ b/internal/mime/mime.go
@@ -779,6 +779,8 @@ var types = map[string]string{
 	".spq":         "application/scvp-vp-request",
 	".spx":         "audio/ogg",
 	".sql":         "application/x-sql",
+	".sqlite":      "application/vnd.sqlite3",
+	".sqlite3":     "application/vnd.sqlite3",
 	".src":         "application/x-wais-source",
 	".srt":         "application/x-subrip",
 	".sru":         "application/sru+xml",

From f654bc589e983949463d11b9c2b891cdd8bc85ae Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Fri, 14 Nov 2025 10:20:45 +1100
Subject: [PATCH 068/242] Fix the pipeline upload --reject-secrets flag not
 rejecting secrets

---
 clicommand/pipeline_upload.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index dbe9791ba0..0b4050ffcd 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -319,7 +319,11 @@ var PipelineUploadCommand = cli.Command{
 				if len(cfg.RedactedVars) > 0 {
 					// Secret detection uses the original environment, since
 					// Interpolate merges the pipeline's env block into `environ`.
-					searchForSecrets(l, &cfg, environ, result, input.name)
+					err := searchForSecrets(l, &cfg, environ, result, input.name)
+					if err != nil {
+						l.Error("%v", err)
+						return NewSilentExitError(1)
+					}
 				}
 
 				var (
@@ -509,6 +513,7 @@ func searchForSecrets(
 	if err != nil {
 		l.Warn("Couldn't match environment variable names against redacted-vars: %v", err)
 	}
+
 	for _, name := range short {
 		shortValues[name] = struct{}{}
 	}

From 562381a2b8c0487106a2192e0aa4cc2b002cc55c Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 17 Nov 2025 10:45:20 +1100
Subject: [PATCH 069/242] Put secret scan error into exit message

---
 clicommand/pipeline_upload.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index 0b4050ffcd..e71cea9674 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -321,8 +321,7 @@ var PipelineUploadCommand = cli.Command{
 					// Interpolate merges the pipeline's env block into `environ`.
 					err := searchForSecrets(l, &cfg, environ, result, input.name)
 					if err != nil {
-						l.Error("%v", err)
-						return NewSilentExitError(1)
+						return NewExitError(1, err)
 					}
 				}
 

From 25c9741dfed2aaa7f96334178cffa46a45fa80e8 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 17 Nov 2025 11:15:29 +1100
Subject: [PATCH 070/242] Clarify agent idlemonitor states

---
 agent/idle_monitor.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/agent/idle_monitor.go b/agent/idle_monitor.go
index d9a4557118..1f3743c676 100644
--- a/agent/idle_monitor.go
+++ b/agent/idle_monitor.go
@@ -9,7 +9,9 @@ import (
 // logic.
 //
 // In addition to "busy", "idle", and "dead", idleMonitor has an implicit
-// "initial" state. Agents always start in the "initial" state
+// "initial" state. Agents always start in the "initial" state, but typically
+// quickly transistion into either the idle or busy states (as soon as they
+// have completed their first ping.)
 /*
 //                -> Busy --
 //              /     ^      \
@@ -17,8 +19,6 @@ import (
 //              \     v      /
 //                -> Idle --
 */
-// This (intentionally) ensures the DisconnectAfterIdleTimeout doesn't fire
-// before agents have had a chance to run a job.
 type idleMonitor struct {
 	mu          sync.Mutex
 	exiting     bool

From cb40e03222a08c562de6664b55f32768d6fb8edf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Nov 2025 15:09:28 +0000
Subject: [PATCH 071/242] build(deps): bump the cloud-providers group with 7
 updates

Bumps the cloud-providers group with 7 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/Azure/azure-sdk-for-go/sdk/azidentity](https://github.com/Azure/azure-sdk-for-go) | `1.13.0` | `1.13.1` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.17` | `1.31.20` |
| [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.20.4` | `1.20.7` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.264.0` | `1.269.0` |
| [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) | `1.48.0` | `1.48.2` |
| [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.90.0` | `1.90.2` |
| [google.golang.org/api](https://github.com/googleapis/google-api-go-client) | `0.255.0` | `0.256.0` |


Updates `github.com/Azure/azure-sdk-for-go/sdk/azidentity` from 1.13.0 to 1.13.1
- [Release notes](https://github.com/Azure/azure-sdk-for-go/releases)
- [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.13.0...sdk/azidentity/v1.13.1)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.17 to 1.31.20
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.17...config/v1.31.20)

Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.20.4 to 1.20.7
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/mq/v1.20.4...service/mq/v1.20.7)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.264.0 to 1.269.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.264.0...service/ec2/v1.269.0)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.48.0 to 1.48.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.48.0...service/kms/v1.48.2)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.90.0 to 1.90.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.90.0...service/s3/v1.90.2)

Updates `google.golang.org/api` from 0.255.0 to 0.256.0
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.255.0...v0.256.0)

---
updated-dependencies:
- dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azidentity
  dependency-version: 1.13.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.31.20
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-version: 1.20.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.269.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.48.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-version: 1.90.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: google.golang.org/api
  dependency-version: 0.256.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 30 ++++++++++++++---------------
 go.sum | 60 +++++++++++++++++++++++++++++-----------------------------
 2 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/go.mod b/go.mod
index 3f8c807019..1e8714a0c7 100644
--- a/go.mod
+++ b/go.mod
@@ -7,17 +7,17 @@ toolchain go1.24.5
 require (
 	cloud.google.com/go/compute/metadata v0.9.0
 	drjosh.dev/zzglob v0.4.2
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go-v2 v1.39.6
-	github.com/aws/aws-sdk-go-v2/config v1.31.17
+	github.com/aws/aws-sdk-go-v2/config v1.31.20
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.264.0
-	github.com/aws/aws-sdk-go-v2/service/kms v1.48.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.269.0
+	github.com/aws/aws-sdk-go-v2/service/kms v1.48.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2
 	github.com/aws/smithy-go v1.23.2
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
@@ -61,7 +61,7 @@ require (
 	golang.org/x/sync v0.18.0
 	golang.org/x/sys v0.38.0
 	golang.org/x/term v0.36.0
-	google.golang.org/api v0.255.0
+	google.golang.org/api v0.256.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
@@ -70,9 +70,9 @@ require (
 require (
 	cloud.google.com/go/auth v0.17.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
 	github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/obfuscate v0.67.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/proto v0.67.0 // indirect
@@ -95,7 +95,7 @@ require (
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.21 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.18.24 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -104,9 +104,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
@@ -132,7 +132,7 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
 	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
@@ -201,7 +201,7 @@ require (
 	golang.org/x/tools v0.37.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect
 	google.golang.org/grpc v1.76.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/go.sum b/go.sum
index 6cb6478b21..d6bd24a6b7 100644
--- a/go.sum
+++ b/go.sum
@@ -6,10 +6,10 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 drjosh.dev/zzglob v0.4.2 h1:q+e5Cp6SFCyz+Yurhk/edSrTKEk3tn60vzoaXLmtiBo=
 drjosh.dev/zzglob v0.4.2/go.mod h1:SbYDdesQC13iyGiEwV8dJfJbyz7/Qiawrd5ODdJQCoo=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 h1:KpMC6LFL7mqpExyMC9jVOYRiVhLmamjeZfRsUpB7l4s=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
@@ -20,8 +20,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehw
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0 h1:2mEwRWvhIPHMPK4CMD8iKbsrYBxeMBSuuCXumQAwShU=
 github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0/go.mod h1:ejJHsyJTG7NU6c6TDbF7dmckD3g+AUGSdiSXy+ZyaCE=
@@ -80,14 +80,14 @@ github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+X
 github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
-github.com/aws/aws-sdk-go-v2/config v1.31.17 h1:QFl8lL6RgakNK86vusim14P2k8BFSxjvUkcWLDjgz9Y=
-github.com/aws/aws-sdk-go-v2/config v1.31.17/go.mod h1:V8P7ILjp/Uef/aX8TjGk6OHZN6IKPM5YW6S78QnRD5c=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.21 h1:56HGpsgnmD+2/KpG0ikvvR8+3v3COCwaF4r+oWwOeNA=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.21/go.mod h1:3YELwedmQbw7cXNaII2Wywd+YY58AmLPwX4LzARgmmA=
+github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc=
+github.com/aws/aws-sdk-go-v2/config v1.31.20/go.mod h1:95Hh1Tc5VYKL9NJ7tAkDcqeKt+MCXQB1hQZaRdJIZE0=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.24 h1:iJ2FmPT35EaIB0+kMa6TnQ+PwG5A1prEdAw+PsMzfHg=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.24/go.mod h1:U91+DrfjAiXPDEGYhh/x29o4p0qHX5HDqG7y5VViv64=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7 h1:u8danF+A2Zv//pFZvj5V23v/6XG4AxuSVup5s6nxSnI=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7/go.mod h1:uvLIvU8iJPEU5so7b6lLDNArWpOX6sRBfL5wBABmlfc=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
@@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.264.0 h1:3SsIzhGS28WMDppm5VLeTM9qxrN7vhxDRlUUi54NXRE=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.264.0/go.mod h1:NDdDLLW5PtLLXN661gKcvJvqAH5OBXsfhMlmKVu1/pY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.269.0 h1:JMU0UqLvPUKovF/kXcIQf1ZVyv3+BwVW5O3bZrXDqBo=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.269.0/go.mod h1:NDdDLLW5PtLLXN661gKcvJvqAH5OBXsfhMlmKVu1/pY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
@@ -106,16 +106,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
-github.com/aws/aws-sdk-go-v2/service/kms v1.48.0 h1:pQgVxqqNOacqb19+xaoih/wNLil4d8tgi+FxtBi/qQY=
-github.com/aws/aws-sdk-go-v2/service/kms v1.48.0/go.mod h1:VJcNH6BLr+3VJwinRKdotLOMglHO8mIKlD3ea5c7hbw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 h1:mLlUgHn02ue8whiR4BmxxGJLR2gwU6s6ZzJ5wDamBUs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.1/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
+github.com/aws/aws-sdk-go-v2/service/kms v1.48.2 h1:aL8Y/AbB6I+uw0MjLbdo68NQ8t5lNs3CY3S848HpETk=
+github.com/aws/aws-sdk-go-v2/service/kms v1.48.2/go.mod h1:VJcNH6BLr+3VJwinRKdotLOMglHO8mIKlD3ea5c7hbw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 h1:DhdbtDl4FdNlj31+xiRXANxEE+eC7n8JQz+/ilwQ8Uc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 h1:NjShtS1t8r5LUfFVtFeI8xLAHQNTa7UI0VawXlrBMFQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.3/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 h1:gTsnx0xXNQ6SBbymoDvcoRHL+q4l/dAFsQuKfDWSaGc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 h1:HK5ON3KmQV2HcAunnx4sKLB9aPf3gKGwVAf7xnx0QT0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.40.2/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
 github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
 github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -227,8 +227,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
-github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
+github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
+github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
 github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
 github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
@@ -546,16 +546,16 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.255.0 h1:OaF+IbRwOottVCYV2wZan7KUq7UeNUQn1BcPc4K7lE4=
-google.golang.org/api v0.255.0/go.mod h1:d1/EtvCLdtiWEV4rAEHDHGh2bCnqsWhw+M8y2ECN4a8=
+google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI=
+google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
 google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 h1:8XJ4pajGwOlasW+L13MnEGA8W4115jJySQtVfS2/IBU=
 google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4/go.mod h1:NnuHhy+bxcg30o7FnVAZbXsPHUDQ9qKWAQKCD7VxFtk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
 google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
 google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=

From b9c64619f17d7609abf32e7df9be8294992448b1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Nov 2025 22:27:13 +0000
Subject: [PATCH 072/242] build(deps): bump the golang-x group with 3 updates

Bumps the golang-x group with 3 updates: [golang.org/x/crypto](https://github.com/golang/crypto), [golang.org/x/net](https://github.com/golang/net) and [golang.org/x/term](https://github.com/golang/term).


Updates `golang.org/x/crypto` from 0.43.0 to 0.44.0
- [Commits](https://github.com/golang/crypto/compare/v0.43.0...v0.44.0)

Updates `golang.org/x/net` from 0.46.0 to 0.47.0
- [Commits](https://github.com/golang/net/compare/v0.46.0...v0.47.0)

Updates `golang.org/x/term` from 0.36.0 to 0.37.0
- [Commits](https://github.com/golang/term/compare/v0.36.0...v0.37.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.44.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/net
  dependency-version: 0.47.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/term
  dependency-version: 0.37.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 12 ++++++------
 go.sum | 24 ++++++++++++------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/go.mod b/go.mod
index 1e8714a0c7..c97d07c469 100644
--- a/go.mod
+++ b/go.mod
@@ -55,12 +55,12 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
 	go.opentelemetry.io/otel/sdk v1.38.0
 	go.opentelemetry.io/otel/trace v1.38.0
-	golang.org/x/crypto v0.43.0
-	golang.org/x/net v0.46.0
+	golang.org/x/crypto v0.44.0
+	golang.org/x/net v0.47.0
 	golang.org/x/oauth2 v0.33.0
 	golang.org/x/sync v0.18.0
 	golang.org/x/sys v0.38.0
-	golang.org/x/term v0.36.0
+	golang.org/x/term v0.37.0
 	google.golang.org/api v0.256.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
@@ -195,10 +195,10 @@ require (
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
-	golang.org/x/mod v0.28.0 // indirect
-	golang.org/x/text v0.30.0 // indirect
+	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.37.0 // indirect
+	golang.org/x/tools v0.38.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect
diff --git a/go.sum b/go.sum
index d6bd24a6b7..1dfd7e2f27 100644
--- a/go.sum
+++ b/go.sum
@@ -483,22 +483,22 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
+golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U=
-golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI=
+golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
+golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
-golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo=
 golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -523,12 +523,12 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
 golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
-golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -536,8 +536,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
-golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
+golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
+golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From 55dcd3e3989da6eb03eec0f19e82839fae991ae1 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 18 Nov 2025 10:00:53 +1100
Subject: [PATCH 073/242] Bump version and changelog for v3.113.0

---
 CHANGELOG.md    | 18 ++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8767ee5ba7..487964b1fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.113.0](https://github.com/buildkite/agent/tree/v3.113.0) (2025-11-18)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.112.0...v3.113.0)
+
+### Added
+- Add Prometheus /metrics handler and some basic metrics [#3576](https://github.com/buildkite/agent/pull/3576) (@DrJosh9000)
+
+### Fixed
+- Fix the pipeline upload --reject-secrets flag not rejecting secrets [#3580](https://github.com/buildkite/agent/pull/3580) (@moskyb)
+- Fix idle tracking for agents that never received jobs [#3579](https://github.com/buildkite/agent/pull/3579) (@scadu)
+
+### Internal
+- Clarify agent idlemonitor states in comment [#3582](https://github.com/buildkite/agent/pull/3582) (@DrJosh9000)
+- Put secret scan error into exit message [#3581](https://github.com/buildkite/agent/pull/3581) (@DrJosh9000)
+
+### Dependency updates
+- build(deps): bump the golang-x group with 3 updates [#3583](https://github.com/buildkite/agent/pull/3583) (@dependabot[bot])
+- build(deps): bump the cloud-providers group with 7 updates [#3584](https://github.com/buildkite/agent/pull/3584) (@dependabot[bot])
+
 ## [v3.112.0](https://github.com/buildkite/agent/tree/v3.112.0) (2025-11-12)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.111.0...v3.112.0)
 
diff --git a/version/VERSION b/version/VERSION
index 87f3781675..e5abe2231a 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.112.0
+3.113.0

From 5aef3cc7b6a7a9e4a5530cd6317612771f9456bf Mon Sep 17 00:00:00 2001
From: Linjie Ding <linjie@anthropic.com>
Date: Tue, 18 Nov 2025 20:16:52 -0500
Subject: [PATCH 074/242] feat: add agent metadata to OTEL trace attributes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add agent registration tags as OTEL trace attributes with the prefix
`buildkite.agent.metadata.*`. The tags are read from existing
BUILDKITE_AGENT_META_DATA_* environment variables that come from the
API's Job.Env response.

Example: BUILDKITE_AGENT_META_DATA_QUEUE=default becomes
buildkite.agent.metadata.queue=default

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 internal/job/tracing.go | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/internal/job/tracing.go b/internal/job/tracing.go
index 1576bb3507..3d8e593684 100644
--- a/internal/job/tracing.go
+++ b/internal/job/tracing.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"slices"
 	"strconv"
+	"strings"
 
 	"github.com/buildkite/agent/v3/env"
 	"github.com/buildkite/agent/v3/tracetools"
@@ -259,7 +260,7 @@ func GenericTracingExtras(e *Executor, env *env.Environment) map[string]any {
 		jobKey = "n/a"
 	}
 
-	return map[string]any{
+	result := map[string]any{
 		"buildkite.agent":             e.AgentName,
 		"buildkite.version":           version.Version(),
 		"buildkite.queue":             e.Queue,
@@ -279,6 +280,19 @@ func GenericTracingExtras(e *Executor, env *env.Environment) map[string]any {
 		"buildkite.rebuilt_from_id":   rebuiltFromID,
 		"buildkite.triggered_from_id": triggeredFromID,
 	}
+
+	// Add agent metadata from BUILDKITE_AGENT_META_DATA_* env vars
+	// These come from the agent's registration tags
+	const metaDataPrefix = "BUILDKITE_AGENT_META_DATA_"
+	for key, value := range env.Dump() {
+		if after, found := strings.CutPrefix(key, metaDataPrefix); found {
+			// Convert key to lowercase for attribute naming
+			attrKey := "buildkite.agent.metadata." + strings.ToLower(after)
+			result[attrKey] = value
+		}
+	}
+
+	return result
 }
 
 func DDTracingExtras() map[string]any {

From 47b5a39d3853af7f7019f0e544db50e153f7a4c6 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 20 Nov 2025 11:28:03 +1100
Subject: [PATCH 075/242] Run gofumpt as part of CI

---
 .buildkite/steps/check-code-committed.sh | 14 ++++++++------
 go.mod                                   |  2 ++
 go.sum                                   |  4 ++++
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.buildkite/steps/check-code-committed.sh b/.buildkite/steps/check-code-committed.sh
index 3e029c7067..5c8c43633b 100755
--- a/.buildkite/steps/check-code-committed.sh
+++ b/.buildkite/steps/check-code-committed.sh
@@ -12,12 +12,14 @@ if ! git diff --no-ext-diff --exit-code; then
   exit 1
 fi
 
-echo --- :go: Checking go formatting
-gofmt -w .
-if ! git diff --no-ext-diff --exit-code; then
+echo +++ :go: Checking go formatting
+
+fumpt_out=$(go tool gofumpt -extra -l .)
+if ! [ -z "${fumpt_out}" ]; then
   echo ^^^ +++
-  echo "Files have not been formatted with gofmt."
-  echo "Fix this by running \`go fmt ./...\` locally, and committing the result."
+  echo "Files have not been formatted with gofumpt:"
+  echo "${fumpt_out}"
+  echo "Fix this by running \`gofumpt -extra -w .\` locally, and committing the result."
 
   exit 1
 fi
@@ -35,7 +37,7 @@ if ! git diff --no-ext-diff --exit-code; then
 fi
 
 echo +++ :go: Running golangci-lint...
-if ! lint_out="$(golangci-lint run --color=always)" ; then 
+if ! lint_out="$(golangci-lint run --color=always)" ; then
   echo ^^^ +++
   echo "golangci-lint found the following issues:"
   echo ""
diff --git a/go.mod b/go.mod
index c97d07c469..57e488e1b9 100644
--- a/go.mod
+++ b/go.mod
@@ -207,10 +207,12 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/gotestsum v1.13.0 // indirect
+	mvdan.cc/gofumpt v0.9.2 // indirect
 )
 
 tool (
 	github.com/Khan/genqlient/generate
 	github.com/buildkite/test-engine-client
 	gotest.tools/gotestsum
+	mvdan.cc/gofumpt
 )
diff --git a/go.sum b/go.sum
index 1dfd7e2f27..6a52593829 100644
--- a/go.sum
+++ b/go.sum
@@ -195,6 +195,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
@@ -578,3 +580,5 @@ gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
 gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
 k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
 k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
+mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4=
+mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s=

From 455047ae428520ac112f52c8892728e881483068 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 20 Nov 2025 11:55:38 +1100
Subject: [PATCH 076/242] Run gofumpt -extra -w across the whole repo

---
 agent/agent_pool.go                           |  1 -
 agent/agent_worker.go                         |  1 -
 agent/agent_worker_test.go                    |  1 -
 agent/api.go                                  |  3 ++-
 agent/ec2_meta_data.go                        |  3 +--
 agent/gcp_labels.go                           |  1 -
 .../config_allowlisting_integration_test.go   |  1 -
 agent/integration/test_helpers.go             |  1 -
 agent/job_runner.go                           |  2 --
 agent/log_streamer.go                         |  2 +-
 agent/plugin/definition_test.go               |  1 -
 agent/tags.go                                 |  5 -----
 api/annotations.go                            |  2 +-
 api/client.go                                 |  3 +--
 api/github_code_access_token.go               |  1 -
 clicommand/agent_pause.go                     |  1 -
 clicommand/agent_start.go                     | 19 +++++++++----------
 clicommand/agent_stop.go                      |  1 -
 clicommand/build_cancel.go                    |  1 -
 clicommand/meta_data_get.go                   |  1 -
 clicommand/pipeline_upload.go                 |  4 +---
 clicommand/pipeline_upload_test.go            |  1 -
 cliconfig/loader.go                           |  6 +++---
 env/environment.go                            |  2 +-
 env/environment_test.go                       |  1 -
 internal/artifact/artifactory_uploader.go     |  4 ++--
 internal/artifact/azure_blob.go               |  1 -
 internal/artifact/batch_creator.go            |  1 -
 internal/artifact/bk_uploader_test.go         |  3 ++-
 internal/artifact/gs_uploader.go              |  6 +++---
 internal/artifact/s3_uploader.go              |  1 -
 internal/artifact/uploader.go                 |  3 +--
 internal/artifact/uploader_test.go            |  2 +-
 internal/file/opened_by.go                    |  2 +-
 internal/job/checkout.go                      |  2 +-
 internal/job/docker.go                        |  2 +-
 internal/job/executor.go                      |  3 +--
 internal/job/githttptest/githttptest.go       |  3 ---
 .../checkout_git_mirrors_integration_test.go  |  2 +-
 .../integration/checkout_integration_test.go  |  2 +-
 internal/job/integration/executor_tester.go   |  2 +-
 .../job/integration/test-binary-hook/main.go  |  1 -
 internal/job/ssh.go                           |  5 +----
 internal/mime/generate.go                     | 14 ++++++++------
 internal/replacer/bm_redactor_test.go         |  2 --
 internal/replacer/replacer_test.go            |  3 +--
 internal/shell/logger.go                      |  4 ++--
 internal/shell/logger_test.go                 |  1 -
 internal/shell/lookpath.go                    |  2 +-
 internal/shell/lookpath_windows.go            |  2 +-
 internal/system/version_dump_windows.go       |  1 +
 jobapi/server_test.go                         |  1 -
 logger/buffer.go                              |  6 ++++++
 main.go                                       |  2 +-
 process/format.go                             |  2 +-
 process/run.go                                |  1 -
 process/scanner.go                            |  2 +-
 tracetools/propagate.go                       |  6 ++++--
 tracetools/span.go                            |  2 +-
 59 files changed, 65 insertions(+), 96 deletions(-)

diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index 8fccaae80f..cf769544d1 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -154,7 +154,6 @@ func (ap *AgentPool) statusJSONHandler(l logger.Logger) http.HandlerFunc {
 			AggregateStatus: aggregateState,
 			Workers:         statuses,
 		})
-
 		if err != nil {
 			l.Error("Could not encode status.json response: %v", err)
 		}
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 9ff4bbcc54..8ceb3287f2 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -558,7 +558,6 @@ func (a *AgentWorker) Connect(ctx context.Context) error {
 
 // Performs a heatbeat
 func (a *AgentWorker) Heartbeat(ctx context.Context) error {
-
 	// Retry the heartbeat a few times
 	r := roko.NewRetrier(
 		roko.WithMaxAttempts(10),
diff --git a/agent/agent_worker_test.go b/agent/agent_worker_test.go
index 6fed4f6f1b..67053b32f0 100644
--- a/agent/agent_worker_test.go
+++ b/agent/agent_worker_test.go
@@ -325,7 +325,6 @@ func TestAgentWorker_Start_AcquireJob_JobAcquisitionRejected(t *testing.T) {
 	if !errors.Is(err, core.ErrJobAcquisitionRejected) {
 		t.Fatalf("expected worker.AcquireAndRunJob(%q) = core.ErrJobAcquisitionRejected, got %v", jobID, err)
 	}
-
 }
 
 func TestAgentWorker_Start_AcquireJob_Pause_Unpause(t *testing.T) {
diff --git a/agent/api.go b/agent/api.go
index 9534e1922d..6065193ae4 100644
--- a/agent/api.go
+++ b/agent/api.go
@@ -4,8 +4,9 @@ package agent
 
 import (
 	"context"
-	"github.com/buildkite/agent/v3/api"
 	"net/http"
+
+	"github.com/buildkite/agent/v3/api"
 )
 
 // APIClient is an interface generated for "github.com/buildkite/agent/v3/api.Client".
diff --git a/agent/ec2_meta_data.go b/agent/ec2_meta_data.go
index 27c7a364c2..1084999bd5 100644
--- a/agent/ec2_meta_data.go
+++ b/agent/ec2_meta_data.go
@@ -9,8 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 )
 
-type EC2MetaData struct {
-}
+type EC2MetaData struct{}
 
 // Takes a map of tags and meta-data paths to get, returns a map of tags and fetched values.
 func (e EC2MetaData) GetPaths(ctx context.Context, paths map[string]string) (map[string]string, error) {
diff --git a/agent/gcp_labels.go b/agent/gcp_labels.go
index a6fee5eb5c..f8b1e33bf0 100644
--- a/agent/gcp_labels.go
+++ b/agent/gcp_labels.go
@@ -33,7 +33,6 @@ func (e GCPLabels) Get(ctx context.Context) (map[string]string, error) {
 		meta["gcp:zone"],
 		meta["gcp:instance-name"],
 	).Context(ctx).Do()
-
 	if err != nil {
 		return nil, err
 	}
diff --git a/agent/integration/config_allowlisting_integration_test.go b/agent/integration/config_allowlisting_integration_test.go
index bdc5742098..a85b65d27c 100644
--- a/agent/integration/config_allowlisting_integration_test.go
+++ b/agent/integration/config_allowlisting_integration_test.go
@@ -107,7 +107,6 @@ func TestConfigAllowlisting(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
diff --git a/agent/integration/test_helpers.go b/agent/integration/test_helpers.go
index 1adbc67a5b..89d9cda392 100644
--- a/agent/integration/test_helpers.go
+++ b/agent/integration/test_helpers.go
@@ -76,7 +76,6 @@ func runJob(t *testing.T, ctx context.Context, cfg testRunJobConfig) error {
 		MetricsScope:       scope,
 		JobStatusInterval:  1 * time.Second,
 	})
-
 	if err != nil {
 		t.Fatalf("agent.NewJobRunner() error = %v", err)
 	}
diff --git a/agent/job_runner.go b/agent/job_runner.go
index 51ca52e568..62e6412609 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -822,7 +822,6 @@ func (r *JobRunner) jobCancellationChecker(ctx context.Context, wg *sync.WaitGro
 
 		// Re-get the job and check its status to see if it's been cancelled
 		jobState, response, err := r.apiClient.GetJobState(ctx, r.conf.Job.ID)
-
 		if err != nil {
 			if response != nil && response.StatusCode == 401 {
 				r.agentLogger.Error("Invalid access token, cancelling job %s", r.conf.Job.ID)
@@ -860,7 +859,6 @@ func (r *JobRunner) onUploadHeaderTime(ctx context.Context, cursor, total int, t
 
 		return err
 	})
-
 	if err != nil {
 		r.agentLogger.Error("Ultimately unable to upload header times: %v", err)
 	}
diff --git a/agent/log_streamer.go b/agent/log_streamer.go
index 3ec4d33904..8db1d239c2 100644
--- a/agent/log_streamer.go
+++ b/agent/log_streamer.go
@@ -127,7 +127,7 @@ func (ls *LogStreamer) Process(ctx context.Context, output []byte) error {
 				humanize.IBytes(ls.bytes), humanize.IBytes(ls.conf.MaxSizeBytes))
 			ls.warnedAboutSize = true
 			// In a future version, this will error out, e.g.:
-			//return fmt.Errorf("%w (%d > %d)", errLogExceededMaxSize, ls.bytes, ls.conf.MaxSizeBytes)
+			// return fmt.Errorf("%w (%d > %d)", errLogExceededMaxSize, ls.bytes, ls.conf.MaxSizeBytes)
 		}
 
 		// The next chunk will be up to MaxChunkSizeBytes in size.
diff --git a/agent/plugin/definition_test.go b/agent/plugin/definition_test.go
index ef0de6b462..5d2447b95b 100644
--- a/agent/plugin/definition_test.go
+++ b/agent/plugin/definition_test.go
@@ -34,7 +34,6 @@ configuration:
 
 func TestDefinitionParsesYaml(t *testing.T) {
 	def, err := ParseDefinition([]byte(testPluginDef))
-
 	if err != nil {
 		t.Fatalf("ParseDefinition(testPluginDef) error = %v", err)
 	}
diff --git a/agent/tags.go b/agent/tags.go
index 468e5fa60e..7d9033a6fe 100644
--- a/agent/tags.go
+++ b/agent/tags.go
@@ -129,7 +129,6 @@ func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsC
 
 			return err
 		})
-
 		// Don't blow up if we can't find them, just show a nasty error.
 		if err != nil {
 			l.Error(fmt.Sprintf("Failed to fetch EC2 meta-data: %s", err.Error()))
@@ -180,7 +179,6 @@ func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsC
 			}
 			return err
 		})
-
 		// Don't blow up if we can't find them, just show a nasty error.
 		if err != nil {
 			l.Error(fmt.Sprintf("Failed to find EC2 tags: %s", err.Error()))
@@ -209,7 +207,6 @@ func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsC
 
 			return err
 		})
-
 		// Don't blow up if we can't find them, just show a nasty error.
 		if err != nil {
 			l.Error(fmt.Sprintf("Failed to fetch ECS meta-data: %s", err.Error()))
@@ -238,7 +235,6 @@ func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsC
 
 			return nil
 		})
-
 		// Don't blow up if we can't find them, just show a nasty error.
 		if err != nil {
 			l.Error(fmt.Sprintf("Failed to fetch GCP meta-data: %s", err.Error()))
@@ -286,7 +282,6 @@ func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsC
 			}
 			return err
 		})
-
 		// Don't blow up if we can't find them, just show a nasty error.
 		if err != nil {
 			l.Error(fmt.Sprintf("Failed to find GCP instance labels: %s", err.Error()))
diff --git a/api/annotations.go b/api/annotations.go
index ae8157c8e4..2871d75bae 100644
--- a/api/annotations.go
+++ b/api/annotations.go
@@ -29,7 +29,7 @@ func (c *Client) Annotate(ctx context.Context, jobId string, annotation *Annotat
 }
 
 // Remove an annotation from a build
-func (c *Client) AnnotationRemove(ctx context.Context, jobId string, context, scope string) (*Response, error) {
+func (c *Client) AnnotationRemove(ctx context.Context, jobId, context, scope string) (*Response, error) {
 	u := fmt.Sprintf("jobs/%s/annotations/%s", railsPathEscape(jobId), railsPathEscape(context))
 
 	req, err := c.newRequest(ctx, "DELETE", u, nil)
diff --git a/api/client.go b/api/client.go
index 11a8515365..1dc70f6512 100644
--- a/api/client.go
+++ b/api/client.go
@@ -309,7 +309,6 @@ func newResponse(r *http.Response) *Response {
 // interface, the raw response body will be written to v, without attempting to
 // first decode it.
 func (c *Client) doRequest(req *http.Request, v any) (*Response, error) {
-
 	resp, err := agenthttp.Do(c.logger, c.client, req,
 		agenthttp.WithDebugHTTP(c.conf.DebugHTTP),
 		agenthttp.WithTraceHTTP(c.conf.TraceHTTP),
@@ -411,7 +410,7 @@ func addOptions(s string, opt any) (string, error) {
 	return u.String(), nil
 }
 
-func joinURLPath(endpoint string, path string) string {
+func joinURLPath(endpoint, path string) string {
 	return strings.TrimRight(endpoint, "/") + "/" + strings.TrimLeft(path, "/")
 }
 
diff --git a/api/github_code_access_token.go b/api/github_code_access_token.go
index 2ec224b29d..b7a369521b 100644
--- a/api/github_code_access_token.go
+++ b/api/github_code_access_token.go
@@ -54,7 +54,6 @@ func (c *Client) GenerateGithubCodeAccessToken(ctx context.Context, repoURL, job
 
 		return resp, err
 	})
-
 	if err != nil {
 		return "", resp, err
 	}
diff --git a/clicommand/agent_pause.go b/clicommand/agent_pause.go
index 5b17a21e6c..94cbaa3c1f 100644
--- a/clicommand/agent_pause.go
+++ b/clicommand/agent_pause.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"slices"
-
 	"time"
 
 	"github.com/buildkite/agent/v3/api"
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 0ea9a2f2d8..571ef31407 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1071,16 +1071,15 @@ var AgentStartCommand = cli.Command{
 		}
 
 		if cfg.LogFormat == "text" {
-			welcomeMessage :=
-				"\n" +
-					"%s   _           _ _     _ _    _ _                                _\n" +
-					"  | |         (_) |   | | |  (_) |                              | |\n" +
-					"  | |__  _   _ _| | __| | | ___| |_ ___    __ _  __ _  ___ _ __ | |_\n" +
-					"  | '_ \\| | | | | |/ _` | |/ / | __/ _ \\  / _` |/ _` |/ _ \\ '_ \\| __|\n" +
-					"  | |_) | |_| | | | (_| |   <| | ||  __/ | (_| | (_| |  __/ | | | |_\n" +
-					"  |_.__/ \\__,_|_|_|\\__,_|_|\\_\\_|\\__\\___|  \\__,_|\\__, |\\___|_| |_|\\__|\n" +
-					"                                                 __/ |\n" +
-					" https://buildkite.com/agent                    |___/\n%s\n"
+			welcomeMessage := "\n" +
+				"%s   _           _ _     _ _    _ _                                _\n" +
+				"  | |         (_) |   | | |  (_) |                              | |\n" +
+				"  | |__  _   _ _| | __| | | ___| |_ ___    __ _  __ _  ___ _ __ | |_\n" +
+				"  | '_ \\| | | | | |/ _` | |/ / | __/ _ \\  / _` |/ _` |/ _ \\ '_ \\| __|\n" +
+				"  | |_) | |_| | | | (_| |   <| | ||  __/ | (_| | (_| |  __/ | | | |_\n" +
+				"  |_.__/ \\__,_|_|_|\\__,_|_|\\_\\_|\\__\\___|  \\__,_|\\__, |\\___|_| |_|\\__|\n" +
+				"                                                 __/ |\n" +
+				" https://buildkite.com/agent                    |___/\n%s\n"
 
 			if !cfg.NoColor {
 				fmt.Fprintf(os.Stderr, welcomeMessage, "\x1b[38;5;48m", "\x1b[0m")
diff --git a/clicommand/agent_stop.go b/clicommand/agent_stop.go
index dcbf703aba..7186aa4069 100644
--- a/clicommand/agent_stop.go
+++ b/clicommand/agent_stop.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"slices"
-
 	"time"
 
 	"github.com/buildkite/agent/v3/api"
diff --git a/clicommand/build_cancel.go b/clicommand/build_cancel.go
index 099a53e180..c38558d7de 100644
--- a/clicommand/build_cancel.go
+++ b/clicommand/build_cancel.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"slices"
-
 	"time"
 
 	"github.com/buildkite/agent/v3/api"
diff --git a/clicommand/meta_data_get.go b/clicommand/meta_data_get.go
index 9ea00ef4d8..aa4346f66a 100644
--- a/clicommand/meta_data_get.go
+++ b/clicommand/meta_data_get.go
@@ -91,7 +91,6 @@ var MetaDataGetCommand = cli.Command{
 			}
 			return metaData, resp, nil
 		})
-
 		if err != nil {
 			// Buildkite returns a 404 if the key doesn't exist. If
 			// we get this status, and we've got a default - return
diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index e71cea9674..b9df628264 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -325,9 +325,7 @@ var PipelineUploadCommand = cli.Command{
 					}
 				}
 
-				var (
-					key signature.Key
-				)
+				var key signature.Key
 
 				switch {
 				case cfg.SigningAWSKMSKey != "":
diff --git a/clicommand/pipeline_upload_test.go b/clicommand/pipeline_upload_test.go
index 0641891c76..32bbf9a391 100644
--- a/clicommand/pipeline_upload_test.go
+++ b/clicommand/pipeline_upload_test.go
@@ -740,5 +740,4 @@ func TestIfChangedApplicator_WeirdPipeline(t *testing.T) {
 	if diff := cmp.Diff(steps, want); diff != "" {
 		t.Errorf("after ica.apply(l, steps) (-got, +want):\n%s", diff)
 	}
-
 }
diff --git a/cliconfig/loader.go b/cliconfig/loader.go
index 9ec4b2eeea..6361a21d39 100644
--- a/cliconfig/loader.go
+++ b/cliconfig/loader.go
@@ -173,7 +173,7 @@ func (l *Loader) Load() (warnings []string, err error) {
 	return warnings, nil
 }
 
-func (l Loader) setFieldValueFromCLI(fieldName string, cliName string) error {
+func (l Loader) setFieldValueFromCLI(fieldName, cliName string) error {
 	// Get the kind of field we need to set
 	fieldKind, err := reflections.GetFieldKind(l.Config, fieldName)
 	if err != nil {
@@ -341,7 +341,7 @@ func (l Loader) fieldValueIsEmpty(fieldName string) bool {
 	}
 }
 
-func (l Loader) validateField(fieldName string, label string, validationRules string) error {
+func (l Loader) validateField(fieldName, label, validationRules string) error {
 	// Split up the validation rules
 	rules := strings.SplitSeq(validationRules, ",")
 
@@ -372,7 +372,7 @@ func (l Loader) validateField(fieldName string, label string, validationRules st
 	return nil
 }
 
-func (l Loader) normalizeField(fieldName string, normalization string) error {
+func (l Loader) normalizeField(fieldName, normalization string) error {
 	if normalization == "filepath" {
 		value, _ := reflections.GetField(l.Config, fieldName)
 		fieldKind, _ := reflections.GetFieldKind(l.Config, fieldName)
diff --git a/env/environment.go b/env/environment.go
index 22654e6463..c40a1ab935 100644
--- a/env/environment.go
+++ b/env/environment.go
@@ -146,7 +146,7 @@ func (e *Environment) Exists(key string) bool {
 }
 
 // Set sets a key in the environment
-func (e *Environment) Set(key string, value string) {
+func (e *Environment) Set(key, value string) {
 	e.underlying.Store(normalizeKeyName(key), value)
 }
 
diff --git a/env/environment_test.go b/env/environment_test.go
index a5ee99fefc..120b22c010 100644
--- a/env/environment_test.go
+++ b/env/environment_test.go
@@ -72,7 +72,6 @@ func TestEnvironmentSet_NormalizesKeyNames(t *testing.T) {
 		upper, _ := e.Get(strings.ToUpper(mountain))
 		assert.Equal(t, upper, "Cerro Poincenot")
 	}
-
 }
 
 func TestEnvironmentGetBool(t *testing.T) {
diff --git a/internal/artifact/artifactory_uploader.go b/internal/artifact/artifactory_uploader.go
index 3b1f498241..34180d29e7 100644
--- a/internal/artifact/artifactory_uploader.go
+++ b/internal/artifact/artifactory_uploader.go
@@ -88,11 +88,11 @@ func NewArtifactoryUploader(l logger.Logger, c ArtifactoryUploaderConfig) (*Arti
 	}, nil
 }
 
-func ParseArtifactoryDestination(destination string) (repo string, path string) {
+func ParseArtifactoryDestination(destination string) (repo, path string) {
 	parts := strings.Split(strings.TrimPrefix(string(destination), "rt://"), "/")
 	path = strings.Join(parts[1:], "/")
 	repo = parts[0]
-	return
+	return repo, path
 }
 
 func (u *ArtifactoryUploader) URL(artifact *api.Artifact) string {
diff --git a/internal/artifact/azure_blob.go b/internal/artifact/azure_blob.go
index f84dd85c13..7212518681 100644
--- a/internal/artifact/azure_blob.go
+++ b/internal/artifact/azure_blob.go
@@ -17,7 +17,6 @@ const azureBlobHostSuffix = ".blob.core.windows.net"
 
 // NewAzureBlobClient creates a new Azure Blob Storage client.
 func NewAzureBlobClient(l logger.Logger, storageAccountName string) (*service.Client, error) {
-
 	// TODO: Other credential types?
 	// https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#readme-credential-types
 
diff --git a/internal/artifact/batch_creator.go b/internal/artifact/batch_creator.go
index f761ae950b..208043f3d7 100644
--- a/internal/artifact/batch_creator.go
+++ b/internal/artifact/batch_creator.go
@@ -108,7 +108,6 @@ func (a *BatchCreator) Create(ctx context.Context) ([]*api.Artifact, error) {
 
 			return creation, err
 		})
-
 		// Did the batch creation eventually fail?
 		if err != nil {
 			return nil, err
diff --git a/internal/artifact/bk_uploader_test.go b/internal/artifact/bk_uploader_test.go
index 0888694081..49434edf18 100644
--- a/internal/artifact/bk_uploader_test.go
+++ b/internal/artifact/bk_uploader_test.go
@@ -276,7 +276,8 @@ func TestFormUploadFileMissing(t *testing.T) {
 				Method:    "POST",
 				Path:      "buildkiteartifacts.com",
 				FileInput: "file",
-			}},
+			},
+		},
 	}
 
 	work, err := uploader.CreateWork(artifact)
diff --git a/internal/artifact/gs_uploader.go b/internal/artifact/gs_uploader.go
index bea4616d6a..9d9487e042 100644
--- a/internal/artifact/gs_uploader.go
+++ b/internal/artifact/gs_uploader.go
@@ -60,11 +60,11 @@ func NewGSUploader(ctx context.Context, l logger.Logger, c GSUploaderConfig) (*G
 	}, nil
 }
 
-func ParseGSDestination(destination string) (name string, path string) {
+func ParseGSDestination(destination string) (name, path string) {
 	parts := strings.Split(strings.TrimPrefix(string(destination), "gs://"), "/")
 	path = strings.Join(parts[1:], "/")
 	name = parts[0]
-	return
+	return name, path
 }
 
 func clientFromJSON(ctx context.Context, data []byte, scope string) (*http.Client, error) {
@@ -109,7 +109,7 @@ func (u *GSUploader) URL(artifact *api.Artifact) string {
 	// Also ensure that we always have exactly one / between prefix and artifactPath
 	path := path.Join(pathPrefix, u.artifactPath(artifact))
 
-	var artifactURL = &url.URL{
+	artifactURL := &url.URL{
 		Scheme: "https",
 		Host:   host,
 		Path:   path,
diff --git a/internal/artifact/s3_uploader.go b/internal/artifact/s3_uploader.go
index 34ebf78d4f..9fc8613ec2 100644
--- a/internal/artifact/s3_uploader.go
+++ b/internal/artifact/s3_uploader.go
@@ -54,7 +54,6 @@ func NewS3Uploader(ctx context.Context, l logger.Logger, c S3UploaderConfig) (*S
 		// Initialize the s3 client, and authenticate it
 		return NewS3Client(ctx, l, bucketName)
 	})
-
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/artifact/uploader.go b/internal/artifact/uploader.go
index 283c3fd063..484bdcf9f1 100644
--- a/internal/artifact/uploader.go
+++ b/internal/artifact/uploader.go
@@ -347,7 +347,7 @@ func (c *artifactCollector) worker(ctx context.Context, filesCh <-chan string) e
 	}
 }
 
-func (a *Uploader) build(path string, absolutePath string) (*api.Artifact, error) {
+func (a *Uploader) build(path, absolutePath string) (*api.Artifact, error) {
 	// Open the file to hash its contents.
 	file, err := os.Open(absolutePath)
 	if err != nil {
@@ -633,7 +633,6 @@ func (a *artifactUploadWorker) doWorkUnits(ctx context.Context, unitsCh <-chan w
 				}
 				return etag, err
 			})
-
 			// If it failed, abort any other work items for this artifact.
 			if err != nil {
 				a.logger.Info("Upload failed for %s: %v", workUnit.Description(), err)
diff --git a/internal/artifact/uploader_test.go b/internal/artifact/uploader_test.go
index e59e7de5f2..9c199830f6 100644
--- a/internal/artifact/uploader_test.go
+++ b/internal/artifact/uploader_test.go
@@ -34,7 +34,7 @@ func TestCollect(t *testing.T) {
 	volumeName := filepath.VolumeName(root)
 	rootWithoutVolume := strings.TrimPrefix(root, volumeName)
 
-	var testCases = []struct {
+	testCases := []struct {
 		Name         string
 		Path         []string
 		AbsolutePath string
diff --git a/internal/file/opened_by.go b/internal/file/opened_by.go
index f652395ac5..db987c0122 100644
--- a/internal/file/opened_by.go
+++ b/internal/file/opened_by.go
@@ -52,7 +52,7 @@ func OpenedBy(l shell.Logger, debug bool, path string) (string, error) {
 	return "", ErrFileNotOpen
 }
 
-func openedByPid(l shell.Logger, debug bool, absPath string, pid string) bool {
+func openedByPid(l shell.Logger, debug bool, absPath, pid string) bool {
 	dirEntries, err := os.ReadDir(fmt.Sprintf("/proc/%s/fd", pid))
 	if err != nil {
 		if debug {
diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index 3746a66b1a..1836c0b948 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -331,7 +331,7 @@ func hasGitSubmodules(sh *shell.Shell) bool {
 	return osutil.FileExists(filepath.Join(sh.Getwd(), ".gitmodules"))
 }
 
-func hasGitCommit(ctx context.Context, sh *shell.Shell, gitDir string, commit string) bool {
+func hasGitCommit(ctx context.Context, sh *shell.Shell, gitDir, commit string) bool {
 	// Resolve commit to an actual commit object
 	output, err := sh.Command("git", "--git-dir", gitDir, "rev-parse", commit+"^{commit}").RunAndCaptureStdout(ctx, shell.ShowStderr(false))
 	if err != nil {
diff --git a/internal/job/docker.go b/internal/job/docker.go
index e9b47a6284..c0b17c137c 100644
--- a/internal/job/docker.go
+++ b/internal/job/docker.go
@@ -24,7 +24,7 @@ func hasDeprecatedDockerIntegration(sh *shell.Shell) bool {
 }
 
 func runDeprecatedDockerIntegration(ctx context.Context, sh *shell.Shell, cmd []string) error {
-	var warnNotSet = func(k1, k2 string) {
+	warnNotSet := func(k1, k2 string) {
 		sh.Warningf("%s is set, but without %s, which it requires. You should be able to safely remove this from your pipeline.", k1, k2)
 	}
 
diff --git a/internal/job/executor.go b/internal/job/executor.go
index 1dc77f2d1e..f4a51ddd0e 100644
--- a/internal/job/executor.go
+++ b/internal/job/executor.go
@@ -565,7 +565,6 @@ func (e *Executor) runWrappedShellScriptHook(ctx context.Context, hookName strin
 		r.Break()
 		return err
 	})
-
 	if err != nil {
 		exitCode := shell.ExitCode(err)
 		e.shell.Env.Set("BUILDKITE_LAST_HOOK_EXIT_STATUS", strconv.Itoa(exitCode))
@@ -1068,7 +1067,7 @@ func (e *Executor) runPostCommandHooks(ctx context.Context) (err error) {
 }
 
 // CommandPhase determines how to run the build, and then runs it
-func (e *Executor) CommandPhase(ctx context.Context) (hookErr error, commandErr error) {
+func (e *Executor) CommandPhase(ctx context.Context) (hookErr, commandErr error) {
 	var preCommandErr error
 
 	span, ctx := tracetools.StartSpanFromContext(ctx, "command", e.TracingBackend)
diff --git a/internal/job/githttptest/githttptest.go b/internal/job/githttptest/githttptest.go
index e6f9ec900e..640a8e84dd 100644
--- a/internal/job/githttptest/githttptest.go
+++ b/internal/job/githttptest/githttptest.go
@@ -153,7 +153,6 @@ func (s *Server) PushBranch(repoName, branchName string) (string, []byte, error)
 	commitCmd = exec.Command("git", "commit", "-m", "Add new file")
 	commitCmd.Dir = tempDir
 	if out, err := commitCmd.CombinedOutput(); err != nil {
-
 		return "", out, fmt.Errorf("failed to commit new file: %w", err)
 	}
 
@@ -247,7 +246,6 @@ func (s *Server) handleGitUploadPack(w http.ResponseWriter, r *http.Request) {
 
 // handleGitReceivePack handles the git-receive-pack endpoint (used for git push)
 func (s *Server) handleGitReceivePack(w http.ResponseWriter, r *http.Request) {
-
 	repoName := r.PathValue("repository")
 
 	repoName = strings.TrimSuffix(repoName, ".git")
@@ -284,7 +282,6 @@ func (s *Server) handleGitReceivePack(w http.ResponseWriter, r *http.Request) {
 
 // handleGitInfoRefs handles the info/refs endpoint
 func (s *Server) handleGitInfoRefs(w http.ResponseWriter, r *http.Request) {
-
 	repoName := r.PathValue("repository")
 
 	repoName = strings.TrimSuffix(repoName, ".git")
diff --git a/internal/job/integration/checkout_git_mirrors_integration_test.go b/internal/job/integration/checkout_git_mirrors_integration_test.go
index 8808082466..2a00ad2d01 100644
--- a/internal/job/integration/checkout_git_mirrors_integration_test.go
+++ b/internal/job/integration/checkout_git_mirrors_integration_test.go
@@ -644,7 +644,7 @@ func TestRepositorylessCheckout_WithGitMirrors(t *testing.T) {
 		t.Fatalf("EnableGitMirrors() error = %v", err)
 	}
 
-	var script = []string{
+	script := []string{
 		"#!/usr/bin/env bash",
 		"export BUILDKITE_REPO=",
 	}
diff --git a/internal/job/integration/checkout_integration_test.go b/internal/job/integration/checkout_integration_test.go
index 3ee8ac5525..e1fd8f9791 100644
--- a/internal/job/integration/checkout_integration_test.go
+++ b/internal/job/integration/checkout_integration_test.go
@@ -972,7 +972,7 @@ func TestRepositorylessCheckout(t *testing.T) {
 	}
 	defer tester.Close()
 
-	var script = []string{
+	script := []string{
 		"#!/usr/bin/env bash",
 		"export BUILDKITE_REPO=",
 	}
diff --git a/internal/job/integration/executor_tester.go b/internal/job/integration/executor_tester.go
index 8ffa40c9a6..5f199006ee 100644
--- a/internal/job/integration/executor_tester.go
+++ b/internal/job/integration/executor_tester.go
@@ -212,7 +212,7 @@ func (e *ExecutorTester) MockAgent(t *testing.T) *bintest.Mock {
 }
 
 // writeHookScript generates a buildkite-agent hook script that calls a mock binary
-func (e *ExecutorTester) writeHookScript(m *bintest.Mock, name string, dir string, args ...string) (string, error) {
+func (e *ExecutorTester) writeHookScript(m *bintest.Mock, name, dir string, args ...string) (string, error) {
 	hookScript := filepath.Join(dir, name)
 	body := ""
 
diff --git a/internal/job/integration/test-binary-hook/main.go b/internal/job/integration/test-binary-hook/main.go
index 7eaf02b7b9..19581cffc9 100644
--- a/internal/job/integration/test-binary-hook/main.go
+++ b/internal/job/integration/test-binary-hook/main.go
@@ -22,7 +22,6 @@ func main() {
 			"MOUNTAIN": "chimborazo",
 		},
 	})
-
 	if err != nil {
 		log.Fatalf("error: %v", fmt.Errorf("updating env: %w", err))
 	}
diff --git a/internal/job/ssh.go b/internal/job/ssh.go
index 7262418111..5eea97b58f 100644
--- a/internal/job/ssh.go
+++ b/internal/job/ssh.go
@@ -13,9 +13,7 @@ import (
 	"github.com/buildkite/roko"
 )
 
-var (
-	sshKeyscanRetryInterval = 2 * time.Second
-)
+var sshKeyscanRetryInterval = 2 * time.Second
 
 func sshKeyScan(ctx context.Context, sh *shell.Shell, host string) (string, error) {
 	toolsDir, err := findPathToSSHTools(ctx, sh)
@@ -31,7 +29,6 @@ func sshKeyScan(ctx context.Context, sh *shell.Shell, host string) (string, erro
 		roko.WithStrategy(roko.Constant(sshKeyscanRetryInterval)),
 	)
 	return roko.DoFunc(ctx, r, func(r *roko.Retrier) (string, error) {
-
 		sshKeyScanCommand := fmt.Sprintf("ssh-keyscan %q", host)
 		args := []string{host}
 
diff --git a/internal/mime/generate.go b/internal/mime/generate.go
index fe1262bf32..8548f6957b 100644
--- a/internal/mime/generate.go
+++ b/internal/mime/generate.go
@@ -17,12 +17,14 @@ import (
 	"time"
 )
 
-var _, sourcePath, _, _ = runtime.Caller(0)
-var targetFile = path.Join(path.Dir(sourcePath), "mime.go")
-var urls = map[string]string{
-	"nginx":  "https://hg.nginx.org/nginx/raw-file/default/conf/mime.types",
-	"apache": "https://raw.githubusercontent.com/apache/httpd/trunk/docs/conf/mime.types",
-}
+var (
+	_, sourcePath, _, _ = runtime.Caller(0)
+	targetFile          = path.Join(path.Dir(sourcePath), "mime.go")
+	urls                = map[string]string{
+		"nginx":  "https://hg.nginx.org/nginx/raw-file/default/conf/mime.types",
+		"apache": "https://raw.githubusercontent.com/apache/httpd/trunk/docs/conf/mime.types",
+	}
+)
 
 var mimeFileTemplate = template.Must(template.New("").Parse(
 	`// Code generated by go generate; DO NOT EDIT.
diff --git a/internal/replacer/bm_redactor_test.go b/internal/replacer/bm_redactor_test.go
index f8e84ff76b..ea94fbbd96 100644
--- a/internal/replacer/bm_redactor_test.go
+++ b/internal/replacer/bm_redactor_test.go
@@ -8,7 +8,6 @@ import (
 )
 
 func BenchmarkBMRedactor(b *testing.B) {
-
 	r := NewBMRedactor(io.Discard, "[REDACTED]", bigLipsumSecrets)
 	for b.Loop() {
 		if _, err := fmt.Fprintln(r, bigLipsum); err != nil {
@@ -116,7 +115,6 @@ func (redactor *BoyerMooreRedactor) Reset(needles []string) {
 			}
 		}
 	}
-
 }
 
 func (redactor *BoyerMooreRedactor) Write(input []byte) (int, error) {
diff --git a/internal/replacer/replacer_test.go b/internal/replacer/replacer_test.go
index 2f9617ab29..720ba7a306 100644
--- a/internal/replacer/replacer_test.go
+++ b/internal/replacer/replacer_test.go
@@ -201,7 +201,7 @@ func TestReplacerResetMidStream(t *testing.T) {
 	}
 
 	// update the replacer with a new secret
-	//replacer.Flush() // manual flush is NOT necessary before Reset
+	// replacer.Flush() // manual flush is NOT necessary before Reset
 	replacer.Reset([]string{"secret1111", "secret2222"})
 
 	// finish writing
@@ -354,7 +354,6 @@ func TestAddingNeedles(t *testing.T) {
 }
 
 func BenchmarkReplacer(b *testing.B) {
-
 	r := replacer.New(io.Discard, bigLipsumSecrets, redact.Redacted)
 	for b.Loop() {
 		if _, err := fmt.Fprintln(r, bigLipsum); err != nil {
diff --git a/internal/shell/logger.go b/internal/shell/logger.go
index adcd40fd78..f5ab57df72 100644
--- a/internal/shell/logger.go
+++ b/internal/shell/logger.go
@@ -191,11 +191,11 @@ func (l *LoggerStreamer) Write(p []byte) (n int, err error) {
 	}
 
 	if n, err = l.buf.Write(p); err != nil {
-		return
+		return n, err
 	}
 
 	err = l.Output()
-	return
+	return n, err
 }
 
 func (l *LoggerStreamer) Close() error {
diff --git a/internal/shell/logger_test.go b/internal/shell/logger_test.go
index fadbd07f13..dd8ea7ab3e 100644
--- a/internal/shell/logger_test.go
+++ b/internal/shell/logger_test.go
@@ -107,7 +107,6 @@ func TestLoggerStreamer(t *testing.T) {
 
 	//nolint:errcheck // Writes to bytes.Buffer never error.
 	func() {
-
 		fmt.Fprintln(want, "TEST># Rest of the line")
 		fmt.Fprintln(want, "TEST># And another")
 		fmt.Fprintln(want, "TEST># No line end")
diff --git a/internal/shell/lookpath.go b/internal/shell/lookpath.go
index a651522cb3..93b3a3ce2a 100644
--- a/internal/shell/lookpath.go
+++ b/internal/shell/lookpath.go
@@ -32,7 +32,7 @@ func findExecutable(file string) error {
 // LookPath searches for an executable binary named file in the directories within the path variable,
 // which is a colon delimited path.
 // If file contains a slash, it is tried directly
-func LookPath(file string, path string, fileExtensions string) (string, error) {
+func LookPath(file, path, fileExtensions string) (string, error) {
 	if strings.Contains(file, "/") {
 		err := findExecutable(file)
 		if err == nil {
diff --git a/internal/shell/lookpath_windows.go b/internal/shell/lookpath_windows.go
index eb2b4db66a..d5087cccaf 100644
--- a/internal/shell/lookpath_windows.go
+++ b/internal/shell/lookpath_windows.go
@@ -56,7 +56,7 @@ func findExecutable(file string, exts []string) (string, error) {
 // If file contains a slash, it is tried directly
 // LookPath also uses PATHEXT environment variable to match a suitable candidate.
 // The result may be an absolute path or a path relative to the current directory.
-func LookPath(file string, path string, fileExtensions string) (string, error) {
+func LookPath(file, path, fileExtensions string) (string, error) {
 	var exts []string
 	if fileExtensions != "" {
 		for _, e := range strings.Split(strings.ToLower(fileExtensions), ";") {
diff --git a/internal/system/version_dump_windows.go b/internal/system/version_dump_windows.go
index 2140226229..39ba050ea6 100644
--- a/internal/system/version_dump_windows.go
+++ b/internal/system/version_dump_windows.go
@@ -4,6 +4,7 @@ package system
 
 import (
 	"fmt"
+
 	"github.com/buildkite/agent/v3/logger"
 	"golang.org/x/sys/windows"
 )
diff --git a/jobapi/server_test.go b/jobapi/server_test.go
index 5fdf106373..232c862a70 100644
--- a/jobapi/server_test.go
+++ b/jobapi/server_test.go
@@ -343,7 +343,6 @@ func TestPatchEnv(t *testing.T) {
 			testAPI(t, environ, req, client, c)
 		})
 	}
-
 }
 
 func TestGetEnv(t *testing.T) {
diff --git a/logger/buffer.go b/logger/buffer.go
index 84480b7517..e120aacfc4 100644
--- a/logger/buffer.go
+++ b/logger/buffer.go
@@ -26,31 +26,37 @@ func (b *Buffer) Debug(format string, v ...any) {
 	defer b.mu.Unlock()
 	b.Messages = append(b.Messages, "[debug] "+fmt.Sprintf(format, v...))
 }
+
 func (b *Buffer) Error(format string, v ...any) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	b.Messages = append(b.Messages, "[error] "+fmt.Sprintf(format, v...))
 }
+
 func (b *Buffer) Fatal(format string, v ...any) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	b.Messages = append(b.Messages, "[fatal] "+fmt.Sprintf(format, v...))
 }
+
 func (b *Buffer) Notice(format string, v ...any) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	b.Messages = append(b.Messages, "[notice] "+fmt.Sprintf(format, v...))
 }
+
 func (b *Buffer) Warn(format string, v ...any) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	b.Messages = append(b.Messages, "[warn] "+fmt.Sprintf(format, v...))
 }
+
 func (b *Buffer) Info(format string, v ...any) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	b.Messages = append(b.Messages, "[info] "+fmt.Sprintf(format, v...))
 }
+
 func (b *Buffer) WithFields(fields ...Field) Logger {
 	return b
 }
diff --git a/main.go b/main.go
index bbc363112a..a4e99f774b 100644
--- a/main.go
+++ b/main.go
@@ -4,7 +4,7 @@ package main
 
 // see https://blog.golang.org/generate
 //go:generate go run internal/mime/generate.go
-//go:generate go fmt internal/mime/mime.go
+//go:generate go tool gofumpt -w internal/mime/mime.go
 
 import (
 	"fmt"
diff --git a/process/format.go b/process/format.go
index 96e66e1c53..d05c19f46c 100644
--- a/process/format.go
+++ b/process/format.go
@@ -8,7 +8,7 @@ import (
 
 // FormatCommand formats a command amd arguments for human reading
 func FormatCommand(command string, args []string) string {
-	var truncate = func(s string, i int) string {
+	truncate := func(s string, i int) string {
 		if len(s) < i {
 			return s
 		}
diff --git a/process/run.go b/process/run.go
index 32663daca2..72cbfc75c3 100644
--- a/process/run.go
+++ b/process/run.go
@@ -9,7 +9,6 @@ import (
 
 func Run(l logger.Logger, command string, arg ...string) (string, error) {
 	output, err := exec.Command(command, arg...).Output()
-
 	if err != nil {
 		l.Debug("Could not run: %s %s (returned %s) (%T: %v)", command, arg, output, err, err)
 		return "", err
diff --git a/process/scanner.go b/process/scanner.go
index 6c1a0d6125..090dd8cf25 100644
--- a/process/scanner.go
+++ b/process/scanner.go
@@ -18,7 +18,7 @@ func NewScanner(l logger.Logger) *Scanner {
 }
 
 func (s *Scanner) ScanLines(r io.Reader, f func(line string)) error {
-	var reader = bufio.NewReader(r)
+	reader := bufio.NewReader(r)
 	var appending []byte
 
 	s.logger.Debug("[LineScanner] Starting to read lines")
diff --git a/tracetools/propagate.go b/tracetools/propagate.go
index 70ec2f8036..d901da896c 100644
--- a/tracetools/propagate.go
+++ b/tracetools/propagate.go
@@ -57,8 +57,10 @@ func DecodeTraceContext(env map[string]string, codec Codec) (opentracing.SpanCon
 }
 
 // Encoder impls can encode values. Decoder impls can decode values.
-type Encoder interface{ Encode(v any) error }
-type Decoder interface{ Decode(v any) error }
+type (
+	Encoder interface{ Encode(v any) error }
+	Decoder interface{ Decode(v any) error }
+)
 
 // Codec implementations produce encoders/decoders.
 type Codec interface {
diff --git a/tracetools/span.go b/tracetools/span.go
index f966a55950..8c7e80fd0f 100644
--- a/tracetools/span.go
+++ b/tracetools/span.go
@@ -27,7 +27,7 @@ var ValidTracingBackends = map[string]struct{}{
 // StartSpanFromContext will start a span appropriate to the given tracing backend from the given context with the given
 // operation name. It will also do some common/repeated setup on the span to keep code a little more DRY.
 // If an unknown tracing backend is specified, it will return a span that noops on every operation
-func StartSpanFromContext(ctx context.Context, operation string, tracingBackend string) (Span, context.Context) {
+func StartSpanFromContext(ctx context.Context, operation, tracingBackend string) (Span, context.Context) {
 	switch tracingBackend {
 	case BackendDatadog:
 		span, ctx := opentracing.StartSpanFromContext(ctx, operation)

From 9f4cc34370fd6de0a4c7be48f02624fd0eb0df38 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 20 Nov 2025 12:28:53 +1100
Subject: [PATCH 077/242] Remove the APIClient interface and just call
 `*api.Client` directly

We were never using this for any of the useful things an interface provides
---
 agent/agent_worker.go      |  4 +--
 agent/api.go               | 52 --------------------------------------
 agent/job_runner.go        |  4 +--
 agent/pipeline_uploader.go |  2 +-
 api/client.go              |  2 --
 5 files changed, 5 insertions(+), 59 deletions(-)
 delete mode 100644 agent/api.go

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 8ceb3287f2..556e15f6cf 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -59,7 +59,7 @@ type AgentWorker struct {
 	stats agentStats
 
 	// The API Client used when this agent is communicating with the API
-	apiClient APIClient
+	apiClient *api.Client
 
 	// The core Client is used to drive some APIClient methods
 	client *core.Client
@@ -173,7 +173,7 @@ func (e *errUnrecoverable) Unwrap() error {
 }
 
 // Creates the agent worker and initializes its API Client
-func NewAgentWorker(l logger.Logger, reg *api.AgentRegisterResponse, m *metrics.Collector, apiClient APIClient, c AgentWorkerConfig) *AgentWorker {
+func NewAgentWorker(l logger.Logger, reg *api.AgentRegisterResponse, m *metrics.Collector, apiClient *api.Client, c AgentWorkerConfig) *AgentWorker {
 	apiClient = apiClient.FromAgentRegisterResponse(reg)
 	return &AgentWorker{
 		logger:           l,
diff --git a/agent/api.go b/agent/api.go
deleted file mode 100644
index 6065193ae4..0000000000
--- a/agent/api.go
+++ /dev/null
@@ -1,52 +0,0 @@
-// Code generated by interfacer; DO NOT EDIT
-
-package agent
-
-import (
-	"context"
-	"net/http"
-
-	"github.com/buildkite/agent/v3/api"
-)
-
-// APIClient is an interface generated for "github.com/buildkite/agent/v3/api.Client".
-type APIClient interface {
-	AcceptJob(context.Context, *api.Job) (*api.Job, *api.Response, error)
-	AcquireJob(context.Context, string, ...api.Header) (*api.Job, *api.Response, error)
-	Annotate(context.Context, string, *api.Annotation) (*api.Response, error)
-	AnnotationRemove(context.Context, string, string, string) (*api.Response, error)
-	CancelBuild(context.Context, string) (*api.Build, *api.Response, error)
-	Config() api.Config
-	Connect(context.Context) (*api.Response, error)
-	CreateArtifacts(context.Context, string, *api.ArtifactBatch) (*api.ArtifactBatchCreateResponse, *api.Response, error)
-	Disconnect(context.Context) (*api.Response, error)
-	ExistsMetaData(context.Context, string, string, string) (*api.MetaDataExists, *api.Response, error)
-	FinishJob(context.Context, *api.Job, *bool) (*api.Response, error)
-	FromAgentRegisterResponse(*api.AgentRegisterResponse) *api.Client
-	FromPing(*api.Ping) *api.Client
-	GenerateGithubCodeAccessToken(context.Context, string, string) (string, *api.Response, error)
-	GetJobState(context.Context, string) (*api.JobState, *api.Response, error)
-	GetMetaData(context.Context, string, string, string) (*api.MetaData, *api.Response, error)
-	GetSecret(context.Context, *api.GetSecretRequest) (*api.Secret, *api.Response, error)
-	Heartbeat(context.Context) (*api.Heartbeat, *api.Response, error)
-	MetaDataKeys(context.Context, string, string) ([]string, *api.Response, error)
-	New(api.Config) *api.Client
-	OIDCToken(context.Context, *api.OIDCTokenRequest) (*api.OIDCToken, *api.Response, error)
-	Pause(context.Context, *api.AgentPauseRequest) (*api.Response, error)
-	Ping(context.Context) (*api.Ping, *api.Response, error)
-	PipelineUploadStatus(context.Context, string, string, ...api.Header) (*api.PipelineUploadStatus, *api.Response, error)
-	Register(context.Context, *api.AgentRegisterRequest) (*api.AgentRegisterResponse, *api.Response, error)
-	Resume(context.Context) (*api.Response, error)
-	SaveHeaderTimes(context.Context, string, *api.HeaderTimes) (*api.Response, error)
-	SearchArtifacts(context.Context, string, *api.ArtifactSearchOptions) ([]*api.Artifact, *api.Response, error)
-	ServerSpecifiedRequestHeaders() http.Header
-	SetMetaData(context.Context, string, *api.MetaData) (*api.Response, error)
-	StartJob(context.Context, *api.Job) (*api.Response, error)
-	StepCancel(context.Context, string, *api.StepCancel) (*api.StepCancelResponse, *api.Response, error)
-	StepExport(context.Context, string, *api.StepExportRequest) (*api.StepExportResponse, *api.Response, error)
-	StepUpdate(context.Context, string, *api.StepUpdate) (*api.Response, error)
-	Stop(context.Context, *api.AgentStopRequest) (*api.Response, error)
-	UpdateArtifacts(context.Context, string, []api.ArtifactState) (*api.Response, error)
-	UploadChunk(context.Context, string, *api.Chunk) (*api.Response, error)
-	UploadPipeline(context.Context, string, *api.PipelineChange, ...api.Header) (*api.Response, error)
-}
diff --git a/agent/job_runner.go b/agent/job_runner.go
index 62e6412609..829988e813 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -96,7 +96,7 @@ type JobRunner struct {
 	agentLogger logger.Logger
 
 	// The APIClient that will be used when updating the job
-	apiClient APIClient
+	apiClient *api.Client
 
 	// The agentlib Client is used to drive some APIClient methods
 	client *core.Client
@@ -142,7 +142,7 @@ type jobProcess interface {
 }
 
 // Initializes the job runner
-func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, conf JobRunnerConfig) (*JobRunner, error) {
+func NewJobRunner(ctx context.Context, l logger.Logger, apiClient *api.Client, conf JobRunnerConfig) (*JobRunner, error) {
 	// If the accept response has a token attached, we should use that instead of the Agent Access Token that
 	// our current apiClient is using
 	if conf.Job.Token != "" {
diff --git a/agent/pipeline_uploader.go b/agent/pipeline_uploader.go
index 786ff2284b..0f925a38c4 100644
--- a/agent/pipeline_uploader.go
+++ b/agent/pipeline_uploader.go
@@ -26,7 +26,7 @@ var locationRegex = regexp.MustCompile(`jobs/(?P<jobID>[^/]+)/pipelines/(?P<uplo
 
 // PipelineUploader contains the data needed to upload a pipeline to Buildkite
 type PipelineUploader struct {
-	Client         APIClient
+	Client         *api.Client
 	Change         *api.PipelineChange
 	JobID          string
 	RetrySleepFunc func(time.Duration)
diff --git a/api/client.go b/api/client.go
index 1dc70f6512..8891f57be8 100644
--- a/api/client.go
+++ b/api/client.go
@@ -1,7 +1,5 @@
 package api
 
-//go:generate go run github.com/rjeczalik/interfaces/cmd/interfacer@v0.3.0 -for github.com/buildkite/agent/v3/api.Client -as agent.APIClient -o ../agent/api.go
-
 import (
 	"bytes"
 	"context"

From 1b188d30ef1e1c7f0b0f3e0dddf448d12a5ddffb Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 20 Nov 2025 12:31:16 +1100
Subject: [PATCH 078/242] Update suggested format commands in docs

---
 .github/pull_request_template.md | 6 +++---
 AGENT.md                         | 4 ++--
 CONTRIBUTING.md                  | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 2d264ab65a..cdc507a72a 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -21,7 +21,7 @@ Can skip if changes are simple or clear from the commit messages.
 
 ### Testing
 - [ ] Tests have run locally (with `go test ./...`). Buildkite employees may check this if the pipeline has run automatically.
-- [ ] Code is formatted (with `go fmt ./...`)
+- [ ] Code is formatted (with `go tool gofumpt -extra -w .`)
 
 <!--
 Note: if the tests fail to run locally, please let us know!
@@ -34,10 +34,10 @@ Note: if the tests fail to run locally, please let us know!
 If you used AI in any way to produce this PR (beyond typo fixes or small amounts of tab-autocompletion), please describe the extent of the contribution here, and the tools used.
 Feel free to claim credit for work _not_ done by an AI here too, or to give credit to others who helped in any meaningful way.
 
-Examples: 
+Examples:
  - "Claude Code wrote the unit tests, then I implemented the rest of the change"
  - "I consulted ChatGPT on potential approaches, then wrote the implementation myself"
  - "I used Gemini to write the code and Midjourney to produce the diagrams"
  - "Special thanks to the Wikipedia page on ANSI escape codes"
  - "I did not use AI tools at all"
--->
\ No newline at end of file
+-->
diff --git a/AGENT.md b/AGENT.md
index 0be11bf101..b24bc6c5a4 100644
--- a/AGENT.md
+++ b/AGENT.md
@@ -6,7 +6,7 @@
 - **Test:** `go test ./...` (run all tests)
 - **Test (single package):** `go test ./path/to/package`
 - **Test (race detection):** `go test -race ./...`
-- **Lint/Format:** `go fmt ./...` and `golangci-lint run`
+- **Lint/Format:** `go tool gofumpt -extra -w .` and `golangci-lint run`
 - **Generate:** `go generate ./...`
 - **Deps:** `go mod tidy`
 
@@ -25,7 +25,7 @@ Go CLI application with main packages:
 
 ## Code Style
 
-- Standard Go formatting with `go fmt`
+- Formatting with `gofumpt` in extra mode: `go tool gofumpt -extra -w .`
 - Struct-based configuration patterns (e.g., `AgentWorkerConfig`, `JobRunnerConfig`)
 - Context-aware functions: `func Name(ctx context.Context, ...)`
 - Import organization: stdlib, external deps, internal packages
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7837ccbbd0..b6f022b461 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -3,7 +3,7 @@
 1. Fork this repo
 1. Create a feature branch with a nice name (`git checkout -b my-new-feature`)
 1. Write your code!
-    - Make sure your code is correctly formatted by running `go fmt ./...`, and that the tests are passing by running `go test ./...`
+    - Make sure your code is correctly formatted by running `go tool gofumpt -extra -w .`, and that the tests are passing by running `go test ./...`
 1. Commit your changes (`git commit -am 'Add some feature'`)
     - In an ideal world we have [atomic commits](https://www.pauline-vos.nl/atomic-commits/) and use [Tim Pope-style commit messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html), but so long as it's clear what's happening in your PR, that's fine. We try to not be super persnickety about these things.
 1. Push to your branch (`git push origin my-new-feature`)

From 977054b22e6959e7c9319f44c7fdd479b4404c4e Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 20 Nov 2025 12:36:58 +1100
Subject: [PATCH 079/242] Add .vscode setting to use gofumpt as the default
 formatter

---
 .gitignore            | 1 -
 .vscode/settings.json | 5 +++++
 2 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 .vscode/settings.json

diff --git a/.gitignore b/.gitignore
index 3a567bc7a7..624ab42167 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,7 +23,6 @@ packaging/docker/*/hooks/
 .buildkite-agent.cfg
 
 .idea
-.vscode
 
 # Editor config boils down to personal preference
 .editorconfig
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000000..219c87aea9
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,5 @@
+{
+  "gopls": {
+    "formatting.gofumpt": true,
+  },
+}

From d2ccc7f6590f4be80861a9bbcba1243f311ecd7e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 20 Nov 2025 04:02:34 +0000
Subject: [PATCH 080/242] build(deps): bump golang.org/x/crypto from 0.44.0 to
 0.45.0

Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.44.0 to 0.45.0.
- [Commits](https://github.com/golang/crypto/compare/v0.44.0...v0.45.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.45.0
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index c97d07c469..173c682d12 100644
--- a/go.mod
+++ b/go.mod
@@ -55,7 +55,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
 	go.opentelemetry.io/otel/sdk v1.38.0
 	go.opentelemetry.io/otel/trace v1.38.0
-	golang.org/x/crypto v0.44.0
+	golang.org/x/crypto v0.45.0
 	golang.org/x/net v0.47.0
 	golang.org/x/oauth2 v0.33.0
 	golang.org/x/sync v0.18.0
diff --git a/go.sum b/go.sum
index 1dfd7e2f27..56cbba7f6b 100644
--- a/go.sum
+++ b/go.sum
@@ -483,8 +483,8 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
-golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

From fa65dd44e29bff1a3a5e144225727a654cd2dc3f Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 20 Nov 2025 10:19:07 +1100
Subject: [PATCH 081/242] internal/redact: Add another test with minor cleanup

---
 internal/redact/redact_test.go | 95 ++++++++++++++++++++--------------
 1 file changed, 55 insertions(+), 40 deletions(-)

diff --git a/internal/redact/redact_test.go b/internal/redact/redact_test.go
index d205a70dd1..8d035fc636 100644
--- a/internal/redact/redact_test.go
+++ b/internal/redact/redact_test.go
@@ -10,49 +10,64 @@ import (
 func TestVars(t *testing.T) {
 	t.Parallel()
 
-	redactConfig := []string{
-		"*_PASSWORD",
-		"*_TOKEN",
-	}
-	environment := []env.Pair{
-		{Name: "BUILDKITE_PIPELINE", Value: "unit-test"},
-		// These are example values, and are not leaked credentials
-		{Name: "DATABASE_USERNAME", Value: "AzureDiamond"},
-		{Name: "DATABASE_PASSWORD", Value: "hunter2"},
-	}
-
-	got, short, err := Vars(redactConfig, environment)
-	if err != nil {
-		t.Errorf("Vars(%q, %q) error = %v", redactConfig, environment, err)
-	}
-	if len(short) > 0 {
-		t.Errorf("Vars(%q, %q) short = %q", redactConfig, environment, short)
+	tests := []struct {
+		name         string
+		redactConfig []string
+		environment  []env.Pair
+		wantMatched  []env.Pair
+		wantShort    []string
+	}{
+		{
+			name:         "hunter2",
+			redactConfig: []string{"*_PASSWORD", "*_TOKEN"},
+			environment: []env.Pair{
+				{Name: "BUILDKITE_PIPELINE", Value: "unit-test"},
+				// These are example values, and are not leaked credentials
+				{Name: "DATABASE_USERNAME", Value: "AzureDiamond"},
+				{Name: "DATABASE_PASSWORD", Value: "hunter2"},
+			},
+			wantMatched: []env.Pair{{Name: "DATABASE_PASSWORD", Value: "hunter2"}},
+			wantShort:   nil,
+		},
+		{
+			name:         "short",
+			redactConfig: []string{"*_PASSWORD", "*_TOKEN"},
+			environment: []env.Pair{
+				{Name: "BUILDKITE_PIPELINE", Value: "unit-test"},
+				// These are example values, and are not leaked credentials
+				{Name: "DATABASE_USERNAME", Value: "AzureDiamond"},
+				{Name: "DATABASE_PASSWORD", Value: "hunt"},
+			},
+			wantMatched: nil,
+			wantShort:   []string{"DATABASE_PASSWORD"},
+		},
+		{
+			name:         "empty",
+			redactConfig: nil,
+			environment: []env.Pair{
+				{Name: "FOO", Value: "BAR"},
+				{Name: "BUILDKITE_PIPELINE", Value: "unit-test"},
+			},
+			wantMatched: nil,
+			wantShort:   nil,
+		},
 	}
-	want := []env.Pair{{Name: "DATABASE_PASSWORD", Value: "hunter2"}}
 
-	if diff := cmp.Diff(got, want); diff != "" {
-		t.Errorf("Vars(%q, %q) diff (-got +want)\n%s", redactConfig, environment, diff)
-	}
-}
-
-func TestValuesToRedactEmpty(t *testing.T) {
-	t.Parallel()
-
-	redactConfig := []string{}
-	environment := []env.Pair{
-		{Name: "FOO", Value: "BAR"},
-		{Name: "BUILDKITE_PIPELINE", Value: "unit-test"},
-	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
 
-	got, short, err := Vars(redactConfig, environment)
-	if err != nil {
-		t.Errorf("Vars(%q, %q) error = %v", redactConfig, environment, err)
-	}
-	if len(short) > 0 {
-		t.Errorf("Vars(%q, %q) short = %q", redactConfig, environment, short)
-	}
-	if len(got) != 0 {
-		t.Errorf("Vars(%q, %q) = %q, want empty slice", redactConfig, environment, got)
+			matched, short, err := Vars(test.redactConfig, test.environment)
+			if err != nil {
+				t.Fatalf("Vars(%q, %q) error = %v", test.redactConfig, test.environment, err)
+			}
+			if diff := cmp.Diff(matched, test.wantMatched); diff != "" {
+				t.Errorf("Vars(%q, %q) matched diff (-got +want)\n%s", test.redactConfig, test.environment, diff)
+			}
+			if diff := cmp.Diff(short, test.wantShort); diff != "" {
+				t.Errorf("Vars(%q, %q) short diff (-got +want)\n%s", test.redactConfig, test.environment, diff)
+			}
+		})
 	}
 }
 

From a1782504f27741a645f9d51ee9998c20771c96bf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 24 Nov 2025 15:15:51 +0000
Subject: [PATCH 082/242] build(deps): bump the cloud-providers group with 7
 updates

Bumps the cloud-providers group with 7 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.39.6` | `1.40.0` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.20` | `1.32.1` |
| [github.com/aws/aws-sdk-go-v2/feature/ec2/imds](https://github.com/aws/aws-sdk-go-v2) | `1.18.13` | `1.18.14` |
| [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.20.7` | `1.20.11` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.269.0` | `1.274.0` |
| [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) | `1.48.2` | `1.49.0` |
| [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.90.2` | `1.92.0` |


Updates `github.com/aws/aws-sdk-go-v2` from 1.39.6 to 1.40.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.39.6...v1.40.0)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.20 to 1.32.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.20...v1.32.1)

Updates `github.com/aws/aws-sdk-go-v2/feature/ec2/imds` from 1.18.13 to 1.18.14
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.14/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.13...config/v1.18.14)

Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.20.7 to 1.20.11
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/mq/v1.20.7...service/sns/v1.20.11)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.269.0 to 1.274.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.269.0...service/ec2/v1.274.0)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.48.2 to 1.49.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/kms/v1.48.2...service/s3/v1.49.0)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.90.2 to 1.92.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.90.2...service/s3/v1.92.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.32.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/ec2/imds
  dependency-version: 1.18.14
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-version: 1.20.11
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.274.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.49.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-version: 1.92.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 35 +++++++++++++++--------------
 go.sum | 70 ++++++++++++++++++++++++++++++----------------------------
 2 files changed, 54 insertions(+), 51 deletions(-)

diff --git a/go.mod b/go.mod
index 748dd50790..a25726023b 100644
--- a/go.mod
+++ b/go.mod
@@ -11,13 +11,13 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
-	github.com/aws/aws-sdk-go-v2 v1.39.6
-	github.com/aws/aws-sdk-go-v2/config v1.31.20
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.269.0
-	github.com/aws/aws-sdk-go-v2/service/kms v1.48.2
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2
+	github.com/aws/aws-sdk-go-v2 v1.40.0
+	github.com/aws/aws-sdk-go-v2/config v1.32.1
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.274.0
+	github.com/aws/aws-sdk-go-v2/service/kms v1.49.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0
 	github.com/aws/smithy-go v1.23.2
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
@@ -95,18 +95,19 @@ require (
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.24 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
diff --git a/go.sum b/go.sum
index 399b3e6032..67a03fc863 100644
--- a/go.sum
+++ b/go.sum
@@ -76,46 +76,48 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk=
-github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
+github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
+github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
-github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc=
-github.com/aws/aws-sdk-go-v2/config v1.31.20/go.mod h1:95Hh1Tc5VYKL9NJ7tAkDcqeKt+MCXQB1hQZaRdJIZE0=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.24 h1:iJ2FmPT35EaIB0+kMa6TnQ+PwG5A1prEdAw+PsMzfHg=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.24/go.mod h1:U91+DrfjAiXPDEGYhh/x29o4p0qHX5HDqG7y5VViv64=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7 h1:u8danF+A2Zv//pFZvj5V23v/6XG4AxuSVup5s6nxSnI=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7/go.mod h1:uvLIvU8iJPEU5so7b6lLDNArWpOX6sRBfL5wBABmlfc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M=
+github.com/aws/aws-sdk-go-v2/config v1.32.1 h1:iODUDLgk3q8/flEC7ymhmxjfoAnBDwEEYEVyKZ9mzjU=
+github.com/aws/aws-sdk-go-v2/config v1.32.1/go.mod h1:xoAgo17AGrPpJBSLg81W+ikM0cpOZG8ad04T2r+d5P0=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.1 h1:JeW+EwmtTE0yXFK8SmklrFh/cGTTXsQJumgMZNlbxfM=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.1/go.mod h1:BOoXiStwTF+fT2XufhO0Efssbi1CNIO/ZXpZu87N0pw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11 h1:NMchKj9gGzIJH4yln7g+Ci4BeVSCayE8CQ7cc+xH9FM=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11/go.mod h1:eTZ6Kj2kFJ7UkKEWjlRPYI3fKcH+jKnsSaIom2XABBQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.269.0 h1:JMU0UqLvPUKovF/kXcIQf1ZVyv3+BwVW5O3bZrXDqBo=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.269.0/go.mod h1:NDdDLLW5PtLLXN661gKcvJvqAH5OBXsfhMlmKVu1/pY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.274.0 h1:Q2+WD4KSVRkd27QxD9I30nM3O7B4WYwE+ua5dm2NJY0=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.274.0/go.mod h1:QrV+/GjhSrJh6MRRuTO6ZEg4M2I0nwPakf0lZHSrE1o=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
-github.com/aws/aws-sdk-go-v2/service/kms v1.48.2 h1:aL8Y/AbB6I+uw0MjLbdo68NQ8t5lNs3CY3S848HpETk=
-github.com/aws/aws-sdk-go-v2/service/kms v1.48.2/go.mod h1:VJcNH6BLr+3VJwinRKdotLOMglHO8mIKlD3ea5c7hbw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 h1:DhdbtDl4FdNlj31+xiRXANxEE+eC7n8JQz+/ilwQ8Uc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 h1:NjShtS1t8r5LUfFVtFeI8xLAHQNTa7UI0VawXlrBMFQ=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.3/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 h1:gTsnx0xXNQ6SBbymoDvcoRHL+q4l/dAFsQuKfDWSaGc=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo=
-github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 h1:HK5ON3KmQV2HcAunnx4sKLB9aPf3gKGwVAf7xnx0QT0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.40.2/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.0 h1:QbztCKLI2qnjFZ/QYq3hZ8SW7SnTwB5h0NjREtKXIGo=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.0/go.mod h1:NZo9WJqQ0sxQ1Yqu1IwCHQFQunTms2MlVgejg16S1rY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0 h1:8FshVvnV2sr9kOSAbOnc/vwVmmAwMjOedKH6JW2ddPM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 h1:BDgIUYGEo5TkayOWv/oBLPphWwNm/A91AebUjAu5L5g=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.1/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 h1:U//SlnkE1wOQiIImxzdY5PXat4Wq+8rlfVEw4Y7J8as=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.4/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 h1:LU8S9W/mPDAU9q0FjCLi0TrCheLMGwzbRpvUMwYspcA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 h1:GdGmKtG+/Krag7VfyOXV17xjTCz0i9NT+JnqLTOI5nA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.1/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso=
 github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
 github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

From ba1125c8efe9e07238fef2aff3759c6b738bf7e9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 24 Nov 2025 17:25:36 +0000
Subject: [PATCH 083/242] build(deps): bump the container-images group across 5
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `d3529e9` to `ed5d56b`

Updates `buildkite/agent-base` from `0d16202` to `dff2305`

Updates `buildkite/agent-base` from `767ff4a` to `dfc9dfd`

Updates `buildkite/agent-base` from `b600458` to `25b5e99`

Updates `buildkite/agent-base` from `e33daff` to `37d39a8`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 1f5dd93e18..a246ebfeef 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:0d1620227851573d0d9ae37132d58d6c56d7264f9ec933615683f6cdea39b79f
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:dff2305d45920beebe5203ac6c9bfb91d2d2ad0fd15f2a18f3822705593313fd
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 2090cc8dd7..d0737c5628 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:d3529e9b56cce5f6d846efdee15fda4053ec7048b67c7c354ac8fc806e5cce0d
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:ed5d56b89ecb72b0c847a45bb4214d3a8c4f72f890c2fd9d9331de20c64cdca8
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 9127c8d07d..5a8e9f1241 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:767ff4a1d4d1a8b94aad0646ce8c6498ed5f312019b7ec897561f318b138326f
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:dfc9dfd401b17e513644e2b1bff702e0e5d4dd482bd93480622a09f9c13b521d
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index f1fa93dabf..ba1cb6ffa8 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:b6004588a34e4e9ce919a30ad627826d39835dc656b3855bb717ed7545e17bf7
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:25b5e99e6480ec51bf3c83232847a862bd64b06fbfce2047e57c0b4a008c97a3
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index 9899b416ae..8948f77e97 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:e33daff4bca2e47ef4d6018c857ed479903cc5d986a4face3c6a322657dd9005
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:37d39a8932facca2c9dcfb3da0c7f4c93bb0e3c974de2b3a244de11662cf54f8
 ARG TARGETOS
 ARG TARGETARCH
 

From e7c74f7ab38d8b2a82951d26542a6abd777c018b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 24 Nov 2025 17:36:26 +0000
Subject: [PATCH 084/242] build(deps): bump the container-images group across 1
 directory with 2 updates

Bumps the container-images group with 2 updates in the /.buildkite directory: docker/library/golang and golangci/golangci-lint.


Updates `docker/library/golang` from `c3ea417` to `7b13449`

Updates `golangci/golangci-lint` from `a7da515` to `a3abd55`

---
updated-dependencies:
- dependency-name: docker/library/golang
  dependency-version: 1.24.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: container-images
- dependency-name: golangci/golangci-lint
  dependency-version: v2.6-alpine
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-compile | 2 +-
 .buildkite/Dockerfile-lint    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index 268182ec83..b72c8c3f27 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.10@sha256:c3ea4172c1dd39e1c90bb36a11ef95af6d0ccbb1d7cdedbb5dd14988c324d689
+FROM public.ecr.aws/docker/library/golang:1.24.10@sha256:7b13449f08287fdb53114d65bdf20eb3965e4e54997903b5cb9477df0ea37c12
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest
diff --git a/.buildkite/Dockerfile-lint b/.buildkite/Dockerfile-lint
index cc0ea8a671..799244b58b 100644
--- a/.buildkite/Dockerfile-lint
+++ b/.buildkite/Dockerfile-lint
@@ -1 +1 @@
-FROM golangci/golangci-lint:v2.6-alpine@sha256:a7da5151e0bd61bd7f99e1ebd8e5e144b535b73b2762c498443ff4f6a4a538c4
\ No newline at end of file
+FROM golangci/golangci-lint:v2.6-alpine@sha256:a3abd55b17a74e36703f3d27e8183958b8bee15a1b1ffb25529192e114e0edbe
\ No newline at end of file

From c6a31df4574b9dc05bc3b0d2731b6026b9730315 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 20 Nov 2025 10:19:07 +1100
Subject: [PATCH 085/242] agent pool: Send error after disconnecting

---
 agent/agent_pool.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index cf769544d1..064b344a88 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -64,7 +64,9 @@ func (r *AgentPool) Start(ctx context.Context) error {
 
 	// Spawn each worker "in parallel" (in its own goroutine)
 	for _, worker := range r.workers {
-		go runWorker(ctx, worker, idleMon, errCh)
+		go func() {
+			errCh <- runWorker(ctx, worker, idleMon)
+		}()
 	}
 
 	setStat("✅ Workers spawned!")
@@ -77,21 +79,20 @@ func (r *AgentPool) Start(ctx context.Context) error {
 	return errors.Join(errs...) // nil if all errs are nil
 }
 
-func runWorker(ctx context.Context, worker *AgentWorker, idleMon *idleMonitor, errCh chan<- error) {
+func runWorker(ctx context.Context, worker *AgentWorker, idleMon *idleMonitor) error {
 	agentWorkersStarted.Inc()
 	defer agentWorkersEnded.Inc()
 	defer idleMon.markDead(worker)
 
 	// Connect the worker to the API
 	if err := worker.Connect(ctx); err != nil {
-		errCh <- err
-		return
+		return err
 	}
 	// Ensure the worker is disconnected at the end of this function.
 	defer worker.Disconnect(ctx) //nolint:errcheck // Error is logged within core/client
 
 	// Starts the agent worker and wait for it to finish.
-	errCh <- worker.Start(ctx, idleMon)
+	return worker.Start(ctx, idleMon)
 }
 
 // StopGracefully stops all workers in the pool gracefully.

From d07b6a8ebce1d86d39e8b90e5556c827da38201b Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 25 Nov 2025 16:54:59 +1100
Subject: [PATCH 086/242] Bump version and changelog for v3.114.0

---
 CHANGELOG.md    | 20 ++++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 487964b1fa..329690f34e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.114.0](https://github.com/buildkite/agent/tree/v3.114.0) (2025-11-25)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.113.0...v3.114.0)
+
+### Added
+- feat: add agent metadata to OTEL trace attributes [#3587](https://github.com/buildkite/agent/pull/3587) (@pyrocat101)
+
+### Fixed
+- Fix for the agent sometimes failing to disconnect properly when exiting - agent pool: Send error after disconnecting [#3596](https://github.com/buildkite/agent/pull/3596) (@DrJosh9000)
+
+### Internal
+- internal/redact: Add another test with minor cleanup [#3591](https://github.com/buildkite/agent/pull/3591) (@DrJosh9000)
+- Run gofumpt as part of CI [#3589](https://github.com/buildkite/agent/pull/3589) (@moskyb)
+
+### Dependency updates
+- build(deps): bump the cloud-providers group with 7 updates [#3593](https://github.com/buildkite/agent/pull/3593) (@dependabot[bot])
+- build(deps): bump the container-images group across 5 directories with 1 update [#3594](https://github.com/buildkite/agent/pull/3594) (@dependabot[bot])
+- build(deps): bump the container-images group across 1 directory with 2 updates [#3595](https://github.com/buildkite/agent/pull/3595) (@dependabot[bot])
+- build(deps): bump golang.org/x/crypto from 0.44.0 to 0.45.0 [#3590](https://github.com/buildkite/agent/pull/3590) (@dependabot[bot])
+
+
 ## [v3.113.0](https://github.com/buildkite/agent/tree/v3.113.0) (2025-11-18)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.112.0...v3.113.0)
 
diff --git a/version/VERSION b/version/VERSION
index e5abe2231a..0180c6b58a 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.113.0
+3.114.0

From 7ec157a32051bbc82659f6a3f2bd09931a380c7a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Dec 2025 17:23:50 +0000
Subject: [PATCH 087/242] build(deps): bump the container-images group across 5
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `ed5d56b` to `c308c24`

Updates `buildkite/agent-base` from `dff2305` to `6c9fe8e`

Updates `buildkite/agent-base` from `dfc9dfd` to `8f82f1a`

Updates `buildkite/agent-base` from `25b5e99` to `1b4a294`

Updates `buildkite/agent-base` from `37d39a8` to `53bf7c5`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index a246ebfeef..8c4d66f2a0 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:dff2305d45920beebe5203ac6c9bfb91d2d2ad0fd15f2a18f3822705593313fd
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:6c9fe8e1dba1c7e15395d8023943c3b8d2f07cda4a74fbc9d051d9a70db4801c
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index d0737c5628..ac172adfe0 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:ed5d56b89ecb72b0c847a45bb4214d3a8c4f72f890c2fd9d9331de20c64cdca8
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:c308c2475a1cf2303c575c2c01eee63f8491507523bdf9724fa5bf32f721718d
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 5a8e9f1241..a604929319 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:dfc9dfd401b17e513644e2b1bff702e0e5d4dd482bd93480622a09f9c13b521d
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:8f82f1a7ecdb81eac6ea8ef9e51e9f4f92af2141deda5b772f458f96e2b8ec51
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index ba1cb6ffa8..2f5786b037 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:25b5e99e6480ec51bf3c83232847a862bd64b06fbfce2047e57c0b4a008c97a3
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:1b4a294ffdab57213a6d957b9dac91fecc3d4e7b06f9cfe1e8e6ec6d76aa9257
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index 8948f77e97..9762cab9e8 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:37d39a8932facca2c9dcfb3da0c7f4c93bb0e3c974de2b3a244de11662cf54f8
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:53bf7c5cb541f9741b48625c79d0428c4321ca139b5cc387fd0347dc9dd58c52
 ARG TARGETOS
 ARG TARGETARCH
 

From 4e984b6cad9c8c0fb07da87c38803e44706a0c6a Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 2 Dec 2025 09:46:27 +1100
Subject: [PATCH 088/242] Update MIME types

---
 internal/mime/mime.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/mime/mime.go b/internal/mime/mime.go
index 659a2f720e..8c4db64112 100644
--- a/internal/mime/mime.go
+++ b/internal/mime/mime.go
@@ -325,6 +325,10 @@ var types = map[string]string{
 	".hal":         "application/vnd.hal+xml",
 	".hbci":        "application/vnd.hbci",
 	".hdf":         "application/x-hdf",
+	".heic":        "image/heic",
+	".heics":       "image/heic-sequence",
+	".heif":        "image/heif",
+	".heifs":       "image/heif-sequence",
 	".hh":          "text/x-c",
 	".hlp":         "application/winhlp",
 	".hpgl":        "application/vnd.hp-hpgl",

From 406292c1f8ef842825d8d2a44c4b0a2d9bb115e1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 2 Dec 2025 03:29:28 +0000
Subject: [PATCH 089/242] build(deps): bump the cloud-providers group with 6
 updates

Bumps the cloud-providers group with 6 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.1` | `1.32.2` |
| [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.20.11` | `1.20.12` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.274.0` | `1.275.0` |
| [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) | `1.49.0` | `1.49.1` |
| [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.92.0` | `1.92.1` |
| [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.23.2` | `1.24.0` |


Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.1 to 1.32.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.32.1...v1.32.2)

Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.20.11 to 1.20.12
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/sns/v1.20.11...service/sns/v1.20.12)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.274.0 to 1.275.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.274.0...service/ec2/v1.275.0)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.49.0 to 1.49.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.49.0...service/ssm/v1.49.1)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.92.0 to 1.92.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.92.0...service/s3/v1.92.1)

Updates `github.com/aws/smithy-go` from 1.23.2 to 1.24.0
- [Release notes](https://github.com/aws/smithy-go/releases)
- [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/aws/smithy-go/compare/v1.23.2...v1.24.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.32.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-version: 1.20.12
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.275.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.49.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-version: 1.92.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/smithy-go
  dependency-version: 1.24.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 22 +++++++++++-----------
 go.sum | 44 ++++++++++++++++++++++----------------------
 2 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/go.mod b/go.mod
index a25726023b..74c7dbd325 100644
--- a/go.mod
+++ b/go.mod
@@ -12,13 +12,13 @@ require (
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go-v2 v1.40.0
-	github.com/aws/aws-sdk-go-v2/config v1.32.1
+	github.com/aws/aws-sdk-go-v2/config v1.32.2
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.274.0
-	github.com/aws/aws-sdk-go-v2/service/kms v1.49.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0
-	github.com/aws/smithy-go v1.23.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0
+	github.com/aws/aws-sdk-go-v2/service/kms v1.49.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1
+	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
 	github.com/buildkite/go-pipeline v0.16.0
@@ -95,7 +95,7 @@ require (
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.2 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -104,10 +104,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
diff --git a/go.sum b/go.sum
index 67a03fc863..b0adea349b 100644
--- a/go.sum
+++ b/go.sum
@@ -80,14 +80,14 @@ github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrK
 github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
-github.com/aws/aws-sdk-go-v2/config v1.32.1 h1:iODUDLgk3q8/flEC7ymhmxjfoAnBDwEEYEVyKZ9mzjU=
-github.com/aws/aws-sdk-go-v2/config v1.32.1/go.mod h1:xoAgo17AGrPpJBSLg81W+ikM0cpOZG8ad04T2r+d5P0=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.1 h1:JeW+EwmtTE0yXFK8SmklrFh/cGTTXsQJumgMZNlbxfM=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.1/go.mod h1:BOoXiStwTF+fT2XufhO0Efssbi1CNIO/ZXpZu87N0pw=
+github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk=
+github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11 h1:NMchKj9gGzIJH4yln7g+Ci4BeVSCayE8CQ7cc+xH9FM=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.11/go.mod h1:eTZ6Kj2kFJ7UkKEWjlRPYI3fKcH+jKnsSaIom2XABBQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY=
@@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.274.0 h1:Q2+WD4KSVRkd27QxD9I30nM3O7B4WYwE+ua5dm2NJY0=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.274.0/go.mod h1:QrV+/GjhSrJh6MRRuTO6ZEg4M2I0nwPakf0lZHSrE1o=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0 h1:ymusjrsOjrcVBQNQXYFIQEHJIJ17/m+VoDSmWIMjGe0=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0/go.mod h1:QrV+/GjhSrJh6MRRuTO6ZEg4M2I0nwPakf0lZHSrE1o=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4=
@@ -106,20 +106,20 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.0 h1:QbztCKLI2qnjFZ/QYq3hZ8SW7SnTwB5h0NjREtKXIGo=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.0/go.mod h1:NZo9WJqQ0sxQ1Yqu1IwCHQFQunTms2MlVgejg16S1rY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0 h1:8FshVvnV2sr9kOSAbOnc/vwVmmAwMjOedKH6JW2ddPM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 h1:BDgIUYGEo5TkayOWv/oBLPphWwNm/A91AebUjAu5L5g=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.1/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 h1:U//SlnkE1wOQiIImxzdY5PXat4Wq+8rlfVEw4Y7J8as=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.4/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 h1:LU8S9W/mPDAU9q0FjCLi0TrCheLMGwzbRpvUMwYspcA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 h1:GdGmKtG+/Krag7VfyOXV17xjTCz0i9NT+JnqLTOI5nA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.1/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso=
-github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
-github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.1 h1:U0asSZ3ifpuIehDPkRI2rxHbmFUMplDA2VeR9Uogrmw=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.1/go.mod h1:NZo9WJqQ0sxQ1Yqu1IwCHQFQunTms2MlVgejg16S1rY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.5/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCdYW9ltGQYrFWg8XdkGDd8=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso=
+github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
+github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE=

From a764635240875cdad8b47e441c32a7e7c7ec4a5d Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 27 Nov 2025 14:26:16 +1100
Subject: [PATCH 090/242] Add e2e internal package

---
 go.mod                                      |   2 +
 go.sum                                      |   4 +
 internal/e2e/basic_test.go                  |  28 ++
 internal/e2e/doc.go                         |   4 +
 internal/e2e/testcase.go                    | 311 ++++++++++++++++++++
 internal/job/integration/executor_tester.go |   2 +-
 6 files changed, 350 insertions(+), 1 deletion(-)
 create mode 100644 internal/e2e/basic_test.go
 create mode 100644 internal/e2e/doc.go
 create mode 100644 internal/e2e/testcase.go

diff --git a/go.mod b/go.mod
index a25726023b..15ab113ff0 100644
--- a/go.mod
+++ b/go.mod
@@ -21,6 +21,7 @@ require (
 	github.com/aws/smithy-go v1.23.2
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
+	github.com/buildkite/go-buildkite/v4 v4.11.0
 	github.com/buildkite/go-pipeline v0.16.0
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
@@ -112,6 +113,7 @@ require (
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/buildkite/test-engine-client v1.6.0 // indirect
+	github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect
diff --git a/go.sum b/go.sum
index 67a03fc863..2e00246379 100644
--- a/go.sum
+++ b/go.sum
@@ -132,6 +132,8 @@ github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf/go.mod h1:CeKhh8xSs3WZAc50xABMxu+FlfAAd5PNumo7NfOv7EE=
 github.com/buildkite/bintest/v3 v3.3.0 h1:RTWcSaJRlOT6t/K311ejPf+0J3LE/QEODzVG3vlLnWo=
 github.com/buildkite/bintest/v3 v3.3.0/go.mod h1:btqpTsVODiJcb0NMdkkmtMQ6xoFc2W/nY5yy+3I0zcs=
+github.com/buildkite/go-buildkite/v4 v4.11.0 h1:rEvvUwITrqv433W9JWf6mj+NkkcM45s+ObhNs6C17i4=
+github.com/buildkite/go-buildkite/v4 v4.11.0/go.mod h1:DlebrRJqpZttXDjCW+MJ1QyW9AN++ZWt/UbPtKdbSSk=
 github.com/buildkite/go-pipeline v0.16.0 h1:wEgWUMRAgSg1ZnWOoA3AovtYYdTvN0dLY1zwUWmPP+4=
 github.com/buildkite/go-pipeline v0.16.0/go.mod h1:VE37qY3X5pmAKKUMoDZvPsHOQuyakB9cmXj9Qn6QasA=
 github.com/buildkite/interpolate v0.1.5 h1:v2Ji3voik69UZlbfoqzx+qfcsOKLA61nHdU79VV+tPU=
@@ -144,6 +146,8 @@ github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
 github.com/buildkite/zstash v0.5.0 h1:e70mf8U2EjEB1eixXR78s6bsLgfo6bWLisVlRv58wCI=
 github.com/buildkite/zstash v0.5.0/go.mod h1:h70JfAEa2Ys1GDQQ6CNoKIMfMgJ0LZkNmQnzK710PHQ=
+github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf h1:yxlp0s+Sge9UsKEK0Bsvjiopb9XRk+vxylmZ9eGBfm8=
+github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
diff --git a/internal/e2e/basic_test.go b/internal/e2e/basic_test.go
new file mode 100644
index 0000000000..be41bc2570
--- /dev/null
+++ b/internal/e2e/basic_test.go
@@ -0,0 +1,28 @@
+//go:build e2e
+
+package e2e
+
+import "testing"
+
+func TestBasicE2E(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, `
+agents:
+  queue: {{.queue}}
+steps:
+  - command: echo hello world
+`)
+
+	tc.startAgent()
+
+	build := tc.triggerBuild()
+	state, err := tc.waitForBuild(ctx, build)
+	if err != nil {
+		t.Fatalf("tc.waitForBuild(build %s) error = %v", build.ID, err)
+	}
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+
+	// TODO: add ability to inspect job logs
+}
diff --git a/internal/e2e/doc.go b/internal/e2e/doc.go
new file mode 100644
index 0000000000..d05b9492ee
--- /dev/null
+++ b/internal/e2e/doc.go
@@ -0,0 +1,4 @@
+// Package e2e holds the end-to-end tests and test framework.
+// Test files are tagged go:build e2e so they are not run by default
+// (e.g. with plain `go test ./...`).
+package e2e
diff --git a/internal/e2e/testcase.go b/internal/e2e/testcase.go
new file mode 100644
index 0000000000..b7579c2ff1
--- /dev/null
+++ b/internal/e2e/testcase.go
@@ -0,0 +1,311 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"text/template"
+	"time"
+
+	"github.com/buildkite/agent/v3/version"
+	"github.com/buildkite/roko"
+
+	"github.com/buildkite/go-buildkite/v4"
+)
+
+var (
+	// Filled in from secrets
+	apiToken   = os.Getenv("CI_E2E_TESTS_BUILDKITE_API_TOKEN")
+	agentToken = os.Getenv("CI_E2E_TESTS_AGENT_TOKEN")
+
+	// E2E testing config
+	agentPath     = os.Getenv("CI_E2E_TESTS_AGENT_PATH")
+	targetOrg     = os.Getenv("CI_E2E_TESTS_TARGET_ORG")
+	targetCluster = os.Getenv("CI_E2E_TESTS_TARGET_CLUSTER")
+
+	// Values from the Buildkite job running the tests
+	jobID = cmp.Or(
+		os.Getenv("BUILDKITE_JOB_ID"),
+		strconv.FormatInt(time.Now().UnixNano(), 10),
+	)
+	authorEmail = os.Getenv("BUILDKITE_BUILD_CREATOR_EMAIL")
+	authorName  = os.Getenv("BUILDKITE_BUILD_CREATOR")
+)
+
+const pipelineRepo = "https://github.com/buildkite/agent.git"
+
+type cleanupFn = func() error
+
+func nopCleanup() error { return nil }
+
+// testCase bundles the information needed to run an end-to-end test.
+// Note that it embeds testing.TB - each test should create its own testCase.
+type testCase struct {
+	testing.TB
+
+	fullName       string
+	bkClient       *buildkite.Client
+	pipelineConfig *template.Template
+	queue          *buildkite.ClusterQueue
+	pipeline       *buildkite.Pipeline
+}
+
+// newTestCase creates a new test case with a given pipeline config template,
+// and sets up the temporary queue and pipeline to run it.
+// It also registers cleanups with t.Cleanup so that the queue and pipeline
+// are (usually) automatically deleted.
+// It calls t.Fatal to end the test early if there was a failure setting up.
+func newTestCase(t testing.TB, pipelineConfigTemplate string) *testCase {
+	t.Helper()
+	ctx := t.Context()
+
+	name := strings.ToLower(t.Name() + "-" + jobID)
+
+	tmpl, err := template.New("pipeline").Parse(pipelineConfigTemplate)
+	if err != nil {
+		t.Fatalf("template.New(pipeline).Parse(%q) error = %v", pipelineConfigTemplate, err)
+	}
+
+	client, err := buildkite.NewClient(
+		buildkite.WithTokenAuth(apiToken),
+		buildkite.WithUserAgent("buildkite-agent-e2e-tests/0 "+version.UserAgent()),
+	)
+	if err != nil {
+		t.Fatalf("buildkite.NewClient(...) error = %v", err)
+	}
+
+	queue, cleanupQueue, err := createQueue(ctx, client, name)
+	if err != nil {
+		t.Fatalf("Could not create cluster queue in org %q cluster %q: testHelper.createQueue(ctx, %q) error = %v", targetOrg, targetCluster, name, err)
+	}
+	t.Cleanup(func() {
+		if err := cleanupQueue(); err != nil {
+			t.Logf("Could not clean up cluster queue %q with id %s in org %q cluster %q: cleanup() error = %v", name, queue.ID, targetOrg, targetCluster, err)
+		}
+	})
+
+	var pipelineCfg strings.Builder
+	tmplInput := map[string]string{"queue": queue.Key}
+	if err := tmpl.Execute(&pipelineCfg, tmplInput); err != nil {
+		t.Fatalf("Could not execute pipeline config template: tmpl.Execute(%q) error = %v", tmplInput, err)
+	}
+
+	pipeline, cleanupPipeline, err := createPipeline(ctx, client, name, pipelineCfg.String())
+	if err != nil {
+		t.Fatalf("Could not create pipeline with the following config in org %q: testHelper.createPipeline(%q, pipelineCfg) error = %v\n%s", targetOrg, name, err, pipelineCfg.String())
+	}
+	t.Cleanup(func() {
+		if err := cleanupPipeline(); err != nil {
+			t.Logf("Could not clean up pipeline %q (id = %s) in org %q: cleanup() = %v", pipeline.Slug, pipeline.ID, targetOrg, err)
+		}
+	})
+
+	return &testCase{
+		TB:             t,
+		fullName:       name,
+		bkClient:       client,
+		pipelineConfig: tmpl,
+		queue:          queue,
+		pipeline:       pipeline,
+	}
+}
+
+// triggerBuild creates a new build in the target pipeline. It returns the
+// build object. It also registers cleanups with t.Cleanup so that the build is
+// (usually) automatically cancelled if it is still running.
+// It calls t.Fatal if there was an error creating the build.
+func (tc *testCase) triggerBuild() *buildkite.Build {
+	tc.Helper()
+	ctx := tc.Context()
+
+	createBuild := buildkite.CreateBuild{
+		Author: buildkite.Author{
+			Email: authorEmail,
+			Name:  cmp.Or(authorName, "Agent E2E Tests"),
+		},
+		Commit:  "HEAD",
+		Branch:  "main",
+		Message: tc.fullName,
+	}
+
+	build, _, err := tc.bkClient.Builds.Create(ctx, targetOrg, tc.pipeline.Slug, createBuild)
+	if err != nil {
+		tc.Fatalf("tc.bkClient.Builds.Create(ctx, %q, %q, %v) error = %v", targetOrg, tc.pipeline.Slug, createBuild, err)
+	}
+
+	tc.Logf("Triggered a build at https://buildkite.com/%s/%s/builds/%d", targetOrg, tc.pipeline.Slug, build.Number)
+
+	tc.Cleanup(func() {
+		ctx := context.WithoutCancel(ctx) // allow cleanup after the test
+		_, err := tc.bkClient.Builds.Cancel(ctx, targetOrg, tc.pipeline.Slug, strconv.Itoa(build.Number))
+		if err != nil {
+			reasons := []string{
+				"already finished",
+				"already being canceled",
+				"already been canceled",
+				"No build found",
+			}
+			ignorable := slices.ContainsFunc(reasons, func(r string) bool {
+				return strings.Contains(err.Error(), r)
+			})
+			if ignorable {
+				return
+			}
+			tc.Logf("Couldn't cancel build %s: %v", build.ID, err)
+		}
+	})
+	return &build
+}
+
+// waitForBuild waits until the build is in a terminal state (passed, failed, canceled, etc). It polls the build once per second.
+func (tc *testCase) waitForBuild(ctx context.Context, build *buildkite.Build) (string, error) {
+	tick := time.Tick(time.Second)
+	for {
+		// The arg is called "id" but it needs the build number...
+		state, _, err := tc.bkClient.Builds.Get(ctx, targetOrg, tc.pipeline.Slug, strconv.Itoa(build.Number), &buildkite.BuildGetOptions{})
+		if err != nil {
+			return "", err
+		}
+		switch state.State {
+		case "passed", "failed", "canceled", "canceling":
+			return state.State, nil
+
+		case "scheduled", "running":
+			select {
+			case <-tick:
+				// time to poll again
+			case <-ctx.Done():
+				return "", ctx.Err()
+			}
+
+		default:
+			return state.State, fmt.Errorf("unknown build state %q", state.State)
+		}
+	}
+}
+
+// createQueue creates a cluster queue for running an end-to-end test in.
+// The returned cleanup function deletes the queue and should be called after
+// the test is finished.
+func createQueue(ctx context.Context, client *buildkite.Client, name string) (*buildkite.ClusterQueue, cleanupFn, error) {
+	cq, _, err := client.ClusterQueues.Create(ctx, targetOrg, targetCluster, buildkite.ClusterQueueCreate{
+		Key:         name,
+		Description: "Buildkite Agent E2E Test",
+	})
+	if err != nil {
+		return nil, nopCleanup, err
+	}
+
+	cleanup := func() error {
+		ctx := context.WithoutCancel(ctx) // allow cleanup after the test
+		r := roko.NewRetrier(
+			roko.WithStrategy(roko.Constant(5*time.Second)),
+			// The agent could take a while to become lost after being killed,
+			// so retry for a long time.
+			roko.WithMaxAttempts(65),
+		)
+		return r.Do(func(*roko.Retrier) error {
+			_, err := client.ClusterQueues.Delete(ctx, targetOrg, targetCluster, cq.ID)
+			return err
+		})
+	}
+	return &cq, cleanup, nil
+}
+
+// createPipeline creates a pipeline for running an end-to-end test in.
+// The returned cleanup function deletes the pipeline and should be called after
+// the test is finished.
+func createPipeline(ctx context.Context, client *buildkite.Client, name, config string) (*buildkite.Pipeline, cleanupFn, error) {
+	p, _, err := client.Pipelines.Create(ctx, targetOrg, buildkite.CreatePipeline{
+		Name:        name,
+		Repository:  pipelineRepo,
+		Description: "Buildkite Agent E2E Test",
+		ProviderSettings: &buildkite.GitHubSettings{
+			TriggerMode: "none",
+		},
+		Configuration: config,
+		ClusterID:     targetCluster,
+	})
+	if err != nil {
+		return nil, nopCleanup, err
+	}
+
+	cleanup := func() error {
+		ctx := context.WithoutCancel(ctx) // allow cleanup after the test
+		r := roko.NewRetrier(
+			roko.WithStrategy(roko.Constant(5*time.Second)),
+			roko.WithMaxAttempts(5),
+		)
+		return r.Do(func(*roko.Retrier) error {
+			_, err := client.Pipelines.Delete(ctx, targetOrg, p.Slug)
+			return err
+		})
+	}
+	return &p, cleanup, nil
+}
+
+// startAgent starts a copy of the agent (at agentPath, using agentToken).
+// The agent should be automatically cleaned up at the end of the test.
+func (tc *testCase) startAgent(extraArgs ...string) *exec.Cmd {
+	tc.Helper()
+	dir := tc.TempDir()
+	buildPath := filepath.Join(dir, "builds")
+	hooksPath := filepath.Join(dir, "hooks")
+	socketsPath := filepath.Join(dir, "sockets")
+	pluginsPath := filepath.Join(dir, "plugins")
+	for _, path := range []string{buildPath, hooksPath, socketsPath, pluginsPath} {
+		if err := os.Mkdir(path, 0o700); err != nil {
+			tc.Fatalf("Couldn't create dir inside temporary agent dir: os.Mkdir(%q, %o) = %v", path, 0o700, err)
+		}
+	}
+
+	args := append([]string{
+		"start",
+		"--debug",
+		"--token", agentToken,
+		"--name", tc.fullName,
+		"--queue", tc.queue.Key,
+		"--build-path", buildPath,
+		"--hooks-path", hooksPath,
+		"--sockets-path", socketsPath,
+		"--plugins-path", pluginsPath,
+	}, extraArgs...)
+	tc.Logf("Starting agent with args: %q", args)
+
+	cmd := exec.CommandContext(tc.Context(), agentPath, args...)
+	// Ensure minimal environment variable shenanigans by setting only these:
+	cmd.Env = []string{
+		"HOME=" + os.Getenv("HOME"),
+		"PATH=" + os.Getenv("PATH"),
+	}
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+
+	// The agent should be cancelled automatically by t.Context.
+	// The default Cancel func set by CommandContext is `cmd.Process.Kill()`,
+	// so the agent would exit immediately and disconnect uncleanly.
+	// It is eventually marked lost on the backend, but not for a few minutes,
+	// which blocks queue cleanup for a while.
+	// This replacement Cancel func SIGQUITs it, which forces an ungraceful
+	// but clean exit (jobs are cancelled but it has time to disconnect).
+	// To ensure the agent _is_ eventually SIGKILLed, WaitDelay is set.
+	cmd.Cancel = func() error {
+		return cmd.Process.Signal(syscall.SIGQUIT)
+	}
+	cmd.WaitDelay = 10 * time.Second
+
+	if err := cmd.Start(); err != nil {
+		tc.Fatalf("Couldn't start agent command %v: %v", cmd, err)
+	}
+	return cmd
+}
diff --git a/internal/job/integration/executor_tester.go b/internal/job/integration/executor_tester.go
index 5f199006ee..d9d2bdffe8 100644
--- a/internal/job/integration/executor_tester.go
+++ b/internal/job/integration/executor_tester.go
@@ -184,7 +184,7 @@ func (e *ExecutorTester) MustMock(t *testing.T, name string) *bintest.Mock {
 	t.Helper()
 	mock, err := e.Mock(name)
 	if err != nil {
-		t.Fatalf("BootstrapTester.Mock(%q) error = %v", name, err)
+		t.Fatalf("ExecutorTester.Mock(%q) error = %v", name, err)
 	}
 	return mock
 }

From 41677727bcdd7cdd3d1f5ce04711b908b5d95cf1 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 27 Nov 2025 14:26:16 +1100
Subject: [PATCH 091/242] Add e2e pipeline

---
 .buildkite/Dockerfile-e2e     |  1 +
 .buildkite/docker-compose.yml | 19 +++++++++++++++++--
 .buildkite/pipeline.e2e.yml   | 19 +++++++++++++++++++
 .buildkite/steps/e2e-tests.sh | 10 ++++++++++
 4 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 .buildkite/Dockerfile-e2e
 create mode 100644 .buildkite/pipeline.e2e.yml
 create mode 100755 .buildkite/steps/e2e-tests.sh

diff --git a/.buildkite/Dockerfile-e2e b/.buildkite/Dockerfile-e2e
new file mode 100644
index 0000000000..38ea1c5578
--- /dev/null
+++ b/.buildkite/Dockerfile-e2e
@@ -0,0 +1 @@
+FROM public.ecr.aws/docker/library/golang:1.24.10@sha256:7b13449f08287fdb53114d65bdf20eb3965e4e54997903b5cb9477df0ea37c12
diff --git a/.buildkite/docker-compose.yml b/.buildkite/docker-compose.yml
index 17267fd777..d574b5c47b 100644
--- a/.buildkite/docker-compose.yml
+++ b/.buildkite/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
   lint:
     build:
@@ -32,6 +30,23 @@ services:
       - GOMODCACHE=/gomodcache
       - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN
 
+  e2e:
+    build:
+      context: .
+      dockerfile: Dockerfile-e2e
+    volumes:
+      - ../:/work:cached
+    working_dir: /work
+    environment:
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_BUILD_CREATOR_EMAIL
+      - BUILDKITE_BUILD_CREATOR
+      - CI_E2E_TESTS_BUILDKITE_API_TOKEN
+      - CI_E2E_TESTS_AGENT_TOKEN
+      - CI_E2E_TESTS_TARGET_ORG
+      - CI_E2E_TESTS_TARGET_CLUSTER
+
   ruby:
     image: ruby:2.7.6
     volumes:
diff --git a/.buildkite/pipeline.e2e.yml b/.buildkite/pipeline.e2e.yml
new file mode 100644
index 0000000000..5c231fb4cb
--- /dev/null
+++ b/.buildkite/pipeline.e2e.yml
@@ -0,0 +1,19 @@
+agents:
+  queue: agent-runners-linux-amd64
+
+env:
+  CI_E2E_TESTS_TARGET_ORG: agent-e2e-testing
+  CI_E2E_TESTS_TARGET_CLUSTER: e1773a06-b43c-4b77-84e5-5e088a737f6a
+
+steps:
+  - label: ":go::sleuth_or_spy: Run agent end-to-end tests"
+    command: .buildkite/steps/e2e-tests.sh
+    secrets:
+      - CI_E2E_TESTS_BUILDKITE_API_TOKEN
+      - CI_E2E_TESTS_AGENT_TOKEN
+    plugins:
+      - docker-compose#v4.14.0:
+          config: .buildkite/docker-compose.yml
+          cli-version: 2
+          mount-buildkite-agent: true
+          run: e2e
diff --git a/.buildkite/steps/e2e-tests.sh b/.buildkite/steps/e2e-tests.sh
new file mode 100755
index 0000000000..e8f4eab7a4
--- /dev/null
+++ b/.buildkite/steps/e2e-tests.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# TODO: download the agent to test as an artifact
+#buildkite-agent artifact download buildkite-agent . --build "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}"
+#export CI_E2E_TESTS_AGENT_PATH="${PWD}/buildkite-agent"
+
+# For now, e2e test the agent that's currently running
+export CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
+go test -v -tags e2e ./internal/e2e/... | sed -e 's/^/>  /'

From 94bc85b6e8314f9fe7820b22441a9506c301850a Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 2 Dec 2025 17:00:41 +1100
Subject: [PATCH 092/242] Trigger the new E2E pipeline

---
 .buildkite/docker-compose.yml |  1 +
 .buildkite/pipeline.yml       | 34 ++++++++++++++++++++++++----------
 .buildkite/steps/e2e-tests.sh | 15 ++++++++++-----
 3 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/.buildkite/docker-compose.yml b/.buildkite/docker-compose.yml
index d574b5c47b..d5c59f21b9 100644
--- a/.buildkite/docker-compose.yml
+++ b/.buildkite/docker-compose.yml
@@ -42,6 +42,7 @@ services:
       - BUILDKITE_JOB_ID
       - BUILDKITE_BUILD_CREATOR_EMAIL
       - BUILDKITE_BUILD_CREATOR
+      - BUILDKITE_TRIGGERED_FROM_BUILD_ID
       - CI_E2E_TESTS_BUILDKITE_API_TOKEN
       - CI_E2E_TESTS_AGENT_TOKEN
       - CI_E2E_TESTS_TARGET_ORG
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 6d750e7eb9..cd37e5a7c3 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -44,7 +44,7 @@ steps:
               - "os=linux"
               - "arch=amd64"
               - "race=false"
-  
+
     - name: ":linux: Linux ARM64 Tests"
       key: test-linux-arm64
       command: ".buildkite/steps/tests.sh"
@@ -67,7 +67,7 @@ steps:
               - "os=linux"
               - "arch=arm64"
               - "race=false"
-              
+
     - name: ":windows: Windows AMD64 Tests"
       key: test-windows
       command: "bash .buildkite\\steps\\tests.sh"
@@ -85,7 +85,7 @@ steps:
               - "os=windows"
               - "arch=arm64"
               - "race=false"
-  
+
     - name: ":satellite: Detect Data Races"
       key: test-race-linux-arm64
       command: ".buildkite/steps/tests.sh -race"
@@ -109,7 +109,7 @@ steps:
               - "os=linux"
               - "arch=arm64"
               - "race=true"
-              
+
     - name: ":coverage: Test coverage report Linux ARM64"
       key: test-coverage-linux-arm64
       command: ".buildkite/steps/test-coverage-report.sh"
@@ -126,7 +126,7 @@ steps:
         - artifacts#v1.9.4:
             download: "coverage/**"
             step: test-linux-arm64
-    
+
     - name: ":coverage: Test coverage report Linux AMD64"
       key: test-coverage-linux-amd64
       command: ".buildkite/steps/test-coverage-report.sh"
@@ -143,7 +143,7 @@ steps:
         - artifacts#v1.9.4:
             download: "coverage/**"
             step: test-linux-amd64
-    
+
     - name: ":coverage: Test coverage report Linux ARM64 Race"
       key: test-coverage-linux-arm64-race
       command: ".buildkite/steps/test-coverage-report.sh"
@@ -160,7 +160,7 @@ steps:
         - artifacts#v1.9.4:
             download: "coverage/**"
             step: test-race-linux-arm64
-  
+
     - label: ":writing_hand: Annotate with Test Failures"
       depends_on:
         - test-linux-amd64
@@ -171,7 +171,7 @@ steps:
       plugins:
         - junit-annotate#v1.6.0:
             artifacts: junit-*.xml
-            
+
   # --- end Tests and Coverage ---
 
   - group: ":hammer_and_wrench: Binary builds"
@@ -225,9 +225,23 @@ steps:
 
             - with: { os: openbsd, arch: arm64 }
               skip: "arm64 OpenBSD is not currently supported"
-              
+
   # --- end Binary builds ---
 
+  - label: ":end::two::end: E2E Testing"
+    key: e2e-tests
+    depends_on: build-binary
+    trigger: agent-e2e-tests
+    async: false
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+
   - label: ":bathtub: Check version string is clean"
     key: check-version-string
     depends_on: build-binary
@@ -340,7 +354,7 @@ steps:
               - ubuntu-22.04
               - ubuntu-24.04
               - sidecar
-              
+
   # --- end Docker Image tests ---
 
   - name: ":debian: Debian package build"
diff --git a/.buildkite/steps/e2e-tests.sh b/.buildkite/steps/e2e-tests.sh
index e8f4eab7a4..f60803bacf 100755
--- a/.buildkite/steps/e2e-tests.sh
+++ b/.buildkite/steps/e2e-tests.sh
@@ -1,10 +1,15 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# TODO: download the agent to test as an artifact
-#buildkite-agent artifact download buildkite-agent . --build "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}"
-#export CI_E2E_TESTS_AGENT_PATH="${PWD}/buildkite-agent"
+if [[ -z "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}" ]] ; then
+	# For now, e2e test the agent that's currently running
+	export CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
+else
+	# Download the artifact from the triggering build
+	ARTIFACT="pkg/buildkite-agent-$(go env GOOS)-$(go env GOARCH)"
+	buildkite-agent artifact download "${ARTIFACT}" . --build "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}"
+	chmod +x "${ARTIFACT}"
+	export CI_E2E_TESTS_AGENT_PATH="${PWD}/${ARTIFACT}"
+fi
 
-# For now, e2e test the agent that's currently running
-export CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
 go test -v -tags e2e ./internal/e2e/... | sed -e 's/^/>  /'

From 37616852e665de0d17afd8fa030a3be5f93e9fe9 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 3 Dec 2025 15:51:03 +1100
Subject: [PATCH 093/242] Fetch and compare job logs

---
 internal/e2e/basic_test.go | 16 +++++-----
 internal/e2e/testcase.go   | 60 +++++++++++++++++++++++++++++++++-----
 2 files changed, 62 insertions(+), 14 deletions(-)

diff --git a/internal/e2e/basic_test.go b/internal/e2e/basic_test.go
index be41bc2570..f2194f7252 100644
--- a/internal/e2e/basic_test.go
+++ b/internal/e2e/basic_test.go
@@ -2,7 +2,10 @@
 
 package e2e
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 func TestBasicE2E(t *testing.T) {
 	ctx := t.Context()
@@ -14,15 +17,14 @@ steps:
 `)
 
 	tc.startAgent()
-
 	build := tc.triggerBuild()
-	state, err := tc.waitForBuild(ctx, build)
-	if err != nil {
-		t.Fatalf("tc.waitForBuild(build %s) error = %v", build.ID, err)
-	}
+	state := tc.waitForBuild(ctx, build)
 	if got, want := state, "passed"; got != want {
 		t.Errorf("Build state = %q, want %q", got, want)
 	}
 
-	// TODO: add ability to inspect job logs
+	logs := tc.fetchLogs(ctx, build)
+	if !strings.Contains(logs, "hello world") {
+		t.Errorf("tc.fetchLogs(ctx, build %q) logs as follows, did not contain 'hello world'\n%s", build.ID, logs)
+	}
 }
diff --git a/internal/e2e/testcase.go b/internal/e2e/testcase.go
index b7579c2ff1..83c5d5cec3 100644
--- a/internal/e2e/testcase.go
+++ b/internal/e2e/testcase.go
@@ -18,9 +18,9 @@ import (
 	"time"
 
 	"github.com/buildkite/agent/v3/version"
-	"github.com/buildkite/roko"
 
 	"github.com/buildkite/go-buildkite/v4"
+	"github.com/buildkite/roko"
 )
 
 var (
@@ -167,29 +167,38 @@ func (tc *testCase) triggerBuild() *buildkite.Build {
 	return &build
 }
 
-// waitForBuild waits until the build is in a terminal state (passed, failed, canceled, etc). It polls the build once per second.
-func (tc *testCase) waitForBuild(ctx context.Context, build *buildkite.Build) (string, error) {
+// waitForBuild waits until the build is in a terminal state
+// (passed, failed, canceled, etc). It polls the build once per second.
+// Note that the build pointed to by build is updated with the latest state
+// after each poll. It calls t.Fatal if there was an error fetching the build
+// or the context ends.
+func (tc *testCase) waitForBuild(ctx context.Context, build *buildkite.Build) string {
 	tick := time.Tick(time.Second)
 	for {
 		// The arg is called "id" but it needs the build number...
 		state, _, err := tc.bkClient.Builds.Get(ctx, targetOrg, tc.pipeline.Slug, strconv.Itoa(build.Number), &buildkite.BuildGetOptions{})
 		if err != nil {
-			return "", err
+			tc.Fatalf("buildkite.Client.Builds.Get(ctx, %q, %q, %d, &{}) error = %v", targetOrg, tc.pipeline.Slug, build.Number, err)
+			return ""
 		}
+
+		*build = state
 		switch state.State {
 		case "passed", "failed", "canceled", "canceling":
-			return state.State, nil
+			return state.State
 
 		case "scheduled", "running":
 			select {
 			case <-tick:
 				// time to poll again
 			case <-ctx.Done():
-				return "", ctx.Err()
+				tc.Fatalf("waitForBuild context ended: %v", ctx.Err())
+				return ""
 			}
 
 		default:
-			return state.State, fmt.Errorf("unknown build state %q", state.State)
+			tc.Logf("waitForBuild read an unknown build state: %q", state.State)
+			return state.State
 		}
 	}
 }
@@ -309,3 +318,40 @@ func (tc *testCase) startAgent(extraArgs ...string) *exec.Cmd {
 	}
 	return cmd
 }
+
+// fetchLogs fetches the logs for all jobs in a build, as a single string.
+// It calls t.Fatal to end the test if the logs of any job' cannot be fetched
+// within a few retries.
+func (tc *testCase) fetchLogs(ctx context.Context, build *buildkite.Build) string {
+	tc.Helper()
+
+	r := roko.NewRetrier(
+		roko.WithStrategy(roko.Constant(5*time.Second)),
+		roko.WithMaxAttempts(5),
+	)
+	logs, err := roko.DoFunc(ctx, r, func(*roko.Retrier) (string, error) {
+		var logs strings.Builder
+		for _, job := range build.Jobs {
+			jobLog, _, err := tc.bkClient.Jobs.GetJobLog(
+				ctx,
+				targetOrg,
+				tc.pipeline.Slug,
+				strconv.Itoa(build.Number),
+				job.ID,
+			)
+			if err != nil {
+				return "", err
+			}
+			if jobLog.Content == "" {
+				return "", fmt.Errorf("job %q log empty", job.ID)
+			}
+
+			logs.WriteString(jobLog.Content)
+		}
+		return logs.String(), nil
+	})
+	if err != nil {
+		tc.Fatalf("fetchLogs failed to fetch logs: %v", err)
+	}
+	return logs
+}

From 5bfc4e729c8e7b99ea932a0e0760cd05b652f9c6 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 4 Dec 2025 10:28:49 +1100
Subject: [PATCH 094/242] Read target org and cluster from agent token

---
 .buildkite/pipeline.e2e.yml |  4 ----
 api/token.go                | 35 +++++++++++++++++++++++++++++++++++
 internal/e2e/main_test.go   | 28 ++++++++++++++++++++++++++++
 internal/e2e/testcase.go    |  8 +++++---
 4 files changed, 68 insertions(+), 7 deletions(-)
 create mode 100644 api/token.go
 create mode 100644 internal/e2e/main_test.go

diff --git a/.buildkite/pipeline.e2e.yml b/.buildkite/pipeline.e2e.yml
index 5c231fb4cb..8e6204121d 100644
--- a/.buildkite/pipeline.e2e.yml
+++ b/.buildkite/pipeline.e2e.yml
@@ -1,10 +1,6 @@
 agents:
   queue: agent-runners-linux-amd64
 
-env:
-  CI_E2E_TESTS_TARGET_ORG: agent-e2e-testing
-  CI_E2E_TESTS_TARGET_CLUSTER: e1773a06-b43c-4b77-84e5-5e088a737f6a
-
 steps:
   - label: ":go::sleuth_or_spy: Run agent end-to-end tests"
     command: .buildkite/steps/e2e-tests.sh
diff --git a/api/token.go b/api/token.go
new file mode 100644
index 0000000000..3f3918736e
--- /dev/null
+++ b/api/token.go
@@ -0,0 +1,35 @@
+package api
+
+import (
+	"context"
+	"net/http"
+)
+
+// AgentTokenIdentity describes token identity information.
+type AgentTokenIdentity struct {
+	UUID                  string `json:"uuid"`
+	Description           string `json:"description"`
+	TokenType             string `json:"token_type"`
+	OrganizationSlug      string `json:"organization_slug"`
+	OrganizationUUID      string `json:"organization_uuid"`
+	ClusterUUID           string `json:"cluster_uuid"`
+	ClusterName           string `json:"cluster_name"`
+	OrganizationQueueUUID string `json:"organization_queue_uuid"`
+	OrganizationQueueKey  string `json:"organization_queue_key"`
+}
+
+// GetTokenIdentity gets the identity information of an agent token.
+func (c *Client) GetTokenIdentity(ctx context.Context) (*AgentTokenIdentity, *Response, error) {
+	req, err := c.newRequest(ctx, http.MethodGet, "token", nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	ident := new(AgentTokenIdentity)
+	resp, err := c.doRequest(req, ident)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return ident, resp, nil
+}
diff --git a/internal/e2e/main_test.go b/internal/e2e/main_test.go
new file mode 100644
index 0000000000..655bb50c42
--- /dev/null
+++ b/internal/e2e/main_test.go
@@ -0,0 +1,28 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/buildkite/agent/v3/api"
+	"github.com/buildkite/agent/v3/logger"
+)
+
+func TestMain(m *testing.M) {
+	l := logger.NewConsoleLogger(logger.NewTextPrinter(os.Stderr), os.Exit)
+	client := api.NewClient(l, api.Config{
+		Token: agentToken,
+	})
+	ctx := context.Background()
+	ident, _, err := client.GetTokenIdentity(ctx)
+	if err != nil {
+		l.Fatal("Could not read token identity: %v", err)
+	}
+	targetOrg = ident.OrganizationSlug
+	targetCluster = ident.ClusterUUID
+
+	os.Exit(m.Run())
+}
diff --git a/internal/e2e/testcase.go b/internal/e2e/testcase.go
index 83c5d5cec3..ca09978bf1 100644
--- a/internal/e2e/testcase.go
+++ b/internal/e2e/testcase.go
@@ -29,9 +29,11 @@ var (
 	agentToken = os.Getenv("CI_E2E_TESTS_AGENT_TOKEN")
 
 	// E2E testing config
-	agentPath     = os.Getenv("CI_E2E_TESTS_AGENT_PATH")
-	targetOrg     = os.Getenv("CI_E2E_TESTS_TARGET_ORG")
-	targetCluster = os.Getenv("CI_E2E_TESTS_TARGET_CLUSTER")
+	agentPath = os.Getenv("CI_E2E_TESTS_AGENT_PATH")
+
+	// Obtained from agentToken in main_test.go
+	targetOrg     string
+	targetCluster string
 
 	// Values from the Buildkite job running the tests
 	jobID = cmp.Or(

From f544ac31b0fb24be082ac83dd59bae41f123483f Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 4 Dec 2025 12:13:48 +1100
Subject: [PATCH 095/242] Upload E2E test results to Test Engine

---
 .buildkite/docker-compose.yml | 2 --
 .buildkite/pipeline.e2e.yml   | 6 ++++++
 .buildkite/steps/e2e-tests.sh | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.buildkite/docker-compose.yml b/.buildkite/docker-compose.yml
index d5c59f21b9..720b93f8f5 100644
--- a/.buildkite/docker-compose.yml
+++ b/.buildkite/docker-compose.yml
@@ -45,8 +45,6 @@ services:
       - BUILDKITE_TRIGGERED_FROM_BUILD_ID
       - CI_E2E_TESTS_BUILDKITE_API_TOKEN
       - CI_E2E_TESTS_AGENT_TOKEN
-      - CI_E2E_TESTS_TARGET_ORG
-      - CI_E2E_TESTS_TARGET_CLUSTER
 
   ruby:
     image: ruby:2.7.6
diff --git a/.buildkite/pipeline.e2e.yml b/.buildkite/pipeline.e2e.yml
index 8e6204121d..81e573143f 100644
--- a/.buildkite/pipeline.e2e.yml
+++ b/.buildkite/pipeline.e2e.yml
@@ -7,9 +7,15 @@ steps:
     secrets:
       - CI_E2E_TESTS_BUILDKITE_API_TOKEN
       - CI_E2E_TESTS_AGENT_TOKEN
+      - CI_E2E_TESTS_ANALYTICS_TOKEN
     plugins:
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
           cli-version: 2
           mount-buildkite-agent: true
           run: e2e
+      - test-collector#v1.11.0:
+          files: "junit.xml"
+          format: "junit"
+          api-token-env-name: CI_E2E_TESTS_ANALYTICS_TOKEN
+
diff --git a/.buildkite/steps/e2e-tests.sh b/.buildkite/steps/e2e-tests.sh
index f60803bacf..b27c671624 100755
--- a/.buildkite/steps/e2e-tests.sh
+++ b/.buildkite/steps/e2e-tests.sh
@@ -12,4 +12,4 @@ else
 	export CI_E2E_TESTS_AGENT_PATH="${PWD}/${ARTIFACT}"
 fi
 
-go test -v -tags e2e ./internal/e2e/... | sed -e 's/^/>  /'
+go tool gotestsum --junitfile junit.xml -- -tags e2e ./internal/e2e/...

From 79be252ce7cf8893adf2ae18ca723930b7557577 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 4 Dec 2025 15:19:58 +1100
Subject: [PATCH 096/242] Move pipeline YAML into fixtures dir

---
 internal/e2e/basic_test.go           |  7 +------
 internal/e2e/fixtures/basic_e2e.yaml |  4 ++++
 internal/e2e/testcase.go             | 16 +++++++++++++---
 3 files changed, 18 insertions(+), 9 deletions(-)
 create mode 100644 internal/e2e/fixtures/basic_e2e.yaml

diff --git a/internal/e2e/basic_test.go b/internal/e2e/basic_test.go
index f2194f7252..9862fb16b4 100644
--- a/internal/e2e/basic_test.go
+++ b/internal/e2e/basic_test.go
@@ -9,12 +9,7 @@ import (
 
 func TestBasicE2E(t *testing.T) {
 	ctx := t.Context()
-	tc := newTestCase(t, `
-agents:
-  queue: {{.queue}}
-steps:
-  - command: echo hello world
-`)
+	tc := newTestCase(t, "basic_e2e.yaml")
 
 	tc.startAgent()
 	build := tc.triggerBuild()
diff --git a/internal/e2e/fixtures/basic_e2e.yaml b/internal/e2e/fixtures/basic_e2e.yaml
new file mode 100644
index 0000000000..54b48ef6bd
--- /dev/null
+++ b/internal/e2e/fixtures/basic_e2e.yaml
@@ -0,0 +1,4 @@
+agents:
+  queue: "{{.queue}}"
+steps:
+  - command: echo hello world
diff --git a/internal/e2e/testcase.go b/internal/e2e/testcase.go
index ca09978bf1..ea34af86d1 100644
--- a/internal/e2e/testcase.go
+++ b/internal/e2e/testcase.go
@@ -5,9 +5,11 @@ package e2e
 import (
 	"cmp"
 	"context"
+	"embed"
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"slices"
 	"strconv"
@@ -50,6 +52,9 @@ type cleanupFn = func() error
 
 func nopCleanup() error { return nil }
 
+//go:embed fixtures
+var fixturesFS embed.FS
+
 // testCase bundles the information needed to run an end-to-end test.
 // Note that it embeds testing.TB - each test should create its own testCase.
 type testCase struct {
@@ -67,15 +72,20 @@ type testCase struct {
 // It also registers cleanups with t.Cleanup so that the queue and pipeline
 // are (usually) automatically deleted.
 // It calls t.Fatal to end the test early if there was a failure setting up.
-func newTestCase(t testing.TB, pipelineConfigTemplate string) *testCase {
+func newTestCase(t testing.TB, file string) *testCase {
 	t.Helper()
 	ctx := t.Context()
 
 	name := strings.ToLower(t.Name() + "-" + jobID)
 
-	tmpl, err := template.New("pipeline").Parse(pipelineConfigTemplate)
+	pipelineCfgTmpl, err := fixturesFS.ReadFile(path.Join("fixtures", file))
+	if err != nil {
+		t.Fatalf("fixturesFS.ReadFile(%q) error = %v", file, err)
+	}
+
+	tmpl, err := template.New("pipeline").Parse(string(pipelineCfgTmpl))
 	if err != nil {
-		t.Fatalf("template.New(pipeline).Parse(%q) error = %v", pipelineConfigTemplate, err)
+		t.Fatalf("template.New(pipeline).Parse(%q) error = %v", pipelineCfgTmpl, err)
 	}
 
 	client, err := buildkite.NewClient(

From 20fb02042e04e663de011c74660fbf0f9c1f23c2 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 4 Dec 2025 16:21:10 +1100
Subject: [PATCH 097/242] Log agent logs separately

---
 internal/e2e/testcase.go | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/internal/e2e/testcase.go b/internal/e2e/testcase.go
index ea34af86d1..34b3509e7a 100644
--- a/internal/e2e/testcase.go
+++ b/internal/e2e/testcase.go
@@ -309,8 +309,16 @@ func (tc *testCase) startAgent(extraArgs ...string) *exec.Cmd {
 		"HOME=" + os.Getenv("HOME"),
 		"PATH=" + os.Getenv("PATH"),
 	}
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
+	var buf strings.Builder
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	tc.Cleanup(func() {
+		if err := cmd.Wait(); err != nil {
+			tc.Logf("Couldn't wait for agent to exit: cmd.Wait() = %v", err)
+		}
+		tc.Log("Agent output:")
+		tc.Log(buf.String())
+	})
 
 	// The agent should be cancelled automatically by t.Context.
 	// The default Cancel func set by CommandContext is `cmd.Process.Kill()`,

From 756b0abc10c7864bff540b715fd5cf9c5f9ec09c Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 4 Dec 2025 11:00:18 +1100
Subject: [PATCH 098/242] Fix issue where artifacts uploaded to custom s3
 buckets would have a double slash prefix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There's an edge case in URL construction for artifacts uploaded to custom S3 buckets where the key for the object would be generated as `/object-key` (note preceding slash), and then directly appended to a url like `https://my-definitely-realy-s3-bucket.s3.amazonaws.com/`, leading to a final object url of `https://buildkite-benno-artifacts-test.s3.amazonaws.com//object-key` (note the double slash!). See [the implementation](https://github.com/buildkite/agent/blob/2a95448832f3fa4c2eb08dabb69db06ff01782fb/internal/artifact/s3_uploader.go#L147) for more details — `u.BucketPath` is empty by default in custom artifact bucket cases.

When we were on the AWS Go SDK v1, this was fine, as the SDK [cleaned urls by default](https://github.com/aws/aws-sdk-go/issues/2559), leading to the object's key being normalized from `/object-key` to `object-key`. However, in an [undocumented breaking change](https://github.com/aws/aws-sdk-go-v2/issues/3095) (🫠), the SDK v2 doesn't do this cleaning, so the object would be uploaded with a leading slash on its key.

The artifact downloader relied on the URL cleaning functionality of the go SDK, so when attempting to download the artifact, it would generate a key of `object-key`, without the leading slash, leading to consistent 404s when trying to download artifacts from custom buckets.

This PR updates the artifact upload URL generation logic to do a path-join of the key rather than a crude append, essentially emulating the logic that was present in v1 of the AWS Go SDK.
---
 internal/artifact/s3_uploader.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal/artifact/s3_uploader.go b/internal/artifact/s3_uploader.go
index 9fc8613ec2..54c1208659 100644
--- a/internal/artifact/s3_uploader.go
+++ b/internal/artifact/s3_uploader.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"path"
 	"slices"
 	"strings"
 	"time"
@@ -83,11 +84,10 @@ func (u *S3Uploader) URL(artifact *api.Artifact) string {
 		baseUrl = os.Getenv("BUILDKITE_S3_ACCESS_URL")
 	}
 
-	url, _ := url.Parse(baseUrl)
+	uri, _ := url.Parse(baseUrl)
+	uri.Path = path.Join(uri.Path, u.artifactPath(artifact))
 
-	url.Path += u.artifactPath(artifact)
-
-	return url.String()
+	return uri.String()
 }
 
 func (u *S3Uploader) CreateWork(artifact *api.Artifact) ([]workUnit, error) {

From 537c838f84369234626b36a40aba321bbe5a1c5c Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 4 Dec 2025 13:41:47 +1100
Subject: [PATCH 099/242] Add e2e test for artifact upload/download

---
 .buildkite/steps/e2e-tests.sh                 |  5 +++--
 internal/e2e/artifact_test.go                 | 21 +++++++++++++++++++
 .../fixtures/artifact_upload_download.yaml    | 11 ++++++++++
 3 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 internal/e2e/artifact_test.go
 create mode 100644 internal/e2e/fixtures/artifact_upload_download.yaml

diff --git a/.buildkite/steps/e2e-tests.sh b/.buildkite/steps/e2e-tests.sh
index b27c671624..9b86ffd990 100755
--- a/.buildkite/steps/e2e-tests.sh
+++ b/.buildkite/steps/e2e-tests.sh
@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [[ -z "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}" ]] ; then
+if [[ -z "${BUILDKITE_TRIGGERED_FROM_BUILD_ID:-}" ]] ; then
 	# For now, e2e test the agent that's currently running
-	export CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
+	CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
+	export CI_E2E_TESTS_AGENT_PATH
 else
 	# Download the artifact from the triggering build
 	ARTIFACT="pkg/buildkite-agent-$(go env GOOS)-$(go env GOARCH)"
diff --git a/internal/e2e/artifact_test.go b/internal/e2e/artifact_test.go
new file mode 100644
index 0000000000..da449a932e
--- /dev/null
+++ b/internal/e2e/artifact_test.go
@@ -0,0 +1,21 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"testing"
+)
+
+// Test that an agent can upload and download an artifact across different steps in the same build
+func TestArtifactUploadDownload(t *testing.T) {
+	ctx := t.Context()
+
+	tc := newTestCase(t, "artifact_upload_download.yaml")
+
+	tc.startAgent()
+	build := tc.triggerBuild()
+	state := tc.waitForBuild(ctx, build)
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+}
diff --git a/internal/e2e/fixtures/artifact_upload_download.yaml b/internal/e2e/fixtures/artifact_upload_download.yaml
new file mode 100644
index 0000000000..b35f2e7f2a
--- /dev/null
+++ b/internal/e2e/fixtures/artifact_upload_download.yaml
@@ -0,0 +1,11 @@
+agents:
+  queue: {{ .queue }}
+steps:
+  - key: upload
+    commands:
+      - echo "hello world" > artifact.txt
+      - buildkite-agent artifact upload artifact.txt
+  - key: download
+    depends_on: upload
+    commands:
+      - buildkite-agent artifact download artifact.txt . && if [[ $(cat artifact.txt) == "hello world" ]]; then exit 0; else exit 1; fi

From b07af220dadaf4b9625230a1076e8efaf29cd053 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Fri, 5 Dec 2025 11:28:24 +1100
Subject: [PATCH 100/242] Bump version + changelog for v3.114.1

---
 CHANGELOG.md    | 11 +++++++++++
 version/VERSION |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 329690f34e..4169c40764 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.114.1](https://github.com/buildkite/agent/tree/v3.114.1) (2025-12-05)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.114.0...v3.114.1)
+
+### Fixed
+- Fix issue where artifacts uploaded to customer-managed s3 buckets could not be downloaded [#3607](https://github.com/buildkite/agent/pull/3607) (@moskyb)
+
+### Internal
+- Add an end-to-end testing framework! [#3611](https://github.com/buildkite/agent/pull/3611) [#3610](https://github.com/buildkite/agent/pull/3610) [#3609](https://github.com/buildkite/agent/pull/3609) [#3608](https://github.com/buildkite/agent/pull/3608) [#3606](https://github.com/buildkite/agent/pull/3606) [#3604](https://github.com/buildkite/agent/pull/3604) [#3599](https://github.com/buildkite/agent/pull/3599) (@DrJosh9000)
+- Dependency updates [#3601](https://github.com/buildkite/agent/pull/3601) [#3600](https://github.com/buildkite/agent/pull/3600) (@dependabot[bot])
+- Update MIME types [#3603](https://github.com/buildkite/agent/pull/3603) (@DrJosh9000)
+
 ## [v3.114.0](https://github.com/buildkite/agent/tree/v3.114.0) (2025-11-25)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.113.0...v3.114.0)
 
diff --git a/version/VERSION b/version/VERSION
index 0180c6b58a..dcb4257f32 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.114.0
+3.114.1

From 4e61cec051d8f6b993ac2b22435ade44bf7dbfa4 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Fri, 5 Dec 2025 14:48:39 +1100
Subject: [PATCH 101/242] Add branch_name to aws assume role session tags for
 unstable release pipeline

---
 .buildkite/pipeline.release-unstable.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.buildkite/pipeline.release-unstable.yml b/.buildkite/pipeline.release-unstable.yml
index 903fa52d27..d10cd83c7a 100644
--- a/.buildkite/pipeline.release-unstable.yml
+++ b/.buildkite/pipeline.release-unstable.yml
@@ -17,6 +17,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - branch_name
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -40,6 +41,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - branch_name
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -64,6 +66,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - branch_name
       - docker#v5.8.0:
           environment:
             - "AWS_ACCESS_KEY_ID"
@@ -125,6 +128,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - branch_name
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -190,6 +194,7 @@ steps:
                 - organization_slug
                 - organization_id
                 - pipeline_slug
+                - branch_name
           - ecr#v2.7.0:
               login: true
               account-ids: "445615400570"
@@ -217,6 +222,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
+            - branch_name
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"

From 582e1c252c840c5268f60e38d09e2dd7269cbac7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 8 Dec 2025 15:36:49 +0000
Subject: [PATCH 102/242] build(deps): bump the container-images group across 5
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `c308c24` to `02a60dc`

Updates `buildkite/agent-base` from `6c9fe8e` to `ee66745`

Updates `buildkite/agent-base` from `8f82f1a` to `057dadc`

Updates `buildkite/agent-base` from `1b4a294` to `9fd3c14`

Updates `buildkite/agent-base` from `53bf7c5` to `6c39f3f`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 8c4d66f2a0..902af97120 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:6c9fe8e1dba1c7e15395d8023943c3b8d2f07cda4a74fbc9d051d9a70db4801c
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:ee6674501b40b7fbfd1ffc847d9da104f773db019242fcd016c571929506d967
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index ac172adfe0..01ffc239b5 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:c308c2475a1cf2303c575c2c01eee63f8491507523bdf9724fa5bf32f721718d
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:02a60dc48f278c8ca70f3eba8b603a4c93ec0acbdfea7202b50f208a954dd831
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index a604929319..10edf18fd6 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:8f82f1a7ecdb81eac6ea8ef9e51e9f4f92af2141deda5b772f458f96e2b8ec51
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:057dadcc269f55136b78ca6c98a5c291eb4c5aadb742e6903fe7dfc973005895
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index 2f5786b037..ebbfa66fb6 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:1b4a294ffdab57213a6d957b9dac91fecc3d4e7b06f9cfe1e8e6ec6d76aa9257
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:9fd3c14e80076afa8576816007e07526fceaf6e9a03e4eae14aea9de8c0b90cd
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index 9762cab9e8..4335439d3e 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:53bf7c5cb541f9741b48625c79d0428c4321ca139b5cc387fd0347dc9dd58c52
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:6c39f3f4e0b17045c5f35d1a20154198705d3134418a44cf6fbb12753663438c
 ARG TARGETOS
 ARG TARGETARCH
 

From d2f78dc01c4b343347dd35f892fc24e33beb6d33 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 8 Dec 2025 15:36:55 +0000
Subject: [PATCH 103/242] build(deps): bump the container-images group across 1
 directory with 2 updates

Bumps the container-images group with 2 updates in the /.buildkite directory: docker/library/golang and golangci/golangci-lint.


Updates `docker/library/golang` from 1.24.10 to 1.24.11

Updates `golangci/golangci-lint` from v2.6-alpine to v2.7-alpine

---
updated-dependencies:
- dependency-name: docker/library/golang
  dependency-version: 1.24.11
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: container-images
- dependency-name: golangci/golangci-lint
  dependency-version: v2.7-alpine
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-compile | 2 +-
 .buildkite/Dockerfile-e2e     | 2 +-
 .buildkite/Dockerfile-lint    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index b72c8c3f27..9a07789ae0 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.10@sha256:7b13449f08287fdb53114d65bdf20eb3965e4e54997903b5cb9477df0ea37c12
+FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:cf1272dbf972a94f39a81dcb9dc243a8d2f981e5dd3b5a5c965f6d9ab9268b26
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest
diff --git a/.buildkite/Dockerfile-e2e b/.buildkite/Dockerfile-e2e
index 38ea1c5578..d60d666535 100644
--- a/.buildkite/Dockerfile-e2e
+++ b/.buildkite/Dockerfile-e2e
@@ -1 +1 @@
-FROM public.ecr.aws/docker/library/golang:1.24.10@sha256:7b13449f08287fdb53114d65bdf20eb3965e4e54997903b5cb9477df0ea37c12
+FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:cf1272dbf972a94f39a81dcb9dc243a8d2f981e5dd3b5a5c965f6d9ab9268b26
diff --git a/.buildkite/Dockerfile-lint b/.buildkite/Dockerfile-lint
index 799244b58b..0ae92eaf0a 100644
--- a/.buildkite/Dockerfile-lint
+++ b/.buildkite/Dockerfile-lint
@@ -1 +1 @@
-FROM golangci/golangci-lint:v2.6-alpine@sha256:a3abd55b17a74e36703f3d27e8183958b8bee15a1b1ffb25529192e114e0edbe
\ No newline at end of file
+FROM golangci/golangci-lint:v2.7-alpine@sha256:1e1851102b736971267400e08b3e4b2e7799c73976a998820f6f6b6b86b48343
\ No newline at end of file

From 7ab02989d234f2b1bef1541700f4ac475128fa74 Mon Sep 17 00:00:00 2001
From: Linjie Ding <linjie@anthropic.com>
Date: Mon, 8 Dec 2025 18:04:28 -0500
Subject: [PATCH 104/242] feat: add --changed-files-path option for if_changed
 evaluation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add support for providing a pre-computed list of changed files via a
newline-separated file, instead of having the agent run git commands
to determine the diff.

This is useful when:
- Working with shallow clones where git history is limited
- Using external monorepo tools (Bazel, Nx, Turborepo) that have their
  own change detection
- CI systems that already compute changed files upstream
- Non-git repositories

Usage:
  buildkite-agent pipeline upload --changed-files-path /path/to/files.txt

Or via environment variable:
  BUILDKITE_CHANGED_FILES_PATH=/path/to/files.txt

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 clicommand/pipeline_upload.go      | 113 +++++++++++++++++---------
 clicommand/pipeline_upload_test.go | 125 +++++++++++++++++++++++++++++
 2 files changed, 202 insertions(+), 36 deletions(-)

diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index b9df628264..31dd51875d 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -88,8 +88,9 @@ type PipelineUploadConfig struct {
 	RejectSecrets   bool     `cli:"reject-secrets"`
 
 	// Used for if_changed processing
-	ApplyIfChanged bool   `cli:"apply-if-changed"`
-	GitDiffBase    string `cli:"git-diff-base"`
+	ApplyIfChanged   bool   `cli:"apply-if-changed"`
+	GitDiffBase      string `cli:"git-diff-base"`
+	ChangedFilesPath string `cli:"changed-files-path"`
 
 	// Used for signing
 	JWKSFile         string `cli:"jwks-file"`
@@ -145,6 +146,11 @@ var PipelineUploadCommand = cli.Command{
 			Usage:  "Provides the base from which to find the git diff when processing ′if_changed′, e.g. origin/main. If not provided, it uses the first valid value of {origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH, origin/$BUILDKITE_PIPELINE_DEFAULT_BRANCH, origin/main}.",
 			EnvVar: "BUILDKITE_GIT_DIFF_BASE",
 		},
+		cli.StringFlag{
+			Name:   "changed-files-path",
+			Usage:  "Path to a file containing the list of changed files (newline-separated) to use for ′if_changed′ evaluation. When provided, the agent skips running git commands to determine changed files.",
+			EnvVar: "BUILDKITE_CHANGED_FILES_PATH",
+		},
 
 		// Note: changes to these environment variables need to be reflected in the environment created
 		// in the job runner. At the momenet, that's at agent/job_runner.go:500-507
@@ -300,6 +306,7 @@ var PipelineUploadCommand = cli.Command{
 				prependOriginIfNonempty("BUILDKITE_PIPELINE_DEFAULT_BRANCH"),
 				defaultGitDiffBase,
 			),
+			changedFilesPath: cfg.ChangedFilesPath,
 		}
 
 		// Process all inputs.
@@ -613,6 +620,25 @@ func (cfg *PipelineUploadConfig) parseAndInterpolate(ctx context.Context, src st
 	}
 }
 
+// readChangedFilesFromPath reads a newline-separated list of changed files from a file.
+func readChangedFilesFromPath(l logger.Logger, path string) ([]string, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("reading changed files from %q: %w", path, err)
+	}
+	lines := strings.Split(string(data), "\n")
+	// Filter out empty lines
+	changedPaths := slices.DeleteFunc(lines, func(s string) bool {
+		return strings.TrimSpace(s) == ""
+	})
+	plural := "files"
+	if len(changedPaths) == 1 {
+		plural = "file"
+	}
+	l.Info("if_changed read %d changed %s from %q", len(changedPaths), plural, path)
+	return changedPaths, nil
+}
+
 // gatherChangedFiles determines changed files in this build.
 func gatherChangedFiles(l logger.Logger, diffBase string) (changedPaths []string, err error) {
 	// Corporate needs you to find the differences between diffBase and HEAD.
@@ -721,10 +747,11 @@ func (e gitDiffError) Unwrap() error { return e.wrapped }
 // being uploaded. The `if_changed` attribute takes a glob pattern of files
 // to match. The step is skipped if the glob doesn't match any "changed files".
 type ifChangedApplicator struct {
-	enabled      bool // apply-if-changed is enabled
-	gathered     bool // the changed files have been computed?
-	diffBase     string
-	changedPaths []string
+	enabled          bool // apply-if-changed is enabled
+	gathered         bool // the changed files have been computed?
+	diffBase         string
+	changedFilesPath string // path to a file containing newline-separated changed files
+	changedPaths     []string
 }
 
 // apply applies "if_changed". If it's not enabled, it strips "if_changed"
@@ -770,40 +797,54 @@ stepsLoop:
 			continue
 		}
 
-		// If we don't know the changed paths yet, call out to Git.
+		// If we don't know the changed paths yet, either read from file or call out to Git.
 		if !ica.gathered {
-			cps, err := gatherChangedFiles(l, ica.diffBase)
-			if err != nil {
-				l.Error("Couldn't determine git diff from upstream, not skipping any pipeline steps: %v", err)
-				var exitErr *exec.ExitError
-				if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
-					// stderr came from git, which is typically human readable
-					l.Error("git: %s", exitErr.Stderr)
-				}
-				switch err := err.(type) {
-				case gitRevParseError:
-					l.Error("This could be because %q might not be a commit in the repository.\n"+
-						"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
-						err.arg,
-					)
-
-				case gitMergeBaseError:
-					l.Error("This could be because %q might not be a commit in the repository.\n"+
-						"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
-						err.diffBase,
-					)
+			var cps []string
+			var err error
 
-				case gitDiffError:
-					l.Error("This could be because the merge-base that Git found, %q, might be invalid.\n"+
-						"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
-						err.mergeBase,
-					)
+			if ica.changedFilesPath != "" {
+				// Read changed files from the provided file path.
+				cps, err = readChangedFilesFromPath(l, ica.changedFilesPath)
+				if err != nil {
+					l.Error("Couldn't read changed files from %q, not skipping any pipeline steps: %v", ica.changedFilesPath, err)
+					ica.enabled = false
+					continue stepsLoop
 				}
+			} else {
+				// Determine changed files using git.
+				cps, err = gatherChangedFiles(l, ica.diffBase)
+				if err != nil {
+					l.Error("Couldn't determine git diff from upstream, not skipping any pipeline steps: %v", err)
+					var exitErr *exec.ExitError
+					if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
+						// stderr came from git, which is typically human readable
+						l.Error("git: %s", exitErr.Stderr)
+					}
+					switch err := err.(type) {
+					case gitRevParseError:
+						l.Error("This could be because %q might not be a commit in the repository.\n"+
+							"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+							err.arg,
+						)
+
+					case gitMergeBaseError:
+						l.Error("This could be because %q might not be a commit in the repository.\n"+
+							"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+							err.diffBase,
+						)
+
+					case gitDiffError:
+						l.Error("This could be because the merge-base that Git found, %q, might be invalid.\n"+
+							"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+							err.mergeBase,
+						)
+					}
 
-				// Because changed files couldn't be determined, we switch into
-				// disabled mode.
-				ica.enabled = false
-				continue stepsLoop
+					// Because changed files couldn't be determined, we switch into
+					// disabled mode.
+					ica.enabled = false
+					continue stepsLoop
+				}
 			}
 
 			// The changed files are now known.
diff --git a/clicommand/pipeline_upload_test.go b/clicommand/pipeline_upload_test.go
index 32bbf9a391..fd791b2b14 100644
--- a/clicommand/pipeline_upload_test.go
+++ b/clicommand/pipeline_upload_test.go
@@ -2,6 +2,7 @@ package clicommand
 
 import (
 	"context"
+	"os"
 	"runtime"
 	"strings"
 	"testing"
@@ -741,3 +742,127 @@ func TestIfChangedApplicator_WeirdPipeline(t *testing.T) {
 		t.Errorf("after ica.apply(l, steps) (-got, +want):\n%s", diff)
 	}
 }
+
+func TestReadChangedFilesFromPath(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		content string
+		want    []string
+	}{
+		{
+			name:    "single file",
+			content: "foo/bar.go\n",
+			want:    []string{"foo/bar.go"},
+		},
+		{
+			name:    "multiple files",
+			content: "foo/bar.go\nsrc/main.go\nREADME.md\n",
+			want:    []string{"foo/bar.go", "src/main.go", "README.md"},
+		},
+		{
+			name:    "empty lines filtered",
+			content: "foo/bar.go\n\nsrc/main.go\n\n",
+			want:    []string{"foo/bar.go", "src/main.go"},
+		},
+		{
+			name:    "no trailing newline",
+			content: "foo/bar.go\nsrc/main.go",
+			want:    []string{"foo/bar.go", "src/main.go"},
+		},
+		{
+			name:    "empty file",
+			content: "",
+			want:    []string{},
+		},
+		{
+			name:    "only newlines",
+			content: "\n\n\n",
+			want:    []string{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			tmpFile, err := os.CreateTemp("", "changed-files-*.txt")
+			if err != nil {
+				t.Fatalf("creating temp file: %v", err)
+			}
+			defer os.Remove(tmpFile.Name())
+
+			if _, err := tmpFile.WriteString(test.content); err != nil {
+				t.Fatalf("writing to temp file: %v", err)
+			}
+			tmpFile.Close()
+
+			l := logger.NewBuffer()
+			got, err := readChangedFilesFromPath(l, tmpFile.Name())
+			if err != nil {
+				t.Fatalf("readChangedFilesFromPath() error = %v", err)
+			}
+
+			if diff := cmp.Diff(got, test.want); diff != "" {
+				t.Errorf("readChangedFilesFromPath() diff (-got +want):\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestIfChangedApplicator_WithChangedFilesPath(t *testing.T) {
+	t.Parallel()
+
+	// Create a temp file with changed files
+	tmpFile, err := os.CreateTemp("", "changed-files-*.txt")
+	if err != nil {
+		t.Fatalf("creating temp file: %v", err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	if _, err := tmpFile.WriteString("foo/README.md\nbar/test.go\n"); err != nil {
+		t.Fatalf("writing to temp file: %v", err)
+	}
+	tmpFile.Close()
+
+	steps := pipeline.Steps{
+		&pipeline.CommandStep{
+			Command: "runs when foo changes",
+			RemainingFields: map[string]any{
+				"if_changed": "foo/**",
+			},
+		},
+		&pipeline.CommandStep{
+			Command: "runs when qux changes",
+			RemainingFields: map[string]any{
+				"if_changed": "qux/**",
+			},
+		},
+	}
+
+	want := pipeline.Steps{
+		&pipeline.CommandStep{
+			Command:         "runs when foo changes",
+			RemainingFields: map[string]any{},
+		},
+		&pipeline.CommandStep{
+			Command: "runs when qux changes",
+			RemainingFields: map[string]any{
+				"skip": ifChangedSkippedMsg,
+			},
+		},
+	}
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(i int) { t.Errorf("exitFn(%d) invoked", i) })
+
+	ica := &ifChangedApplicator{
+		enabled:          true,
+		changedFilesPath: tmpFile.Name(),
+	}
+
+	ica.apply(l, steps)
+	if diff := cmp.Diff(steps, want); diff != "" {
+		t.Errorf("after ica.apply(l, steps) (-got, +want):\n%s", diff)
+	}
+}

From 1e0f13dc0fb369b0584f83991fcd6650d8f6d695 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Tue, 9 Dec 2025 11:46:07 +1100
Subject: [PATCH 105/242] chore(deps): bump zstash to 0.5.1 and log parts and
 concurrency

---
 go.mod                  |  76 ++++++++++----------
 go.sum                  | 156 ++++++++++++++++++++--------------------
 internal/cache/cache.go |   4 ++
 3 files changed, 120 insertions(+), 116 deletions(-)

diff --git a/go.mod b/go.mod
index d7e419b990..0e0a7eddc8 100644
--- a/go.mod
+++ b/go.mod
@@ -11,13 +11,13 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
-	github.com/aws/aws-sdk-go-v2 v1.40.0
-	github.com/aws/aws-sdk-go-v2/config v1.32.2
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12
+	github.com/aws/aws-sdk-go-v2 v1.41.0
+	github.com/aws/aws-sdk-go-v2/config v1.32.4
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.14
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0
 	github.com/aws/aws-sdk-go-v2/service/kms v1.49.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
@@ -26,7 +26,7 @@ require (
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
-	github.com/buildkite/zstash v0.5.0
+	github.com/buildkite/zstash v0.5.1
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
@@ -51,17 +51,17 @@ require (
 	go.opentelemetry.io/contrib/propagators/b3 v1.38.0
 	go.opentelemetry.io/contrib/propagators/jaeger v1.38.0
 	go.opentelemetry.io/contrib/propagators/ot v1.38.0
-	go.opentelemetry.io/otel v1.38.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
+	go.opentelemetry.io/otel v1.39.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
-	go.opentelemetry.io/otel/sdk v1.38.0
-	go.opentelemetry.io/otel/trace v1.38.0
-	golang.org/x/crypto v0.45.0
-	golang.org/x/net v0.47.0
+	go.opentelemetry.io/otel/sdk v1.39.0
+	go.opentelemetry.io/otel/trace v1.39.0
+	golang.org/x/crypto v0.46.0
+	golang.org/x/net v0.48.0
 	golang.org/x/oauth2 v0.33.0
-	golang.org/x/sync v0.18.0
-	golang.org/x/sys v0.38.0
-	golang.org/x/term v0.37.0
+	golang.org/x/sync v0.19.0
+	golang.org/x/sys v0.39.0
+	golang.org/x/term v0.38.0
 	google.golang.org/api v0.256.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
@@ -95,20 +95,20 @@ require (
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
@@ -141,7 +141,7 @@ require (
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.18.1 // indirect
+	github.com/klauspost/compress v1.18.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.3 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
@@ -189,23 +189,23 @@ require (
 	go.opentelemetry.io/collector/semconv v0.125.0 // indirect
 	go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
 	go.opentelemetry.io/otel/log v0.11.0 // indirect
-	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.8.0 // indirect
+	go.opentelemetry.io/otel/metric v1.39.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
-	golang.org/x/mod v0.29.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/mod v0.30.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.38.0 // indirect
+	golang.org/x/tools v0.39.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect
-	google.golang.org/grpc v1.76.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
+	google.golang.org/grpc v1.77.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
diff --git a/go.sum b/go.sum
index c6b08c6a42..df3edbeee7 100644
--- a/go.sum
+++ b/go.sum
@@ -76,48 +76,48 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
-github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
-github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk=
-github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM=
+github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
+github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
+github.com/aws/aws-sdk-go-v2/config v1.32.4 h1:gl+DxVuadpkYoaDcWllZqLkhGEbvwyqgNVRTmlaf5PI=
+github.com/aws/aws-sdk-go-v2/config v1.32.4/go.mod h1:MBUp9Og/bzMmQHjMwace4aJfyvJeadzXjoTcR/SxLV0=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.4 h1:KeIZxHVbGWRLhPvhdPbbi/DtFBHNKm6OsVDuiuFefdQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.4/go.mod h1:Smw5n0nCZE9PeFEguofdXyt8kUC4JNrkDTfBOioPhFA=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.14 h1:Ml4JmbZDi48OiAQx7CUst0ZO48ftbfNsWMEYiuhu06Q=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.14/go.mod h1:aBxw6KN/hqD598VrxXR6RXgNSWC3q0/aT14VXHD/MSo=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0 h1:ymusjrsOjrcVBQNQXYFIQEHJIJ17/m+VoDSmWIMjGe0=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0/go.mod h1:QrV+/GjhSrJh6MRRuTO6ZEg4M2I0nwPakf0lZHSrE1o=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
 github.com/aws/aws-sdk-go-v2/service/kms v1.49.1 h1:U0asSZ3ifpuIehDPkRI2rxHbmFUMplDA2VeR9Uogrmw=
 github.com/aws/aws-sdk-go-v2/service/kms v1.49.1/go.mod h1:NZo9WJqQ0sxQ1Yqu1IwCHQFQunTms2MlVgejg16S1rY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.5/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCdYW9ltGQYrFWg8XdkGDd8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1 h1:5FhzzN6JmlGQF6c04kDIb5KNGm6KnNdLISNrfivIhHg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.4 h1:YCu/iAhQer8WZ66lldyKkpvMyv+HkPufMa4dyT6wils=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.4/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
 github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
 github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -144,8 +144,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
-github.com/buildkite/zstash v0.5.0 h1:e70mf8U2EjEB1eixXR78s6bsLgfo6bWLisVlRv58wCI=
-github.com/buildkite/zstash v0.5.0/go.mod h1:h70JfAEa2Ys1GDQQ6CNoKIMfMgJ0LZkNmQnzK710PHQ=
+github.com/buildkite/zstash v0.5.1 h1:0/ujF6VzcuZIOyE7O7+kUclEKce+WKtlft//OiA3MZY=
+github.com/buildkite/zstash v0.5.1/go.mod h1:zAX0o9prXpjxsPAiPxi3sK4DNIC8TTHJ2j5Cp00ARLQ=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf h1:yxlp0s+Sge9UsKEK0Bsvjiopb9XRk+vxylmZ9eGBfm8=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
@@ -259,8 +259,8 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
+github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -457,26 +457,26 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.38.0 h1:nXGeLvT1QtCAhkASkP/ksj
 go.opentelemetry.io/contrib/propagators/jaeger v1.38.0/go.mod h1:oMvOXk78ZR3KEuPMBgp/ThAMDy9ku/eyUVztr+3G6Wo=
 go.opentelemetry.io/contrib/propagators/ot v1.38.0 h1:k4gSyyohaDXI8F9BDXYC3uO2vr5sRNeQFMsN9Zn0EoI=
 go.opentelemetry.io/contrib/propagators/ot v1.38.0/go.mod h1:2hDsuiHRO39SRUMhYGqmj64z/IuMRoxE4bBSFR82Lo8=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
+go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
+go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 h1:in9O8ESIOlwJAEGTkkf34DesGRAc/Pn8qJ7k3r/42LM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0/go.mod h1:Rp0EXBm5tfnv0WL+ARyO/PHBEaEAT8UUHQ6AGJcSq6c=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
 go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y=
 go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
-go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
-go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
-go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
-go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
-go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
-go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE=
-go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0=
+go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
+go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
+go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
+go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
+go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
+go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
+go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
+go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
@@ -491,30 +491,30 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo=
 golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
-golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -528,15 +528,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
+golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -544,8 +544,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -560,12 +560,12 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
-google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 h1:8XJ4pajGwOlasW+L13MnEGA8W4115jJySQtVfS2/IBU=
-google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4/go.mod h1:NnuHhy+bxcg30o7FnVAZbXsPHUDQ9qKWAQKCD7VxFtk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
-google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
-google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
+google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 h1:h96ji92t9eXbPvSWhJ+lrPWetHiQNYlt48JKRO09NFA=
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index a53aa2d520..6a1cb75bcd 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -177,6 +177,8 @@ func restoreWithClient(ctx context.Context, l logger.Logger, client CacheClient,
 				logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
 				logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
 				logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+				logger.IntField("part_count", result.Transfer.PartCount),
+				logger.IntField("concurrency", result.Transfer.Concurrency),
 			).Info("Cache restored")
 		default:
 			l.WithFields(
@@ -208,6 +210,8 @@ func saveWithClient(ctx context.Context, l logger.Logger, client CacheClient, ca
 				logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
 				logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
 				logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+				logger.IntField("part_count", result.Transfer.PartCount),
+				logger.IntField("concurrency", result.Transfer.Concurrency),
 			).Info("Cache created")
 		default:
 			l.WithFields(

From 21dec5e5f03dcc789df276cff739ca41666a0f0c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 9 Dec 2025 05:03:51 +0000
Subject: [PATCH 106/242] build(deps): bump golang.org/x/oauth2

Bumps the golang-x group with 1 update in the / directory: [golang.org/x/oauth2](https://github.com/golang/oauth2).


Updates `golang.org/x/oauth2` from 0.33.0 to 0.34.0
- [Commits](https://github.com/golang/oauth2/compare/v0.33.0...v0.34.0)

---
updated-dependencies:
- dependency-name: golang.org/x/oauth2
  dependency-version: 0.34.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 0e0a7eddc8..a675f27090 100644
--- a/go.mod
+++ b/go.mod
@@ -58,7 +58,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.39.0
 	golang.org/x/crypto v0.46.0
 	golang.org/x/net v0.48.0
-	golang.org/x/oauth2 v0.33.0
+	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.39.0
 	golang.org/x/term v0.38.0
diff --git a/go.sum b/go.sum
index df3edbeee7..6825ba5d35 100644
--- a/go.sum
+++ b/go.sum
@@ -507,8 +507,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
 golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
-golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo=
-golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
+golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

From 0c8b949fd33e874ee768196cfbaca0e54dda85f9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 9 Dec 2025 05:06:31 +0000
Subject: [PATCH 107/242] build(deps): bump the cloud-providers group across 1
 directory with 3 updates

Bumps the cloud-providers group with 3 updates in the / directory: [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) and [google.golang.org/api](https://github.com/googleapis/google-api-go-client).


Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.275.0 to 1.276.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.275.0...service/ec2/v1.276.0)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.49.1 to 1.49.3
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ssm/v1.49.1...service/ssm/v1.49.3)

Updates `google.golang.org/api` from 0.256.0 to 0.257.0
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.256.0...v0.257.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.276.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.49.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: google.golang.org/api
  dependency-version: 0.257.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod |  6 +++---
 go.sum | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index 0e0a7eddc8..6b7589018b 100644
--- a/go.mod
+++ b/go.mod
@@ -15,8 +15,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.32.4
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.14
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0
-	github.com/aws/aws-sdk-go-v2/service/kms v1.49.1
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.0
+	github.com/aws/aws-sdk-go-v2/service/kms v1.49.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
@@ -62,7 +62,7 @@ require (
 	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.39.0
 	golang.org/x/term v0.38.0
-	google.golang.org/api v0.256.0
+	google.golang.org/api v0.257.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
diff --git a/go.sum b/go.sum
index df3edbeee7..e29428c43a 100644
--- a/go.sum
+++ b/go.sum
@@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0 h1:ymusjrsOjrcVBQNQXYFIQEHJIJ17/m+VoDSmWIMjGe0=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.275.0/go.mod h1:QrV+/GjhSrJh6MRRuTO6ZEg4M2I0nwPakf0lZHSrE1o=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.0 h1:EXwbpkq/tsz1lHI5QRoXjnkZRKgW0Xa+mPSv6Dz/9N0=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.0/go.mod h1:Wg68QRgy2gEGGdmTPU/UbVpdv8sM14bUZmF64KFwAsY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
@@ -106,8 +106,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.1 h1:U0asSZ3ifpuIehDPkRI2rxHbmFUMplDA2VeR9Uogrmw=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.1/go.mod h1:NZo9WJqQ0sxQ1Yqu1IwCHQFQunTms2MlVgejg16S1rY=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.3 h1:FtjNR2BaMf/Wu3Um8RNSAfi9An5bEYxJU6f4rLOqSLU=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.3/go.mod h1:HO31s0qt0lso/ADvZQyzKs8js/ku0fMHsfyXW8OPVYc=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1 h1:5FhzzN6JmlGQF6c04kDIb5KNGm6KnNdLISNrfivIhHg=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
@@ -554,8 +554,8 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI=
-google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964=
+google.golang.org/api v0.257.0 h1:8Y0lzvHlZps53PEaw+G29SsQIkuKrumGWs9puiexNAA=
+google.golang.org/api v0.257.0/go.mod h1:4eJrr+vbVaZSqs7vovFd1Jb/A6ml6iw2e6FBYf3GAO4=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=

From 2187dd7d75eda80133f9bd0b21217c20edf8e16f Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 18 Nov 2025 15:25:12 +1100
Subject: [PATCH 108/242] Delete pool

---
 internal/artifact/downloader.go | 112 ++++++++++++++++++++++++--------
 pool/BUILD.bazel                |   8 ---
 pool/pool.go                    |  64 ------------------
 3 files changed, 85 insertions(+), 99 deletions(-)
 delete mode 100644 pool/BUILD.bazel
 delete mode 100644 pool/pool.go

diff --git a/internal/artifact/downloader.go b/internal/artifact/downloader.go
index be876069ae..53193690f5 100644
--- a/internal/artifact/downloader.go
+++ b/internal/artifact/downloader.go
@@ -8,12 +8,12 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/buildkite/agent/v3/api"
 	"github.com/buildkite/agent/v3/internal/agenthttp"
 	"github.com/buildkite/agent/v3/logger"
-	"github.com/buildkite/agent/v3/pool"
 )
 
 type DownloaderConfig struct {
@@ -83,43 +83,101 @@ func (a *Downloader) Download(ctx context.Context) error {
 
 	a.logger.Info("Found %d artifacts. Starting to download to: %s", artifactCount, destination)
 
-	p := pool.New(pool.MaxConcurrencyLimit)
-	errors := []error{}
 	s3Clients, err := a.generateS3Clients(ctx, artifacts)
 	if err != nil {
-		return fmt.Errorf("failed to generate S3 clients for artifact upload: %w", err)
+		return fmt.Errorf("failed to generate S3 clients for artifact download: %w", err)
 	}
 
-	for _, artifact := range artifacts {
-		p.Spawn(func() {
-			// Convert windows paths to slashes, otherwise we get a literal
-			// download of "dir/dir/file" vs sub-directories on non-windows agents
-			path := artifact.Path
-			if runtime.GOOS != "windows" {
-				path = strings.ReplaceAll(path, `\`, `/`)
+	// A goroutine to collect download errors into a slice.
+	errorsCh := make(chan error)
+	// errorsOutCh is buffered (1) in order to let the error collector finish,
+	// even if Download has returned and nothing is receiving from the channel
+	// anymore.
+	errorsOutCh := make(chan []error, 1)
+	go func() {
+		var errors []error
+		for err := range errorsCh {
+			errors = append(errors, err)
+		}
+		errorsOutCh <- errors
+	}()
+
+	// A bunch of worker goroutines. Start the smaller of:
+	// - GOMAXPROCS (often equal to NumCPU) times 10 (historic choice; downloads
+	//   are not likely to be bounded by CPU)
+	// - the number of artifacts to download.
+	var wg sync.WaitGroup
+	artifactsCh := make(chan *api.Artifact)
+	for range min(10*runtime.GOMAXPROCS(0), len(artifacts)) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			for {
+				var artifact *api.Artifact
+				var open bool
+				select {
+				case artifact, open = <-artifactsCh:
+					if !open {
+						return
+					}
+					// continue below
+
+				case <-ctx.Done():
+					return
+				}
+
+				// Convert windows paths to slashes, otherwise we get a literal
+				// download of "dir/dir/file" vs sub-directories on non-windows agents
+				path := artifact.Path
+				if runtime.GOOS != "windows" {
+					path = strings.ReplaceAll(path, `\`, `/`)
+				}
+
+				dler := a.createDownloader(artifact, path, destination, s3Clients)
+
+				if err := dler.Start(ctx); err != nil {
+					a.logger.Error("Failed to download artifact: %s", err)
+					select {
+					case errorsCh <- err:
+						// error sent
+					case <-ctx.Done():
+						return
+					}
+				}
 			}
+		}()
+	}
 
-			dler := a.createDownloader(artifact, path, destination, s3Clients)
+	// Send the artifacts to the workers then signal completion by closing the
+	// channel.
+	for _, artifact := range artifacts {
+		select {
+		case artifactsCh <- artifact:
+			// Artifact being downloaded
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+	close(artifactsCh)
 
-			// If the downloaded encountered an error, lock
-			// the pool, collect it, then unlock the pool
-			// again.
-			if err := dler.Start(ctx); err != nil {
-				a.logger.Error("Failed to download artifact: %s", err)
+	// Wait for downloads to complete.
+	wg.Wait()
 
-				p.Lock()
-				errors = append(errors, err)
-				p.Unlock()
-			}
-		})
-	}
+	// All workers have returned, so all errors have been sent, so close the
+	// error channel.
+	close(errorsCh)
 
-	p.Wait()
+	// Read the slice of all errors from the error collector.
+	select {
+	case errors := <-errorsOutCh:
+		if len(errors) > 0 {
+			return fmt.Errorf("There were errors with downloading some of the artifacts")
+		}
 
-	if len(errors) > 0 {
-		return fmt.Errorf("There were errors with downloading some of the artifacts")
+	case <-ctx.Done():
+		return ctx.Err()
 	}
-
 	return nil
 }
 
diff --git a/pool/BUILD.bazel b/pool/BUILD.bazel
deleted file mode 100644
index f97c16100b..0000000000
--- a/pool/BUILD.bazel
+++ /dev/null
@@ -1,8 +0,0 @@
-load("@rules_go//go:def.bzl", "go_library")
-
-go_library(
-    name = "pool",
-    srcs = ["pool.go"],
-    importpath = "github.com/buildkite/agent/v3/pool",
-    visibility = ["//visibility:public"],
-)
diff --git a/pool/pool.go b/pool/pool.go
deleted file mode 100644
index a40b859fcb..0000000000
--- a/pool/pool.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Package pool provides a worker pool that enforces an upper limit on
-// concurrent workers.
-//
-// It is intended for internal use by buildkite-agent only.
-package pool
-
-import (
-	"runtime"
-	"sync"
-)
-
-type Pool struct {
-	wg         *sync.WaitGroup
-	completion chan bool
-	m          sync.Mutex
-}
-
-const (
-	MaxConcurrencyLimit = -1
-)
-
-func New(concurrencyLimit int) *Pool {
-	if concurrencyLimit == MaxConcurrencyLimit {
-		// Completely arbitrary. Most of the time we could probably have unbounded concurrency, but the situations where we use
-		// this pool is basically just S3 uploading and downloading, so this number is kind of a proxy for "What won't rate limit us"
-		// TODO: Make artifact uploads and downloads gracefully handle rate limiting, remove this pool entirely, and use unbounded concurrency via a WaitGroup
-		concurrencyLimit = runtime.NumCPU() * 10
-	}
-
-	wg := sync.WaitGroup{}
-	completionChan := make(chan bool, concurrencyLimit)
-
-	for range concurrencyLimit {
-		completionChan <- true
-	}
-
-	return &Pool{&wg, completionChan, sync.Mutex{}}
-}
-
-func (pool *Pool) Spawn(job func()) {
-	<-pool.completion
-	pool.wg.Add(1)
-
-	go func() {
-		defer func() {
-			pool.completion <- true
-			pool.wg.Done()
-		}()
-
-		job()
-	}()
-}
-
-func (pool *Pool) Lock() {
-	pool.m.Lock()
-}
-
-func (pool *Pool) Unlock() {
-	pool.m.Unlock()
-}
-
-func (pool *Pool) Wait() {
-	pool.wg.Wait()
-}

From bbb3748dff567cb728dd2c99c64e6342cf8d3129 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 18 Nov 2025 15:25:12 +1100
Subject: [PATCH 109/242] Bazel updates

- bazel mod tidy
- bazel run :gazelle
---
 MODULE.bazel                         |    8 +-
 MODULE.bazel.lock                    | 1175 +++++++++++++++++++++++++-
 agent/BUILD.bazel                    |   17 +-
 agent/integration/BUILD.bazel        |    2 +
 api/BUILD.bazel                      |    2 +
 clicommand/BUILD.bazel               |   13 +
 internal/artifact/BUILD.bazel        |   21 +-
 internal/awslib/BUILD.bazel          |    7 +-
 internal/cache/BUILD.bazel           |   29 +
 internal/e2e/BUILD.bazel             |    8 +
 internal/job/BUILD.bazel             |   11 +-
 internal/job/githttptest/BUILD.bazel |    8 +
 internal/job/hook/BUILD.bazel        |    2 -
 internal/job/integration/BUILD.bazel |    2 +
 internal/ptr/BUILD.bazel             |    8 +
 internal/race/BUILD.bazel            |   11 +
 internal/redact/BUILD.bazel          |    5 +-
 internal/replacer/BUILD.bazel        |    1 +
 internal/secrets/BUILD.bazel         |   27 +
 internal/self/BUILD.bazel            |    8 +
 internal/shell/BUILD.bazel           |    5 +-
 jobapi/BUILD.bazel                   |    1 -
 kubernetes/BUILD.bazel               |   15 -
 23 files changed, 1310 insertions(+), 76 deletions(-)
 create mode 100644 internal/cache/BUILD.bazel
 create mode 100644 internal/e2e/BUILD.bazel
 create mode 100644 internal/job/githttptest/BUILD.bazel
 create mode 100644 internal/ptr/BUILD.bazel
 create mode 100644 internal/race/BUILD.bazel
 create mode 100644 internal/secrets/BUILD.bazel
 create mode 100644 internal/self/BUILD.bazel

diff --git a/MODULE.bazel b/MODULE.bazel
index cf4de82440..a0c528acab 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -1,8 +1,8 @@
-bazel_dep(name = "gazelle", version = "0.40.0")
-bazel_dep(name = "rules_go", version = "0.50.1")
+bazel_dep(name = "gazelle", version = "0.47.0")
+bazel_dep(name = "rules_go", version = "0.59.0")
 
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
-go_sdk.download(version = "1.23.7")
+go_sdk.download(version = "1.24.10")
 
 go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//:go.mod")
@@ -14,4 +14,4 @@ go_deps.gazelle_override(
 )
 
 # All *direct* dependencies are required to be listed explicitly
-use_repo(go_deps, "com_github_aws_aws_sdk_go", "com_github_aws_aws_sdk_go_v2", "com_github_aws_aws_sdk_go_v2_config", "com_github_aws_aws_sdk_go_v2_feature_ec2_imds", "com_github_aws_aws_sdk_go_v2_service_kms", "com_github_azure_azure_sdk_for_go_sdk_azidentity", "com_github_azure_azure_sdk_for_go_sdk_storage_azblob", "com_github_brunoscheufler_aws_ecs_metadata_go", "com_github_buildkite_bintest_v3", "com_github_buildkite_go_pipeline", "com_github_buildkite_interpolate", "com_github_buildkite_roko", "com_github_buildkite_shellwords", "com_github_creack_pty", "com_github_datadog_datadog_go_v5", "com_github_denisbrodbeck_machineid", "com_github_dustin_go_humanize", "com_github_dustinkirkland_golang_petname", "com_github_gliderlabs_ssh", "com_github_go_chi_chi_v5", "com_github_gofrs_flock", "com_github_google_go_cmp", "com_github_google_go_querystring", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_khan_genqlient", "com_github_lestrrat_go_jwx_v2", "com_github_mattn_go_zglob", "com_github_oleiade_reflections", "com_github_opentracing_opentracing_go", "com_github_pborman_uuid", "com_github_puzpuzpuz_xsync_v2", "com_github_qri_io_jsonschema", "com_github_stretchr_testify", "com_github_urfave_cli", "com_google_cloud_go_compute_metadata", "dev_drjosh_zzglob", "in_gopkg_datadog_dd_trace_go_v1", "in_gopkg_yaml_v3", "io_opentelemetry_go_contrib_propagators_aws", "io_opentelemetry_go_contrib_propagators_b3", "io_opentelemetry_go_contrib_propagators_jaeger", "io_opentelemetry_go_contrib_propagators_ot", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlptrace", "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_trace", "org_golang_google_api", "org_golang_x_crypto", "org_golang_x_net", "org_golang_x_oauth2", "org_golang_x_sys", "org_golang_x_term", "tools_gotest_v3")
+use_repo(go_deps, "cc_mvdan_gofumpt", "com_github_aws_aws_sdk_go_v2", "com_github_aws_aws_sdk_go_v2_config", "com_github_aws_aws_sdk_go_v2_feature_ec2_imds", "com_github_aws_aws_sdk_go_v2_feature_s3_manager", "com_github_aws_aws_sdk_go_v2_service_ec2", "com_github_aws_aws_sdk_go_v2_service_kms", "com_github_aws_aws_sdk_go_v2_service_s3", "com_github_aws_smithy_go", "com_github_azure_azure_sdk_for_go_sdk_azidentity", "com_github_azure_azure_sdk_for_go_sdk_storage_azblob", "com_github_brunoscheufler_aws_ecs_metadata_go", "com_github_buildkite_bintest_v3", "com_github_buildkite_go_buildkite_v4", "com_github_buildkite_go_pipeline", "com_github_buildkite_interpolate", "com_github_buildkite_roko", "com_github_buildkite_shellwords", "com_github_buildkite_test_engine_client", "com_github_buildkite_zstash", "com_github_creack_pty", "com_github_datadog_datadog_go_v5", "com_github_denisbrodbeck_machineid", "com_github_dustin_go_humanize", "com_github_dustinkirkland_golang_petname", "com_github_gliderlabs_ssh", "com_github_go_chi_chi_v5", "com_github_gofrs_flock", "com_github_google_go_cmp", "com_github_google_go_querystring", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_khan_genqlient", "com_github_lestrrat_go_jwx_v2", "com_github_oleiade_reflections", "com_github_opentracing_opentracing_go", "com_github_pborman_uuid", "com_github_prometheus_client_golang", "com_github_puzpuzpuz_xsync_v2", "com_github_qri_io_jsonschema", "com_github_stretchr_testify", "com_github_urfave_cli", "com_google_cloud_go_compute_metadata", "dev_drjosh_zzglob", "in_gopkg_datadog_dd_trace_go_v1", "in_gopkg_yaml_v3", "io_opentelemetry_go_contrib_propagators_aws", "io_opentelemetry_go_contrib_propagators_b3", "io_opentelemetry_go_contrib_propagators_jaeger", "io_opentelemetry_go_contrib_propagators_ot", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc", "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_trace", "org_golang_google_api", "org_golang_x_crypto", "org_golang_x_net", "org_golang_x_oauth2", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term", "tools_gotest_gotestsum", "tools_gotest_v3")
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
index 4442f94cb6..dce92405ae 100644
--- a/MODULE.bazel.lock
+++ b/MODULE.bazel.lock
@@ -4,84 +4,147 @@
     "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497",
     "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2",
     "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589",
-    "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/source.json": "7e3a9adf473e9af076ae485ed649d5641ad50ec5c11718103f34de03170d94ad",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915",
+    "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed",
+    "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/source.json": "9be551b8d4e3ef76875c0d744b5d6a504a27e3ae67bc6b28f46415fd2d2957da",
     "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef",
     "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862",
     "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b",
     "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd",
     "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8",
+    "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d",
+    "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d",
     "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a",
-    "https://bcr.bazel.build/modules/bazel_features/1.18.0/source.json": "cde886d88c8164b50b9b97dba7c0a64ca24d257b72ca3a2fcb06bee1fdb47ee4",
+    "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58",
+    "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d",
+    "https://bcr.bazel.build/modules/bazel_features/1.28.0/source.json": "16a3fc5b4483cb307643791f5a4b7365fa98d2e70da7c378cdbde55f0c0b32cf",
     "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
     "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
     "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e",
     "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686",
     "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a",
     "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651",
     "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138",
     "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917",
-    "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/source.json": "082ed5f9837901fada8c68c2f3ddc958bb22b6d654f71dd73f3df30d45d4b749",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953",
     "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84",
     "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
     "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8",
     "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350",
     "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a",
     "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0",
-    "https://bcr.bazel.build/modules/gazelle/0.40.0/MODULE.bazel": "42ba5378ebe845fca43989a53186ab436d956db498acde790685fe0e8f9c6146",
-    "https://bcr.bazel.build/modules/gazelle/0.40.0/source.json": "1e5ef6e4d8b9b6836d93273c781e78ff829ea2e077afef7a57298040fa4f010a",
+    "https://bcr.bazel.build/modules/gazelle/0.47.0/MODULE.bazel": "b61bb007c4efad134aa30ee7f4a8e2a39b22aa5685f005edaa022fbd1de43ebc",
+    "https://bcr.bazel.build/modules/gazelle/0.47.0/source.json": "aeb2e5df14b7fb298625d75d08b9c65bdb0b56014c5eb89da9e5dd0572280ae6",
+    "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb",
     "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4",
-    "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206",
+    "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6",
+    "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4",
+    "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f",
+    "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075",
+    "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d",
+    "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902",
+    "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4",
+    "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85",
     "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5",
-    "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5",
     "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee",
     "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37",
     "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615",
     "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814",
+    "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d",
     "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc",
+    "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580",
+    "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96",
     "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7",
-    "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b",
+    "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c",
+    "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d",
+    "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c",
+    "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/source.json": "cfbee3381201f20e35c304041b4abb3b793e66c9b1829b5478d033ad4a5e3aef",
     "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0",
     "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573",
     "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858",
+    "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e",
+    "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022",
+    "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206",
+    "https://bcr.bazel.build/modules/re2/2023-09-01/source.json": "e044ce89c2883cd957a2969a43e79f7752f9656f6b20050b62f90ede21ec6eb4",
+    "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8",
+    "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e",
     "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a",
     "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f",
     "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
     "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
-    "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430",
+    "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8",
+    "https://bcr.bazel.build/modules/rules_cc/0.1.5/source.json": "4bb4fed7f5499775d495739f785a5494a1f854645fa1bac5de131264f5acdf01",
+    "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6",
+    "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8",
+    "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e",
     "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8",
     "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270",
     "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd",
-    "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0",
-    "https://bcr.bazel.build/modules/rules_go/0.50.1/source.json": "205765fd30216c70321f84c9a967267684bdc74350af3f3c46c857d9f80a4fa2",
+    "https://bcr.bazel.build/modules/rules_go/0.53.0/MODULE.bazel": "a4ed760d3ac0dbc0d7b967631a9a3fd9100d28f7d9fcf214b4df87d4bfff5f9a",
+    "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae",
+    "https://bcr.bazel.build/modules/rules_go/0.59.0/source.json": "1df17bb7865cfc029492c30163cee891d0dd8658ea0d5bfdf252c4b6db5c1ef6",
     "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
+    "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86",
+    "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31",
+    "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a",
+    "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6",
+    "https://bcr.bazel.build/modules/rules_java/7.12.2/source.json": "b0890f9cda8ff1b8e691a3ac6037b5c14b7fd4134765a3946b89f31ea02e5884",
+    "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab",
+    "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
     "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1",
-    "https://bcr.bazel.build/modules/rules_java/7.6.5/source.json": "a805b889531d1690e3c72a7a7e47a870d00323186a9904b36af83aa3d053ee8d",
     "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
-    "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/source.json": "a075731e1b46bc8425098512d038d416e966ab19684a10a34f4741295642fc35",
+    "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909",
+    "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036",
+    "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0",
+    "https://bcr.bazel.build/modules/rules_jvm_external/6.3/source.json": "6f5f5a5a4419ae4e37c35a5bb0a6ae657ed40b7abc5a5189111b47fcebe43197",
+    "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3",
+    "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5",
     "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0",
     "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d",
     "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c",
     "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb",
     "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc",
-    "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c",
+    "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff",
+    "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a",
     "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06",
     "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7",
     "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f",
-    "https://bcr.bazel.build/modules/rules_proto/6.0.0/source.json": "de77e10ff0ab16acbf54e6b46eecd37a99c5b290468ea1aee6e95eb1affdaed7",
+    "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73",
+    "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2",
+    "https://bcr.bazel.build/modules/rules_proto/7.0.2/source.json": "1e5e7260ae32ef4f2b52fd1d0de8d03b606a44c91b694d2f1afb1d3b28a48ce1",
     "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f",
     "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7",
-    "https://bcr.bazel.build/modules/rules_python/0.22.1/source.json": "57226905e783bae7c37c2dd662be078728e48fa28ee4324a7eabcafb5a43d014",
+    "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300",
+    "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382",
+    "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed",
+    "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58",
+    "https://bcr.bazel.build/modules/rules_python/0.31.0/source.json": "a41c836d4065888eef4377f2f27b6eea0fedb9b5adb1bab1970437373fe90dc7",
     "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c",
     "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c",
-    "https://bcr.bazel.build/modules/rules_shell/0.2.0/source.json": "7f27af3c28037d9701487c4744b5448d26537cc66cdef0d8df7ae85411f8de95",
+    "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b",
+    "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3",
     "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8",
-    "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29",
+    "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c",
+    "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c",
     "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
-    "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459",
     "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
     "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27",
     "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79",
-    "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d"
+    "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d",
+    "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198"
   },
   "selectedYankedVersions": {},
   "moduleExtensions": {
@@ -113,21 +176,1079 @@
         ]
       }
     },
-    "@@platforms//host:extension.bzl%host_platform": {
+    "@@rules_kotlin~//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": {
       "general": {
-        "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=",
-        "usagesDigest": "hgylFkgWSg0ulUwWZzEM1aIftlUnbmw2ynWLdEfHnZc=",
+        "bzlTransitiveDigest": "fus14IFJ/1LGWWGKPH/U18VnJCoMjfDt1ckahqCnM0A=",
+        "usagesDigest": "aJF6fLy82rR95Ff5CZPAqxNoFgOMLMN5ImfBS0nhnkg=",
         "recordedFileInputs": {},
         "recordedDirentsInputs": {},
         "envVariables": {},
         "generatedRepoSpecs": {
-          "host_platform": {
-            "bzlFile": "@@platforms//host:extension.bzl",
-            "ruleClassName": "host_platform_repo",
+          "com_github_jetbrains_kotlin_git": {
+            "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:compiler.bzl",
+            "ruleClassName": "kotlin_compiler_git_repository",
+            "attributes": {
+              "urls": [
+                "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip"
+              ],
+              "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88"
+            }
+          },
+          "com_github_jetbrains_kotlin": {
+            "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:compiler.bzl",
+            "ruleClassName": "kotlin_capabilities_repository",
+            "attributes": {
+              "git_repository_name": "com_github_jetbrains_kotlin_git",
+              "compiler_version": "1.9.23"
+            }
+          },
+          "com_github_google_ksp": {
+            "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:ksp.bzl",
+            "ruleClassName": "ksp_compiler_plugin_repository",
+            "attributes": {
+              "urls": [
+                "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip"
+              ],
+              "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d",
+              "strip_version": "1.9.23-1.0.20"
+            }
+          },
+          "com_github_pinterest_ktlint": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_file",
+            "attributes": {
+              "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985",
+              "urls": [
+                "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint"
+              ],
+              "executable": true
+            }
+          },
+          "rules_android": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806",
+              "strip_prefix": "rules_android-0.1.1",
+              "urls": [
+                "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip"
+              ]
+            }
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "rules_kotlin~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
+    },
+    "@@rules_python~//python/extensions:python.bzl%python": {
+      "general": {
+        "bzlTransitiveDigest": "8vDKUdCc6qEk2/YsFiPsFO1Jqgl+XPFRklapOxMAbE8=",
+        "usagesDigest": "dhjp1hteIlxSdhYDivRRdp1JyPeEip+5RuKlqD4X27w=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {
+          "RULES_PYTHON_BZLMOD_DEBUG": null
+        },
+        "generatedRepoSpecs": {
+          "python_3_8_aarch64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "1825b1f7220bc93ff143f2e70b5c6a79c6469e0eeb40824e07a7277f59aabfda",
+              "patches": [],
+              "platform": "aarch64-apple-darwin",
+              "python_version": "3.8.18",
+              "release_filename": "20231002/cpython-3.8.18+20231002-aarch64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.8.18+20231002-aarch64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_8_aarch64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "236a300f386ead02ca98dbddbc026ff4ef4de6701a394106e291ff8b75445ee1",
+              "patches": [],
+              "platform": "aarch64-unknown-linux-gnu",
+              "python_version": "3.8.18",
+              "release_filename": "20231002/cpython-3.8.18+20231002-aarch64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.8.18+20231002-aarch64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_8_x86_64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "fcf04532e644644213977242cd724fe5e84c0a5ac92ae038e07f1b01b474fca3",
+              "patches": [],
+              "platform": "x86_64-apple-darwin",
+              "python_version": "3.8.18",
+              "release_filename": "20231002/cpython-3.8.18+20231002-x86_64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.8.18+20231002-x86_64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_8_x86_64-pc-windows-msvc": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "a9d203e78caed94de368d154e841610cef6f6b484738573f4ae9059d37e898a5",
+              "patches": [],
+              "platform": "x86_64-pc-windows-msvc",
+              "python_version": "3.8.18",
+              "release_filename": "20231002/cpython-3.8.18+20231002-x86_64-pc-windows-msvc-shared-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.8.18+20231002-x86_64-pc-windows-msvc-shared-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_8_x86_64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "1e8a3babd1500111359b0f5675d770984bcbcb2cc8890b117394f0ed342fb9ec",
+              "patches": [],
+              "platform": "x86_64-unknown-linux-gnu",
+              "python_version": "3.8.18",
+              "release_filename": "20231002/cpython-3.8.18+20231002-x86_64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.8.18+20231002-x86_64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_8_host": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "host_toolchain",
+            "attributes": {
+              "python_version": "3.8.18",
+              "user_repository_name": "python_3_8",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_8": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "toolchain_aliases",
+            "attributes": {
+              "python_version": "3.8.18",
+              "user_repository_name": "python_3_8",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_9_aarch64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "fdc4054837e37b69798c2ef796222a480bc1f80e8ad3a01a95d0168d8282a007",
+              "patches": [],
+              "platform": "aarch64-apple-darwin",
+              "python_version": "3.9.18",
+              "release_filename": "20231002/cpython-3.9.18+20231002-aarch64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.9.18+20231002-aarch64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_9_aarch64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "1e0a3e8ce8e58901a259748c0ab640d2b8294713782d14229e882c6898b2fb36",
+              "patches": [],
+              "platform": "aarch64-unknown-linux-gnu",
+              "python_version": "3.9.18",
+              "release_filename": "20231002/cpython-3.9.18+20231002-aarch64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.9.18+20231002-aarch64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_9_ppc64le-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "101c38b22fb2f5a0945156da4259c8e9efa0c08de9d7f59afa51e7ce6e22a1cc",
+              "patches": [],
+              "platform": "ppc64le-unknown-linux-gnu",
+              "python_version": "3.9.18",
+              "release_filename": "20231002/cpython-3.9.18+20231002-ppc64le-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.9.18+20231002-ppc64le-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_9_s390x-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "eee31e55ffbc1f460d7b17f05dd89e45a2636f374a6f8dc29ea13d0497f7f586",
+              "patches": [],
+              "platform": "s390x-unknown-linux-gnu",
+              "python_version": "3.9.18",
+              "release_filename": "20231002/cpython-3.9.18+20231002-s390x-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.9.18+20231002-s390x-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_9_x86_64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "82231cb77d4a5c8081a1a1d5b8ae440abe6993514eb77a926c826e9a69a94fb1",
+              "patches": [],
+              "platform": "x86_64-apple-darwin",
+              "python_version": "3.9.18",
+              "release_filename": "20231002/cpython-3.9.18+20231002-x86_64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.9.18+20231002-x86_64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_9_x86_64-pc-windows-msvc": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "02ea7bb64524886bd2b05d6b6be4401035e4ba4319146f274f0bcd992822cd75",
+              "patches": [],
+              "platform": "x86_64-pc-windows-msvc",
+              "python_version": "3.9.18",
+              "release_filename": "20231002/cpython-3.9.18+20231002-x86_64-pc-windows-msvc-shared-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.9.18+20231002-x86_64-pc-windows-msvc-shared-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_9_x86_64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "f3ff38b1ccae7dcebd8bbf2e533c9a984fac881de0ffd1636fbb61842bd924de",
+              "patches": [],
+              "platform": "x86_64-unknown-linux-gnu",
+              "python_version": "3.9.18",
+              "release_filename": "20231002/cpython-3.9.18+20231002-x86_64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.9.18+20231002-x86_64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_9_host": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "host_toolchain",
+            "attributes": {
+              "python_version": "3.9.18",
+              "user_repository_name": "python_3_9",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_9": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "toolchain_aliases",
+            "attributes": {
+              "python_version": "3.9.18",
+              "user_repository_name": "python_3_9",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_10_aarch64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "fd027b1dedf1ea034cdaa272e91771bdf75ddef4c8653b05d224a0645aa2ca3c",
+              "patches": [],
+              "platform": "aarch64-apple-darwin",
+              "python_version": "3.10.13",
+              "release_filename": "20231002/cpython-3.10.13+20231002-aarch64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.10.13+20231002-aarch64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_10_aarch64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "8675915ff454ed2f1597e27794bc7df44f5933c26b94aa06af510fe91b58bb97",
+              "patches": [],
+              "platform": "aarch64-unknown-linux-gnu",
+              "python_version": "3.10.13",
+              "release_filename": "20231002/cpython-3.10.13+20231002-aarch64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.10.13+20231002-aarch64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_10_ppc64le-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "f3f9c43eec1a0c3f72845d0b705da17a336d3906b7df212d2640b8f47e8ff375",
+              "patches": [],
+              "platform": "ppc64le-unknown-linux-gnu",
+              "python_version": "3.10.13",
+              "release_filename": "20231002/cpython-3.10.13+20231002-ppc64le-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.10.13+20231002-ppc64le-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_10_s390x-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "859f6cfe9aedb6e8858892fdc124037e83ab05f28d42a7acd314c6a16d6bd66c",
+              "patches": [],
+              "platform": "s390x-unknown-linux-gnu",
+              "python_version": "3.10.13",
+              "release_filename": "20231002/cpython-3.10.13+20231002-s390x-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.10.13+20231002-s390x-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_10_x86_64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "be0b19b6af1f7d8c667e5abef5505ad06cf72e5a11bb5844970c395a7e5b1275",
+              "patches": [],
+              "platform": "x86_64-apple-darwin",
+              "python_version": "3.10.13",
+              "release_filename": "20231002/cpython-3.10.13+20231002-x86_64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.10.13+20231002-x86_64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_10_x86_64-pc-windows-msvc": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "b8d930ce0d04bda83037ad3653d7450f8907c88e24bb8255a29b8dab8930d6f1",
+              "patches": [],
+              "platform": "x86_64-pc-windows-msvc",
+              "python_version": "3.10.13",
+              "release_filename": "20231002/cpython-3.10.13+20231002-x86_64-pc-windows-msvc-shared-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.10.13+20231002-x86_64-pc-windows-msvc-shared-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_10_x86_64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "5d0429c67c992da19ba3eb58b3acd0b35ec5e915b8cae9a4aa8ca565c423847a",
+              "patches": [],
+              "platform": "x86_64-unknown-linux-gnu",
+              "python_version": "3.10.13",
+              "release_filename": "20231002/cpython-3.10.13+20231002-x86_64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20231002/cpython-3.10.13+20231002-x86_64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_10_host": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "host_toolchain",
+            "attributes": {
+              "python_version": "3.10.13",
+              "user_repository_name": "python_3_10",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_10": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "toolchain_aliases",
+            "attributes": {
+              "python_version": "3.10.13",
+              "user_repository_name": "python_3_10",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_11_aarch64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "b042c966920cf8465385ca3522986b12d745151a72c060991088977ca36d3883",
+              "patches": [],
+              "platform": "aarch64-apple-darwin",
+              "python_version": "3.11.7",
+              "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_11_aarch64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "b102eaf865eb715aa98a8a2ef19037b6cc3ae7dfd4a632802650f29de635aa13",
+              "patches": [],
+              "platform": "aarch64-unknown-linux-gnu",
+              "python_version": "3.11.7",
+              "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_11_ppc64le-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "b44e1b74afe75c7b19143413632c4386708ae229117f8f950c2094e9681d34c7",
+              "patches": [],
+              "platform": "ppc64le-unknown-linux-gnu",
+              "python_version": "3.11.7",
+              "release_filename": "20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_11_s390x-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "49520e3ff494708020f306e30b0964f079170be83e956be4504f850557378a22",
+              "patches": [],
+              "platform": "s390x-unknown-linux-gnu",
+              "python_version": "3.11.7",
+              "release_filename": "20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_11_x86_64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "a0e615eef1fafdc742da0008425a9030b7ea68a4ae4e73ac557ef27b112836d4",
+              "patches": [],
+              "platform": "x86_64-apple-darwin",
+              "python_version": "3.11.7",
+              "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_11_x86_64-pc-windows-msvc": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "67077e6fa918e4f4fd60ba169820b00be7c390c497bf9bc9cab2c255ea8e6f3e",
+              "patches": [],
+              "platform": "x86_64-pc-windows-msvc",
+              "python_version": "3.11.7",
+              "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_11_x86_64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "4a51ce60007a6facf64e5495f4cf322e311ba9f39a8cd3f3e4c026eae488e140",
+              "patches": [],
+              "platform": "x86_64-unknown-linux-gnu",
+              "python_version": "3.11.7",
+              "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_11_host": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "host_toolchain",
+            "attributes": {
+              "python_version": "3.11.7",
+              "user_repository_name": "python_3_11",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_11": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "toolchain_aliases",
+            "attributes": {
+              "python_version": "3.11.7",
+              "user_repository_name": "python_3_11",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_12_aarch64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "f93f8375ca6ac0a35d58ff007043cbd3a88d9609113f1cb59cf7c8d215f064af",
+              "patches": [],
+              "platform": "aarch64-apple-darwin",
+              "python_version": "3.12.1",
+              "release_filename": "20240107/cpython-3.12.1+20240107-aarch64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-aarch64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_12_aarch64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "236533ef20e665007a111c2f36efb59c87ae195ad7dca223b6dc03fb07064f0b",
+              "patches": [],
+              "platform": "aarch64-unknown-linux-gnu",
+              "python_version": "3.12.1",
+              "release_filename": "20240107/cpython-3.12.1+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_12_ppc64le-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "78051f0d1411ee62bc2af5edfccf6e8400ac4ef82887a2affc19a7ace6a05267",
+              "patches": [],
+              "platform": "ppc64le-unknown-linux-gnu",
+              "python_version": "3.12.1",
+              "release_filename": "20240107/cpython-3.12.1+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_12_s390x-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "60631211c701f8d2c56e5dd7b154e68868128a019b9db1d53a264f56c0d4aee2",
+              "patches": [],
+              "platform": "s390x-unknown-linux-gnu",
+              "python_version": "3.12.1",
+              "release_filename": "20240107/cpython-3.12.1+20240107-s390x-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-s390x-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_12_x86_64-apple-darwin": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "eca96158c1568dedd9a0b3425375637a83764d1fa74446438293089a8bfac1f8",
+              "patches": [],
+              "platform": "x86_64-apple-darwin",
+              "python_version": "3.12.1",
+              "release_filename": "20240107/cpython-3.12.1+20240107-x86_64-apple-darwin-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-x86_64-apple-darwin-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_12_x86_64-pc-windows-msvc": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "fd5a9e0f41959d0341246d3643f2b8794f638adc0cec8dd5e1b6465198eae08a",
+              "patches": [],
+              "platform": "x86_64-pc-windows-msvc",
+              "python_version": "3.12.1",
+              "release_filename": "20240107/cpython-3.12.1+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_12_x86_64-unknown-linux-gnu": {
+            "bzlFile": "@@rules_python~//python:repositories.bzl",
+            "ruleClassName": "python_repository",
+            "attributes": {
+              "sha256": "74e330b8212ca22fd4d9a2003b9eec14892155566738febc8e5e572f267b9472",
+              "patches": [],
+              "platform": "x86_64-unknown-linux-gnu",
+              "python_version": "3.12.1",
+              "release_filename": "20240107/cpython-3.12.1+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz",
+              "urls": [
+                "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz"
+              ],
+              "distutils_content": "",
+              "strip_prefix": "python",
+              "coverage_tool": "",
+              "ignore_root_user_error": false
+            }
+          },
+          "python_3_12_host": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "host_toolchain",
+            "attributes": {
+              "python_version": "3.12.1",
+              "user_repository_name": "python_3_12",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "python_3_12": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "toolchain_aliases",
+            "attributes": {
+              "python_version": "3.12.1",
+              "user_repository_name": "python_3_12",
+              "platforms": [
+                "aarch64-apple-darwin",
+                "aarch64-unknown-linux-gnu",
+                "ppc64le-unknown-linux-gnu",
+                "s390x-unknown-linux-gnu",
+                "x86_64-apple-darwin",
+                "x86_64-pc-windows-msvc",
+                "x86_64-unknown-linux-gnu"
+              ]
+            }
+          },
+          "pythons_hub": {
+            "bzlFile": "@@rules_python~//python/private/bzlmod:pythons_hub.bzl",
+            "ruleClassName": "hub_repo",
+            "attributes": {
+              "default_python_version": "3.11",
+              "toolchain_prefixes": [
+                "_0000_python_3_8_",
+                "_0001_python_3_9_",
+                "_0002_python_3_10_",
+                "_0003_python_3_12_",
+                "_0004_python_3_11_"
+              ],
+              "toolchain_python_versions": [
+                "3.8",
+                "3.9",
+                "3.10",
+                "3.12",
+                "3.11"
+              ],
+              "toolchain_set_python_version_constraints": [
+                "True",
+                "True",
+                "True",
+                "True",
+                "False"
+              ],
+              "toolchain_user_repository_names": [
+                "python_3_8",
+                "python_3_9",
+                "python_3_10",
+                "python_3_12",
+                "python_3_11"
+              ]
+            }
+          },
+          "python_versions": {
+            "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl",
+            "ruleClassName": "multi_toolchain_aliases",
+            "attributes": {
+              "python_versions": {
+                "3.8": "python_3_8",
+                "3.9": "python_3_9",
+                "3.10": "python_3_10",
+                "3.11": "python_3_11",
+                "3.12": "python_3_12"
+              }
+            }
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "rules_python~",
+            "bazel_skylib",
+            "bazel_skylib~"
+          ],
+          [
+            "rules_python~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
+    },
+    "@@rules_python~//python/private/bzlmod:internal_deps.bzl%internal_deps": {
+      "general": {
+        "bzlTransitiveDigest": "7yogJIhmw7i9Wq/n9sQB8N0F84220dJbw64SjOwrmQk=",
+        "usagesDigest": "r7vtlnQfWxEwrL+QFXux06yzeWEkq/hrcwAssoCoSLY=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "rules_python_internal": {
+            "bzlFile": "@@rules_python~//python/private:internal_config_repo.bzl",
+            "ruleClassName": "internal_config_repo",
             "attributes": {}
+          },
+          "pypi__build": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl",
+              "sha256": "af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__click": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl",
+              "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__colorama": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl",
+              "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__importlib_metadata": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/cc/37/db7ba97e676af155f5fcb1a35466f446eadc9104e25b83366e8088c9c926/importlib_metadata-6.8.0-py3-none-any.whl",
+              "sha256": "3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__installer": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl",
+              "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__more_itertools": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/5a/cb/6dce742ea14e47d6f565589e859ad225f2a5de576d7696e0623b784e226b/more_itertools-10.1.0-py3-none-any.whl",
+              "sha256": "64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__packaging": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl",
+              "sha256": "994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__pep517": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/ee/2f/ef63e64e9429111e73d3d6cbee80591672d16f2725e648ebc52096f3d323/pep517-0.13.0-py3-none-any.whl",
+              "sha256": "4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__pip": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl",
+              "sha256": "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__pip_tools": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/e8/df/47e6267c6b5cdae867adbdd84b437393e6202ce4322de0a5e0b92960e1d6/pip_tools-7.3.0-py3-none-any.whl",
+              "sha256": "8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__pyproject_hooks": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl",
+              "sha256": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__setuptools": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/4f/ab/0bcfebdfc3bfa8554b2b2c97a555569c4c1ebc74ea288741ea8326c51906/setuptools-68.1.2-py3-none-any.whl",
+              "sha256": "3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__tomli": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl",
+              "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__wheel": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl",
+              "sha256": "75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
+          },
+          "pypi__zipp": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "url": "https://files.pythonhosted.org/packages/8c/08/d3006317aefe25ea79d3b76c9650afabaf6d63d1c8443b236e7405447503/zipp-3.16.2-py3-none-any.whl",
+              "sha256": "679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0",
+              "type": "zip",
+              "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n    name = \"lib\",\n    srcs = glob([\"**/*.py\"]),\n    data = glob([\"**/*\"], exclude=[\n        # These entries include those put into user-installed dependencies by\n        # data_exclude in /python/pip_install/tools/bazel.py\n        # to avoid non-determinism following pip install's behavior.\n        \"**/*.py\",\n        \"**/*.pyc\",\n        \"**/*.pyc.*\",  # During pyc creation, temp files named *.pyc.NNN are created\n        \"**/* *\",\n        \"**/*.dist-info/RECORD\",\n        \"BUILD\",\n        \"WORKSPACE\",\n    ]),\n    # This makes this directory a top-level in the python import\n    # search path for anything that depends on this.\n    imports = [\".\"],\n)\n"
+            }
           }
         },
-        "recordedRepoMappingEntries": []
+        "recordedRepoMappingEntries": [
+          [
+            "rules_python~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
       }
     }
   }
diff --git a/agent/BUILD.bazel b/agent/BUILD.bazel
index 6366e8a0b5..767decc001 100644
--- a/agent/BUILD.bazel
+++ b/agent/BUILD.bazel
@@ -6,7 +6,6 @@ go_library(
         "agent_configuration.go",
         "agent_pool.go",
         "agent_worker.go",
-        "api.go",
         "doc.go",
         "ec2_meta_data.go",
         "ec2_tags.go",
@@ -18,6 +17,7 @@ go_library(
         "job_runner.go",
         "k8s_tags.go",
         "log_streamer.go",
+        "metrics.go",
         "pipeline_uploader.go",
         "run_job.go",
         "tags.go",
@@ -33,15 +33,18 @@ go_library(
         "//internal/awslib",
         "//internal/experiments",
         "//internal/job/hook",
+        "//internal/ptr",
         "//internal/shell",
         "//kubernetes",
         "//logger",
         "//metrics",
         "//process",
         "//status",
-        "@com_github_aws_aws_sdk_go//aws",
-        "@com_github_aws_aws_sdk_go//aws/ec2metadata",
-        "@com_github_aws_aws_sdk_go//service/ec2",
+        "@com_github_aws_aws_sdk_go_v2//aws",
+        "@com_github_aws_aws_sdk_go_v2_config//:config",
+        "@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds",
+        "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2",
+        "@com_github_aws_aws_sdk_go_v2_service_ec2//types",
         "@com_github_brunoscheufler_aws_ecs_metadata_go//:aws-ecs-metadata-go",
         "@com_github_buildkite_go_pipeline//:go-pipeline",
         "@com_github_buildkite_go_pipeline//signature",
@@ -50,8 +53,12 @@ go_library(
         "@com_github_denisbrodbeck_machineid//:machineid",
         "@com_github_dustin_go_humanize//:go-humanize",
         "@com_github_gowebpki_jcs//:jcs",
+        "@com_github_prometheus_client_golang//prometheus",
+        "@com_github_prometheus_client_golang//prometheus/promauto",
+        "@com_github_prometheus_client_golang//prometheus/promhttp",
         "@com_google_cloud_go_compute_metadata//:metadata",
         "@org_golang_google_api//compute/v1:compute",
+        "@org_golang_google_api//option",
         "@org_golang_x_oauth2//google",
     ],
 )
@@ -60,7 +67,7 @@ go_test(
     name = "agent_test",
     srcs = [
         "agent_worker_test.go",
-        "ec2_meta_data_test.go",
+        "fake_api_server_test.go",
         "gcp_meta_data_test.go",
         "job_runner_test.go",
         "k8s_tags_test.go",
diff --git a/agent/integration/BUILD.bazel b/agent/integration/BUILD.bazel
index d5bc7283cc..847dd1dfe3 100644
--- a/agent/integration/BUILD.bazel
+++ b/agent/integration/BUILD.bazel
@@ -8,6 +8,7 @@ go_library(
     deps = [
         "//agent",
         "//api",
+        "//internal/ptr",
         "//logger",
         "//metrics",
         "@com_github_buildkite_bintest_v3//:bintest",
@@ -29,6 +30,7 @@ go_test(
         "//agent",
         "//api",
         "//clicommand",
+        "//logger",
         "//version",
         "@com_github_buildkite_bintest_v3//:bintest",
         "@com_github_buildkite_go_pipeline//:go-pipeline",
diff --git a/api/BUILD.bazel b/api/BUILD.bazel
index 507469e307..3e345febb0 100644
--- a/api/BUILD.bazel
+++ b/api/BUILD.bazel
@@ -21,6 +21,7 @@ go_library(
         "retryable.go",
         "secrets.go",
         "steps.go",
+        "token.go",
         "uuid.go",
     ],
     importpath = "github.com/buildkite/agent/v3/api",
@@ -40,6 +41,7 @@ go_test(
     srcs = [
         "api_internal_test.go",
         "client_internal_test.go",
+        "client_private_test.go",
         "client_test.go",
         "oidc_test.go",
         "secrets_test.go",
diff --git a/clicommand/BUILD.bazel b/clicommand/BUILD.bazel
index fdc6f2ac0f..f9c992aef8 100644
--- a/clicommand/BUILD.bazel
+++ b/clicommand/BUILD.bazel
@@ -4,6 +4,8 @@ go_library(
     name = "clicommand",
     srcs = [
         "acknowledgements.go",
+        "agent_pause.go",
+        "agent_resume.go",
         "agent_start.go",
         "agent_stop.go",
         "annotate.go",
@@ -14,6 +16,9 @@ go_library(
         "artifact_upload.go",
         "bootstrap.go",
         "build_cancel.go",
+        "cache_restore.go",
+        "cache_save.go",
+        "cache_shared.go",
         "cancel_signal.go",
         "commands.go",
         "doc.go",
@@ -24,6 +29,7 @@ go_library(
         "errors.go",
         "git_credentials_helper.go",
         "global.go",
+        "kubernetes_bootstrap.go",
         "lock_acquire.go",
         "lock_common.go",
         "lock_do.go",
@@ -58,6 +64,7 @@ go_library(
         "//internal/artifact",
         "//internal/awslib",
         "//internal/bkgql",
+        "//internal/cache",
         "//internal/cryptosigner/aws",
         "//internal/experiments",
         "//internal/job",
@@ -65,9 +72,12 @@ go_library(
         "//internal/osutil",
         "//internal/redact",
         "//internal/replacer",
+        "//internal/secrets",
+        "//internal/self",
         "//internal/shell",
         "//internal/stdin",
         "//jobapi",
+        "//kubernetes",
         "//lock",
         "//logger",
         "//metrics",
@@ -99,6 +109,8 @@ go_library(
 go_test(
     name = "clicommand_test",
     srcs = [
+        "agent_pause_test.go",
+        "agent_resume_test.go",
         "agent_start_test.go",
         "agent_stop_test.go",
         "annotate_test.go",
@@ -112,6 +124,7 @@ go_test(
     ],
     embed = [":clicommand"],
     deps = [
+        "//core",
         "//env",
         "//internal/experiments",
         "//logger",
diff --git a/internal/artifact/BUILD.bazel b/internal/artifact/BUILD.bazel
index 0470d3dbda..46bae8c7d8 100644
--- a/internal/artifact/BUILD.bazel
+++ b/internal/artifact/BUILD.bazel
@@ -31,28 +31,23 @@ go_library(
         "//internal/experiments",
         "//internal/mime",
         "//logger",
-        "//pool",
         "//version",
-        "@com_github_aws_aws_sdk_go//aws",
-        "@com_github_aws_aws_sdk_go//aws/credentials",
-        "@com_github_aws_aws_sdk_go//aws/credentials/stscreds",
-        "@com_github_aws_aws_sdk_go//aws/defaults",
-        "@com_github_aws_aws_sdk_go//aws/session",
-        "@com_github_aws_aws_sdk_go//service/s3",
-        "@com_github_aws_aws_sdk_go//service/s3/s3manager",
-        "@com_github_aws_aws_sdk_go//service/sts",
+        "@com_github_aws_aws_sdk_go_v2//aws",
+        "@com_github_aws_aws_sdk_go_v2_config//:config",
+        "@com_github_aws_aws_sdk_go_v2_feature_s3_manager//:manager",
+        "@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
+        "@com_github_aws_aws_sdk_go_v2_service_s3//types",
+        "@com_github_aws_smithy_go//transport/http",
         "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
         "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob",
         "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//sas",
         "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//service",
         "@com_github_buildkite_roko//:roko",
         "@com_github_dustin_go_humanize//:go-humanize",
-        "@com_github_mattn_go_zglob//:go-zglob",
         "@dev_drjosh_zzglob//:zzglob",
         "@org_golang_google_api//googleapi",
         "@org_golang_google_api//option",
         "@org_golang_google_api//storage/v1:storage",
-        "@org_golang_x_oauth2//:oauth2",
         "@org_golang_x_oauth2//google",
     ] + select({
         "@rules_go//go/platform:aix": [
@@ -103,6 +98,7 @@ go_test(
         "downloader_test.go",
         "gs_uploader_test.go",
         "s3_downloader_test.go",
+        "s3_test.go",
         "s3_uploader_test.go",
         "searcher_test.go",
         "uploader_test.go",
@@ -112,9 +108,10 @@ go_test(
         "//api",
         "//internal/experiments",
         "//logger",
+        "@com_github_aws_aws_sdk_go_v2//aws",
+        "@com_github_aws_aws_sdk_go_v2_service_s3//types",
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_go_cmp//cmp/cmpopts",
         "@com_github_stretchr_testify//assert",
-        "@com_github_stretchr_testify//require",
     ],
 )
diff --git a/internal/awslib/BUILD.bazel b/internal/awslib/BUILD.bazel
index a0e095acd1..3141a734c3 100644
--- a/internal/awslib/BUILD.bazel
+++ b/internal/awslib/BUILD.bazel
@@ -2,15 +2,10 @@ load("@rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "awslib",
-    srcs = [
-        "awsv2.go",
-    ],
+    srcs = ["awsv2.go"],
     importpath = "github.com/buildkite/agent/v3/internal/awslib",
     visibility = ["//:__subpackages__"],
     deps = [
-        "@com_github_aws_aws_sdk_go//aws",
-        "@com_github_aws_aws_sdk_go//aws/ec2metadata",
-        "@com_github_aws_aws_sdk_go//aws/session",
         "@com_github_aws_aws_sdk_go_v2//aws",
         "@com_github_aws_aws_sdk_go_v2_config//:config",
         "@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds",
diff --git a/internal/cache/BUILD.bazel b/internal/cache/BUILD.bazel
new file mode 100644
index 0000000000..4089ec208a
--- /dev/null
+++ b/internal/cache/BUILD.bazel
@@ -0,0 +1,29 @@
+load("@rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "cache",
+    srcs = ["cache.go"],
+    importpath = "github.com/buildkite/agent/v3/internal/cache",
+    visibility = ["//:__subpackages__"],
+    deps = [
+        "//logger",
+        "//version",
+        "@com_github_buildkite_zstash//:zstash",
+        "@com_github_buildkite_zstash//api",
+        "@com_github_buildkite_zstash//cache",
+        "@com_github_dustin_go_humanize//:go-humanize",
+        "@in_gopkg_yaml_v3//:yaml_v3",
+    ],
+)
+
+go_test(
+    name = "cache_test",
+    srcs = ["cache_test.go"],
+    embed = [":cache"],
+    deps = [
+        "//logger",
+        "@com_github_buildkite_zstash//:zstash",
+        "@com_github_buildkite_zstash//cache",
+        "@com_github_stretchr_testify//require",
+    ],
+)
diff --git a/internal/e2e/BUILD.bazel b/internal/e2e/BUILD.bazel
new file mode 100644
index 0000000000..9101c80237
--- /dev/null
+++ b/internal/e2e/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "e2e",
+    srcs = ["doc.go"],
+    importpath = "github.com/buildkite/agent/v3/internal/e2e",
+    visibility = ["//:__subpackages__"],
+)
diff --git a/internal/job/BUILD.bazel b/internal/job/BUILD.bazel
index 12f6b6a8a4..a49d81ac21 100644
--- a/internal/job/BUILD.bazel
+++ b/internal/job/BUILD.bazel
@@ -20,6 +20,7 @@ go_library(
     visibility = ["//:__subpackages__"],
     deps = [
         "//agent/plugin",
+        "//api",
         "//env",
         "//internal/experiments",
         "//internal/file",
@@ -27,15 +28,19 @@ go_library(
         "//internal/osutil",
         "//internal/redact",
         "//internal/replacer",
+        "//internal/secrets",
+        "//internal/self",
         "//internal/shell",
         "//internal/shellscript",
         "//internal/socket",
         "//internal/tempfile",
         "//jobapi",
         "//kubernetes",
+        "//logger",
         "//process",
         "//tracetools",
         "//version",
+        "@com_github_buildkite_go_pipeline//:go-pipeline",
         "@com_github_buildkite_roko//:roko",
         "@com_github_buildkite_shellwords//:shellwords",
         "@com_github_opentracing_opentracing_go//:opentracing-go",
@@ -50,8 +55,8 @@ go_library(
         "@io_opentelemetry_go_otel//attribute",
         "@io_opentelemetry_go_otel//propagation",
         "@io_opentelemetry_go_otel//semconv/v1.4.0:v1_4_0",
-        "@io_opentelemetry_go_otel_exporters_otlp_otlptrace//:otlptrace",
         "@io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc//:otlptracegrpc",
+        "@io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp//:otlptracehttp",
         "@io_opentelemetry_go_otel_sdk//resource",
         "@io_opentelemetry_go_otel_sdk//trace",
         "@io_opentelemetry_go_otel_trace//:trace",
@@ -62,6 +67,7 @@ go_library(
 go_test(
     name = "job_test",
     srcs = [
+        "checkout_test.go",
         "config_test.go",
         "executor_test.go",
         "git_test.go",
@@ -71,6 +77,8 @@ go_test(
     embed = [":job"],
     deps = [
         "//env",
+        "//internal/job/githttptest",
+        "//internal/race",
         "//internal/shell",
         "//tracetools",
         "@com_github_buildkite_bintest_v3//:bintest",
@@ -78,6 +86,7 @@ go_test(
         "@com_github_google_go_cmp//cmp",
         "@com_github_opentracing_opentracing_go//:opentracing-go",
         "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
         "@in_gopkg_datadog_dd_trace_go_v1//ddtrace/opentracer",
     ],
 )
diff --git a/internal/job/githttptest/BUILD.bazel b/internal/job/githttptest/BUILD.bazel
new file mode 100644
index 0000000000..319d210ce8
--- /dev/null
+++ b/internal/job/githttptest/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "githttptest",
+    srcs = ["githttptest.go"],
+    importpath = "github.com/buildkite/agent/v3/internal/job/githttptest",
+    visibility = ["//:__subpackages__"],
+)
diff --git a/internal/job/hook/BUILD.bazel b/internal/job/hook/BUILD.bazel
index c1f3fa0355..f2a6740a5c 100644
--- a/internal/job/hook/BUILD.bazel
+++ b/internal/job/hook/BUILD.bazel
@@ -12,8 +12,6 @@ go_library(
     visibility = ["//:__subpackages__"],
     deps = [
         "//env",
-        "//internal/osutil",
-        "//internal/shell",
         "//internal/shellscript",
         "//internal/tempfile",
     ],
diff --git a/internal/job/integration/BUILD.bazel b/internal/job/integration/BUILD.bazel
index 9ca79a0b9d..54d9018f92 100644
--- a/internal/job/integration/BUILD.bazel
+++ b/internal/job/integration/BUILD.bazel
@@ -33,6 +33,7 @@ go_test(
         "main_test.go",
         "plugin_integration_test.go",
         "redaction_integration_test.go",
+        "secrets_integration_test.go",
     ],
     embed = [":integration"],
     deps = [
@@ -43,6 +44,7 @@ go_test(
         "//jobapi",
         "//version",
         "@com_github_buildkite_bintest_v3//:bintest",
+        "@com_github_buildkite_go_pipeline//:go-pipeline",
         "@com_github_urfave_cli//:cli",
         "@tools_gotest_v3//assert",
     ],
diff --git a/internal/ptr/BUILD.bazel b/internal/ptr/BUILD.bazel
new file mode 100644
index 0000000000..a2da047c22
--- /dev/null
+++ b/internal/ptr/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "ptr",
+    srcs = ["to.go"],
+    importpath = "github.com/buildkite/agent/v3/internal/ptr",
+    visibility = ["//:__subpackages__"],
+)
diff --git a/internal/race/BUILD.bazel b/internal/race/BUILD.bazel
new file mode 100644
index 0000000000..8067e09f15
--- /dev/null
+++ b/internal/race/BUILD.bazel
@@ -0,0 +1,11 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "race",
+    srcs = [
+        "race_disabled.go",
+        "race_enabled.go",
+    ],
+    importpath = "github.com/buildkite/agent/v3/internal/race",
+    visibility = ["//:__subpackages__"],
+)
diff --git a/internal/redact/BUILD.bazel b/internal/redact/BUILD.bazel
index b95587126d..62c827c670 100644
--- a/internal/redact/BUILD.bazel
+++ b/internal/redact/BUILD.bazel
@@ -5,7 +5,10 @@ go_library(
     srcs = ["redact.go"],
     importpath = "github.com/buildkite/agent/v3/internal/redact",
     visibility = ["//:__subpackages__"],
-    deps = ["//env"],
+    deps = [
+        "//env",
+        "//internal/replacer",
+    ],
 )
 
 go_test(
diff --git a/internal/replacer/BUILD.bazel b/internal/replacer/BUILD.bazel
index 19fc25d508..34c6c88f42 100644
--- a/internal/replacer/BUILD.bazel
+++ b/internal/replacer/BUILD.bazel
@@ -17,6 +17,7 @@ go_test(
         "bm_redactor_test.go",
         "replacer_test.go",
     ],
+    data = glob(["testdata/**"]),
     deps = [
         ":replacer",
         "//internal/redact",
diff --git a/internal/secrets/BUILD.bazel b/internal/secrets/BUILD.bazel
new file mode 100644
index 0000000000..9329d7acc4
--- /dev/null
+++ b/internal/secrets/BUILD.bazel
@@ -0,0 +1,27 @@
+load("@rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "secrets",
+    srcs = [
+        "doc.go",
+        "secret.go",
+    ],
+    importpath = "github.com/buildkite/agent/v3/internal/secrets",
+    visibility = ["//:__subpackages__"],
+    deps = [
+        "//api",
+        "@org_golang_x_sync//semaphore",
+    ],
+)
+
+go_test(
+    name = "secrets_test",
+    srcs = ["secret_test.go"],
+    embed = [":secrets"],
+    deps = [
+        "//api",
+        "//logger",
+        "@com_github_google_go_cmp//cmp",
+        "@com_github_google_go_cmp//cmp/cmpopts",
+    ],
+)
diff --git a/internal/self/BUILD.bazel b/internal/self/BUILD.bazel
new file mode 100644
index 0000000000..c5965487a3
--- /dev/null
+++ b/internal/self/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "self",
+    srcs = ["self.go"],
+    importpath = "github.com/buildkite/agent/v3/internal/self",
+    visibility = ["//:__subpackages__"],
+)
diff --git a/internal/shell/BUILD.bazel b/internal/shell/BUILD.bazel
index 64d07841a5..e59b41f8b6 100644
--- a/internal/shell/BUILD.bazel
+++ b/internal/shell/BUILD.bazel
@@ -8,8 +8,6 @@ go_library(
         "lookpath.go",
         "lookpath_windows.go",
         "shell.go",
-        "signal.go",
-        "signal_windows.go",
         "test.go",
     ],
     importpath = "github.com/buildkite/agent/v3/internal/shell",
@@ -24,6 +22,9 @@ go_library(
         "@com_github_buildkite_shellwords//:shellwords",
         "@com_github_gofrs_flock//:flock",
         "@com_github_opentracing_opentracing_go//:opentracing-go",
+        "@io_opentelemetry_go_otel//:otel",
+        "@io_opentelemetry_go_otel//propagation",
+        "@io_opentelemetry_go_otel_trace//:trace",
     ],
 )
 
diff --git a/jobapi/BUILD.bazel b/jobapi/BUILD.bazel
index 99b5a22de7..5da00c50d6 100644
--- a/jobapi/BUILD.bazel
+++ b/jobapi/BUILD.bazel
@@ -15,7 +15,6 @@ go_library(
     importpath = "github.com/buildkite/agent/v3/jobapi",
     visibility = ["//visibility:public"],
     deps = [
-        "//agent",
         "//env",
         "//internal/replacer",
         "//internal/shell",
diff --git a/kubernetes/BUILD.bazel b/kubernetes/BUILD.bazel
index 4ea9dd456b..9aed5ca97a 100644
--- a/kubernetes/BUILD.bazel
+++ b/kubernetes/BUILD.bazel
@@ -71,63 +71,48 @@ go_test(
     deps = select({
         "@rules_go//go/platform:aix": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:android": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:darwin": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:dragonfly": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:freebsd": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:illumos": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:ios": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:js": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:linux": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:netbsd": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:openbsd": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:osx": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:plan9": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:qnx": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:solaris": [
             "//logger",
-            "@com_github_stretchr_testify//require",
         ],
         "//conditions:default": [],
     }),

From 018f3eaf7a47cb63f1b6b3b9ae92d19b794e6c86 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Mon, 8 Dec 2025 16:01:33 +1100
Subject: [PATCH 110/242] Update S3Uploader.artifactPath so that it doesn't
 produce leading slashes

---
 internal/artifact/s3_uploader.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/internal/artifact/s3_uploader.go b/internal/artifact/s3_uploader.go
index 54c1208659..1ac00ee21b 100644
--- a/internal/artifact/s3_uploader.go
+++ b/internal/artifact/s3_uploader.go
@@ -144,9 +144,11 @@ func (u *s3UploaderWork) DoWork(ctx context.Context) (*api.ArtifactPartETag, err
 }
 
 func (u *S3Uploader) artifactPath(artifact *api.Artifact) string {
-	parts := []string{u.BucketPath, artifact.Path}
+	if u.BucketPath == "" {
+		return artifact.Path
+	}
 
-	return strings.Join(parts, "/")
+	return path.Join(u.BucketPath, artifact.Path)
 }
 
 func (u *S3Uploader) resolvePermission() (types.ObjectCannedACL, error) {

From 0f08d8d8928e06e2912339f5aefca9e08c3c8790 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Fri, 5 Dec 2025 11:49:01 +1100
Subject: [PATCH 111/242] Add e2e test for uploading/downloading artifacts from
 custom buckets

---
 .buildkite/steps/e2e-tests.sh                 |  2 ++
 internal/artifact/s3_uploader.go              |  1 +
 internal/e2e/artifact_test.go                 | 14 +++++++++
 .../fixtures/artifact_custom_s3_bucket.yaml   | 31 +++++++++++++++++++
 .../fixtures/artifact_upload_download.yaml    |  4 +--
 internal/e2e/testcase.go                      | 22 +++++++++++--
 6 files changed, 69 insertions(+), 5 deletions(-)
 create mode 100644 internal/e2e/fixtures/artifact_custom_s3_bucket.yaml

diff --git a/.buildkite/steps/e2e-tests.sh b/.buildkite/steps/e2e-tests.sh
index 9b86ffd990..0dfafbc2b4 100755
--- a/.buildkite/steps/e2e-tests.sh
+++ b/.buildkite/steps/e2e-tests.sh
@@ -2,10 +2,12 @@
 set -euo pipefail
 
 if [[ -z "${BUILDKITE_TRIGGERED_FROM_BUILD_ID:-}" ]] ; then
+	echo "Running e2e tests on the agent that's currently running"
 	# For now, e2e test the agent that's currently running
 	CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
 	export CI_E2E_TESTS_AGENT_PATH
 else
+	echo "Running e2e tests on the agent from the triggering build"
 	# Download the artifact from the triggering build
 	ARTIFACT="pkg/buildkite-agent-$(go env GOOS)-$(go env GOARCH)"
 	buildkite-agent artifact download "${ARTIFACT}" . --build "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}"
diff --git a/internal/artifact/s3_uploader.go b/internal/artifact/s3_uploader.go
index 1ac00ee21b..98be45c287 100644
--- a/internal/artifact/s3_uploader.go
+++ b/internal/artifact/s3_uploader.go
@@ -134,6 +134,7 @@ func (u *s3UploaderWork) DoWork(ctx context.Context) (*api.ArtifactPartETag, err
 		ACL:         permission,
 		Body:        f,
 	}
+
 	// if enabled we assign the sse configuration
 	if u.serverSideEncryptionEnabled() {
 		params.ServerSideEncryption = types.ServerSideEncryptionAes256
diff --git a/internal/e2e/artifact_test.go b/internal/e2e/artifact_test.go
index da449a932e..b7f44c2b3a 100644
--- a/internal/e2e/artifact_test.go
+++ b/internal/e2e/artifact_test.go
@@ -19,3 +19,17 @@ func TestArtifactUploadDownload(t *testing.T) {
 		t.Errorf("Build state = %q, want %q", got, want)
 	}
 }
+
+// Test that an agent can upload and download artifact to/from a customer-managed S3 bucket
+func TestArtifactUploadDownload_CustomBucket(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, "artifact_custom_s3_bucket.yaml")
+
+	tc.startAgent()
+	build := tc.triggerBuild()
+	state := tc.waitForBuild(ctx, build)
+
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+}
diff --git a/internal/e2e/fixtures/artifact_custom_s3_bucket.yaml b/internal/e2e/fixtures/artifact_custom_s3_bucket.yaml
new file mode 100644
index 0000000000..c454165de9
--- /dev/null
+++ b/internal/e2e/fixtures/artifact_custom_s3_bucket.yaml
@@ -0,0 +1,31 @@
+env:
+  BUILDKITE_S3_DEFAULT_REGION: us-west-2
+  BUILDKITE_ARTIFACT_UPLOAD_DESTINATION: s3://buildkite-agent-e2e-tests-custom-artifacts
+  # NB: not using ACLs (which is what this ACL does) means artifacts won't be available from the buildkite UI.
+  # This isn't really a big deal because the jobs and pipelines that these tests generate are ephemeral
+  BUILDKITE_S3_ACL: bucket-owner-full-control
+agents:
+  queue: {{ .queue }}
+steps:
+  - key: upload
+    plugins:
+      - aws-assume-role-with-web-identity#v1.4.0:
+          role-arn: arn:aws:iam::172840064832:role/pipeline-agent-e2e-testing-buildkite-agent-e2e
+          session-tags:
+            - organization_slug
+            - organization_id
+            - pipeline_slug
+    commands:
+      - echo "hello world" > artifact-$${BUILDKITE_PIPELINE_SLUG}.txt
+      - {{ .buildkite_agent_binary }} artifact upload artifact-$${BUILDKITE_PIPELINE_SLUG}.txt
+  - key: download
+    depends_on: upload
+    plugins:
+      - aws-assume-role-with-web-identity#v1.4.0:
+          role-arn: arn:aws:iam::172840064832:role/pipeline-agent-e2e-testing-buildkite-agent-e2e
+          session-tags:
+            - organization_slug
+            - organization_id
+            - pipeline_slug
+    commands:
+      - {{ .buildkite_agent_binary }} artifact download artifact-$${BUILDKITE_PIPELINE_SLUG}.txt . && if [[ $(cat artifact-$${BUILDKITE_PIPELINE_SLUG}.txt) == "hello world" ]]; then exit 0; else exit 1; fi
diff --git a/internal/e2e/fixtures/artifact_upload_download.yaml b/internal/e2e/fixtures/artifact_upload_download.yaml
index b35f2e7f2a..825489c7c4 100644
--- a/internal/e2e/fixtures/artifact_upload_download.yaml
+++ b/internal/e2e/fixtures/artifact_upload_download.yaml
@@ -4,8 +4,8 @@ steps:
   - key: upload
     commands:
       - echo "hello world" > artifact.txt
-      - buildkite-agent artifact upload artifact.txt
+      - {{ .buildkite_agent_binary }} artifact upload artifact.txt
   - key: download
     depends_on: upload
     commands:
-      - buildkite-agent artifact download artifact.txt . && if [[ $(cat artifact.txt) == "hello world" ]]; then exit 0; else exit 1; fi
+      - {{ .buildkite_agent_binary }} artifact download artifact.txt . && if [[ $(cat artifact.txt) == "hello world" ]]; then exit 0; else exit 1; fi
diff --git a/internal/e2e/testcase.go b/internal/e2e/testcase.go
index 34b3509e7a..fcb868fb74 100644
--- a/internal/e2e/testcase.go
+++ b/internal/e2e/testcase.go
@@ -106,8 +106,13 @@ func newTestCase(t testing.TB, file string) *testCase {
 		}
 	})
 
+	t.Logf("Created cluster queue %q in org %q", queue.Key, targetOrg)
+
 	var pipelineCfg strings.Builder
-	tmplInput := map[string]string{"queue": queue.Key}
+	tmplInput := map[string]string{
+		"queue":                  queue.Key,
+		"buildkite_agent_binary": agentPath,
+	}
 	if err := tmpl.Execute(&pipelineCfg, tmplInput); err != nil {
 		t.Fatalf("Could not execute pipeline config template: tmpl.Execute(%q) error = %v", tmplInput, err)
 	}
@@ -122,6 +127,8 @@ func newTestCase(t testing.TB, file string) *testCase {
 		}
 	})
 
+	t.Logf("Created pipeline %q in org %q", pipeline.Slug, targetOrg)
+
 	return &testCase{
 		TB:             t,
 		fullName:       name,
@@ -282,14 +289,23 @@ func (tc *testCase) startAgent(extraArgs ...string) *exec.Cmd {
 	dir := tc.TempDir()
 	buildPath := filepath.Join(dir, "builds")
 	hooksPath := filepath.Join(dir, "hooks")
-	socketsPath := filepath.Join(dir, "sockets")
 	pluginsPath := filepath.Join(dir, "plugins")
-	for _, path := range []string{buildPath, hooksPath, socketsPath, pluginsPath} {
+	for _, path := range []string{buildPath, hooksPath, pluginsPath} {
 		if err := os.Mkdir(path, 0o700); err != nil {
 			tc.Fatalf("Couldn't create dir inside temporary agent dir: os.Mkdir(%q, %o) = %v", path, 0o700, err)
 		}
 	}
 
+	// Unix domain sockets have a path length limit (~104 chars), so use a short
+	// path in /tmp instead of the potentially long tc.TempDir() path.
+	socketsPath, err := os.MkdirTemp("/tmp", "bk")
+	if err != nil {
+		tc.Fatalf("Couldn't create sockets dir: os.MkdirTemp(/tmp, bk) = %v", err)
+	}
+	tc.Cleanup(func() {
+		os.RemoveAll(socketsPath)
+	})
+
 	args := append([]string{
 		"start",
 		"--debug",

From a40297e1e798273c25a8f6a41606427e70e9dff2 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Tue, 9 Dec 2025 12:36:36 +1100
Subject: [PATCH 112/242] Install aws cli on e2e image

---
 .buildkite/Dockerfile-e2e | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.buildkite/Dockerfile-e2e b/.buildkite/Dockerfile-e2e
index d60d666535..ae3d728402 100644
--- a/.buildkite/Dockerfile-e2e
+++ b/.buildkite/Dockerfile-e2e
@@ -1 +1,9 @@
 FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:cf1272dbf972a94f39a81dcb9dc243a8d2f981e5dd3b5a5c965f6d9ab9268b26
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    unzip \
+    curl \
+    jq \
+    && curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install

From 86ec3624df733ba23f81ab038c27d832cf9eb031 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 11 Dec 2025 09:44:06 +1100
Subject: [PATCH 113/242] Bump version + changelog for v3.114.2

---
 CHANGELOG.md    | 9 +++++++++
 version/VERSION | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4169c40764..8c81a0c362 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.114.2](https://github.com/buildkite/agent/tree/v3.114.2) (2025-12-10)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.114.1...v3.114.2)
+
+### Fixes
+- Further fixes to custom bucket artifact uploads/downloads [#3615](https://github.com/buildkite/agent/pull/3615) (@moskyb)
+
+## Internal
+- Dependabot updates [#3618](https://github.com/buildkite/agent/pull/3618) [#3619](https://github.com/buildkite/agent/pull/3619) [#3622](https://github.com/buildkite/agent/pull/3622) [#3623](https://github.com/buildkite/agent/pull/3623) [#3621](https://github.com/buildkite/agent/pull/3621) (@dependabot[bot])
+
 ## [v3.114.1](https://github.com/buildkite/agent/tree/v3.114.1) (2025-12-05)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.114.0...v3.114.1)
 
diff --git a/version/VERSION b/version/VERSION
index dcb4257f32..ca16271bd5 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.114.1
+3.114.2

From 760f387820cfb8c196cafd9818f800e846f81f7d Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 11 Dec 2025 11:12:29 +1100
Subject: [PATCH 114/242] Fix beta pipeline OIDC claim names
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In https://github.com/buildkite/agent/pull/3614, we added `branch_name` to the list of claims that we add to the OIDC token we use for interacting with AWS in the process of a beta release.

Unfortunately, we used the wrong claim name — it's `build_branch` not `branch_name`. This PR fixes that.
---
 .buildkite/pipeline.release-unstable.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.buildkite/pipeline.release-unstable.yml b/.buildkite/pipeline.release-unstable.yml
index d10cd83c7a..5c828a6cdf 100644
--- a/.buildkite/pipeline.release-unstable.yml
+++ b/.buildkite/pipeline.release-unstable.yml
@@ -17,7 +17,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
-            - branch_name
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -41,7 +41,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
-            - branch_name
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -66,7 +66,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
-            - branch_name
+            - build_branch
       - docker#v5.8.0:
           environment:
             - "AWS_ACCESS_KEY_ID"
@@ -128,7 +128,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
-            - branch_name
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"
@@ -194,7 +194,7 @@ steps:
                 - organization_slug
                 - organization_id
                 - pipeline_slug
-                - branch_name
+                - build_branch
           - ecr#v2.7.0:
               login: true
               account-ids: "445615400570"
@@ -222,7 +222,7 @@ steps:
             - organization_slug
             - organization_id
             - pipeline_slug
-            - branch_name
+            - build_branch
       - ecr#v2.7.0:
           login: true
           account-ids: "032379705303"

From e8da4df260490089a28868c112cd97d07f01f6ce Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 11 Dec 2025 11:24:18 +1100
Subject: [PATCH 115/242] Bump version + changelog for v3.115.0

This updates the changelog for v3.114.2, as it was never released due to a pipeline hiccup
---
 CHANGELOG.md    | 7 +++++--
 version/VERSION | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c81a0c362..95a985bfd8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,8 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [v3.114.2](https://github.com/buildkite/agent/tree/v3.114.2) (2025-12-10)
-[Full Changelog](https://github.com/buildkite/agent/compare/v3.114.1...v3.114.2)
+## [v3.115.0](https://github.com/buildkite/agent/tree/v3.115.0) (2025-12-10)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.114.1...v3.115.0)
+
+### Added
+- `--changed-files-path` for pipeline upload, which allows users to specify a list of files changed for `if_changed` computation [#3620](https://github.com/buildkite/agent/pull/3620) (@pyrocat101)
 
 ### Fixes
 - Further fixes to custom bucket artifact uploads/downloads [#3615](https://github.com/buildkite/agent/pull/3615) (@moskyb)
diff --git a/version/VERSION b/version/VERSION
index ca16271bd5..2975c1ac0c 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.114.2
+3.115.0

From 36fa689a559f9f93a1386085435dd4cc25b44dec Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 11 Dec 2025 12:19:36 +1100
Subject: [PATCH 116/242] feat: add support for concurrent save and restore
 operations

This may add a bit of contention but will help with situations where there are many small, sub 1gb caches.

I have followed the prevailing style with the concurrency implementation.
---
 clicommand/cache_save.go     |   1 +
 clicommand/cache_shared.go   |   7 ++
 internal/cache/cache.go      | 196 ++++++++++++++++++++++++++---------
 internal/cache/cache_test.go |  22 ++--
 4 files changed, 164 insertions(+), 62 deletions(-)

diff --git a/clicommand/cache_save.go b/clicommand/cache_save.go
index 8da4254d40..d92724f231 100644
--- a/clicommand/cache_save.go
+++ b/clicommand/cache_save.go
@@ -89,6 +89,7 @@ var CacheSaveCommand = cli.Command{
 			Ids:             cfg.Ids,
 			APIEndpoint:     apiCfg.Endpoint,
 			APIToken:        apiCfg.Token,
+			Concurrency:     cfg.Concurrency,
 		}
 
 		// Perform cache save (logging happens inside)
diff --git a/clicommand/cache_shared.go b/clicommand/cache_shared.go
index 5d6b817201..01eb8a4dfb 100644
--- a/clicommand/cache_shared.go
+++ b/clicommand/cache_shared.go
@@ -12,6 +12,7 @@ type CacheConfig struct {
 	Pipeline        string   `cli:"pipeline" validate:"required"`
 	Organization    string   `cli:"organization" validate:"required"`
 	CacheConfigFile string   `cli:"cache-config-file"`
+	Concurrency     int      `cli:"concurrency"`
 }
 
 func cacheFlags() []cli.Flag {
@@ -58,5 +59,11 @@ func cacheFlags() []cli.Flag {
 			Usage:  "Path to the cache configuration YAML file",
 			EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
 		},
+		cli.IntFlag{
+			Name:   "concurrency",
+			Value:  2,
+			Usage:  "Number of concurrent cache operations",
+			EnvVar: "BUILDKITE_CACHE_CONCURRENCY",
+		},
 	}
 }
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index 6a1cb75bcd..7b34064d5c 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
+	"sync"
 
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/version"
@@ -33,6 +35,8 @@ type Config struct {
 	APIEndpoint string
 	// APIToken is the access token used to authenticate
 	APIToken string
+	// Concurrency is the number of concurrent cache operations
+	Concurrency int
 }
 
 // FileConfig represents the structure of the cache configuration YAML file
@@ -60,7 +64,7 @@ func Save(ctx context.Context, l logger.Logger, cfg Config) error {
 		return nil
 	}
 
-	return saveWithClient(ctx, l, cacheClient, cacheIDs)
+	return saveWithClient(ctx, l, cacheClient, cacheIDs, cfg.Concurrency)
 }
 
 // Restore restores caches based on the provided configuration and logs results as each cache is processed
@@ -75,7 +79,7 @@ func Restore(ctx context.Context, l logger.Logger, cfg Config) error {
 		return nil
 	}
 
-	return restoreWithClient(ctx, l, cacheClient, cacheIDs)
+	return restoreWithClient(ctx, l, cacheClient, cacheIDs, cfg.Concurrency)
 }
 
 // loadCacheConfiguration loads cache configuration from a YAML file
@@ -158,67 +162,157 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 }
 
 // restoreWithClient performs the restore operation for the given cache IDs using the provided client
-func restoreWithClient(ctx context.Context, l logger.Logger, client CacheClient, cacheIDs []string) error {
+func restoreWithClient(ctx context.Context, l logger.Logger, client CacheClient, cacheIDs []string, concurrency int) error {
+	if concurrency <= 0 {
+		concurrency = runtime.GOMAXPROCS(0)
+	}
+	workerCount := min(concurrency, len(cacheIDs))
+
+	wctx, cancel := context.WithCancelCause(ctx)
+	defer cancel(nil)
+
+	cacheIDsCh := make(chan string)
+	var wg sync.WaitGroup
+
+	for range workerCount {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			for {
+				select {
+				case cacheID, open := <-cacheIDsCh:
+					if !open {
+						return
+					}
+
+					l.Info("Restoring cache: %s", cacheID)
+					result, err := client.Restore(wctx, cacheID)
+					if err != nil {
+						cancel(fmt.Errorf("failed to restore cache %q: %w", cacheID, err))
+						return
+					}
+
+					switch {
+					case result.CacheHit, result.FallbackUsed:
+						l.WithFields(
+							logger.StringField("cache_id", cacheID),
+							logger.StringField("cache_key", result.Key),
+							logger.StringField("fallback_used", fmt.Sprintf("%t", result.FallbackUsed)),
+							logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
+							logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
+							logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
+							logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
+							logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+							logger.IntField("part_count", result.Transfer.PartCount),
+							logger.IntField("concurrency", result.Transfer.Concurrency),
+						).Info("Cache restored")
+					default:
+						l.WithFields(
+							logger.StringField("cache_id", cacheID),
+							logger.StringField("cache_key", result.Key),
+						).Info("Cache not restored (not found)")
+					}
+
+				case <-wctx.Done():
+					return
+				}
+			}
+		}()
+	}
+
+sendLoop:
 	for _, cacheID := range cacheIDs {
-		l.Info("Restoring cache: %s", cacheID)
-		result, err := client.Restore(ctx, cacheID)
-		if err != nil {
-			return fmt.Errorf("failed to restore cache %q: %w", cacheID, err)
+		select {
+		case cacheIDsCh <- cacheID:
+		case <-wctx.Done():
+			break sendLoop
 		}
+	}
+	close(cacheIDsCh)
 
-		switch {
-		case result.CacheHit, result.FallbackUsed:
-			l.WithFields(
-				logger.StringField("cache_id", cacheID),
-				logger.StringField("cache_key", result.Key),
-				logger.StringField("fallback_used", fmt.Sprintf("%t", result.FallbackUsed)),
-				logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
-				logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
-				logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
-				logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
-				logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
-				logger.IntField("part_count", result.Transfer.PartCount),
-				logger.IntField("concurrency", result.Transfer.Concurrency),
-			).Info("Cache restored")
-		default:
-			l.WithFields(
-				logger.StringField("cache_id", cacheID),
-				logger.StringField("cache_key", result.Key),
-			).Info("Cache not restored (not found)")
-		}
+	wg.Wait()
+
+	if err := context.Cause(wctx); err != nil {
+		return err
 	}
 
 	return nil
 }
 
 // saveWithClient performs the save operation for the given cache IDs using the provided client
-func saveWithClient(ctx context.Context, l logger.Logger, client CacheClient, cacheIDs []string) error {
+func saveWithClient(ctx context.Context, l logger.Logger, client CacheClient, cacheIDs []string, concurrency int) error {
+	if concurrency <= 0 {
+		concurrency = runtime.GOMAXPROCS(0)
+	}
+	workerCount := min(concurrency, len(cacheIDs))
+
+	wctx, cancel := context.WithCancelCause(ctx)
+	defer cancel(nil)
+
+	cacheIDsCh := make(chan string)
+	var wg sync.WaitGroup
+
+	for range workerCount {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			for {
+				select {
+				case cacheID, open := <-cacheIDsCh:
+					if !open {
+						return
+					}
+
+					l.Info("Saving cache: %s", cacheID)
+					result, err := client.Save(wctx, cacheID)
+					if err != nil {
+						cancel(fmt.Errorf("failed to save cache %q: %w", cacheID, err))
+						return
+					}
+
+					switch {
+					case result.CacheCreated:
+						l.WithFields(
+							logger.StringField("cache_id", cacheID),
+							logger.StringField("cache_key", result.Key),
+							logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
+							logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
+							logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
+							logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
+							logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
+							logger.IntField("part_count", result.Transfer.PartCount),
+							logger.IntField("concurrency", result.Transfer.Concurrency),
+						).Info("Cache created")
+					default:
+						l.WithFields(
+							logger.StringField("cache_id", cacheID),
+							logger.StringField("cache_key", result.Key),
+						).Info("Cache already exists, not saving")
+					}
+
+				case <-wctx.Done():
+					return
+				}
+			}
+		}()
+	}
+
+sendLoop:
 	for _, cacheID := range cacheIDs {
-		l.Info("Saving cache: %s", cacheID)
-		result, err := client.Save(ctx, cacheID)
-		if err != nil {
-			return fmt.Errorf("failed to save cache %q: %w", cacheID, err)
+		select {
+		case cacheIDsCh <- cacheID:
+		case <-wctx.Done():
+			break sendLoop
 		}
+	}
+	close(cacheIDsCh)
 
-		switch {
-		case result.CacheCreated:
-			l.WithFields(
-				logger.StringField("cache_id", cacheID),
-				logger.StringField("cache_key", result.Key),
-				logger.StringField("archive_size", humanize.Bytes(uint64(result.Archive.Size))),
-				logger.StringField("written_bytes", humanize.Bytes(uint64(result.Archive.WrittenBytes))),
-				logger.StringField("written_entries", fmt.Sprintf("%d", result.Archive.WrittenEntries)),
-				logger.StringField("compression_ratio", fmt.Sprintf("%.2f", result.Archive.CompressionRatio)),
-				logger.StringField("transfer_speed", fmt.Sprintf("%.2fMB/s", result.Transfer.TransferSpeed)),
-				logger.IntField("part_count", result.Transfer.PartCount),
-				logger.IntField("concurrency", result.Transfer.Concurrency),
-			).Info("Cache created")
-		default:
-			l.WithFields(
-				logger.StringField("cache_id", cacheID),
-				logger.StringField("cache_key", result.Key),
-			).Info("Cache already exists, not saving")
-		}
+	wg.Wait()
+
+	if err := context.Cause(wctx); err != nil {
+		return err
 	}
 
 	return nil
diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go
index 94fccc16ab..3dbdbd2438 100644
--- a/internal/cache/cache_test.go
+++ b/internal/cache/cache_test.go
@@ -77,7 +77,7 @@ func TestSaveWithClient_CacheCreated(t *testing.T) {
 		},
 	}
 
-	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"}, 1)
 	require.NoError(t, err)
 }
 
@@ -94,7 +94,7 @@ func TestSaveWithClient_CacheAlreadyExists(t *testing.T) {
 		},
 	}
 
-	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"}, 1)
 	require.NoError(t, err)
 }
 
@@ -122,7 +122,7 @@ func TestSaveWithClient_MultipleCaches(t *testing.T) {
 		},
 	}
 
-	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1", "cache2", "cache3"})
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1", "cache2", "cache3"}, 1)
 	require.NoError(t, err)
 	require.Equal(t, 3, callCount, "Expected Save to be called 3 times")
 }
@@ -138,7 +138,7 @@ func TestSaveWithClient_Error(t *testing.T) {
 		},
 	}
 
-	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	err := saveWithClient(ctx, logger.Discard, mock, []string{"cache1"}, 1)
 	require.Error(t, err)
 	require.ErrorContains(t, err, "failed to save cache")
 	require.ErrorContains(t, err, "save failed")
@@ -155,7 +155,7 @@ func TestSaveWithClient_EmptyCacheIDs(t *testing.T) {
 		},
 	}
 
-	err := saveWithClient(ctx, logger.Discard, mock, []string{})
+	err := saveWithClient(ctx, logger.Discard, mock, []string{}, 1)
 	require.NoError(t, err)
 }
 
@@ -185,7 +185,7 @@ func TestRestoreWithClient_CacheHit(t *testing.T) {
 		},
 	}
 
-	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"}, 1)
 	require.NoError(t, err)
 }
 
@@ -213,7 +213,7 @@ func TestRestoreWithClient_FallbackUsed(t *testing.T) {
 		},
 	}
 
-	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"}, 1)
 	require.NoError(t, err)
 }
 
@@ -232,7 +232,7 @@ func TestRestoreWithClient_CacheMiss(t *testing.T) {
 		},
 	}
 
-	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"}, 1)
 	require.NoError(t, err)
 }
 
@@ -252,7 +252,7 @@ func TestRestoreWithClient_MultipleCaches(t *testing.T) {
 		},
 	}
 
-	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1", "cache2", "cache3"})
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1", "cache2", "cache3"}, 1)
 	require.NoError(t, err)
 	require.Equal(t, 3, callCount, "Expected Restore to be called 3 times")
 }
@@ -268,7 +268,7 @@ func TestRestoreWithClient_Error(t *testing.T) {
 		},
 	}
 
-	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"})
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{"cache1"}, 1)
 	require.Error(t, err)
 	require.ErrorContains(t, err, "failed to restore cache")
 	require.ErrorContains(t, err, "restore failed")
@@ -285,7 +285,7 @@ func TestRestoreWithClient_EmptyCacheIDs(t *testing.T) {
 		},
 	}
 
-	err := restoreWithClient(ctx, logger.Discard, mock, []string{})
+	err := restoreWithClient(ctx, logger.Discard, mock, []string{}, 1)
 	require.NoError(t, err)
 }
 

From c85cdc3553ab4ef6674268c1a1d0c372128f33bf Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Thu, 11 Dec 2025 13:32:23 +1100
Subject: [PATCH 117/242] fix: adjust cache ids flag description as it was
 misleading

---
 clicommand/cache_shared.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/cache_shared.go b/clicommand/cache_shared.go
index 01eb8a4dfb..94a5703779 100644
--- a/clicommand/cache_shared.go
+++ b/clicommand/cache_shared.go
@@ -20,7 +20,7 @@ func cacheFlags() []cli.Flag {
 		cli.StringSliceFlag{
 			Name:   "ids",
 			Value:  &cli.StringSlice{},
-			Usage:  "Comma-separated list of cache IDs (if empty, processes all caches)",
+			Usage:  "Cache IDs to process (can be specified multiple times; if empty, processes all caches)",
 			EnvVar: "BUILDKITE_CACHE_IDS",
 		},
 		cli.StringFlag{

From b41a11f59dbed2f0adf612ba32be3249dd656bcb Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Thu, 11 Dec 2025 16:15:35 +1100
Subject: [PATCH 118/242] PB-1023: remove old kubernetes bootstrap setup

---
 agent/job_runner.go      |  10 ++--
 clicommand/bootstrap.go  |   6 ---
 internal/job/config.go   |   4 --
 internal/job/executor.go | 103 +--------------------------------------
 kubernetes/doc.go        |  59 ++++++++++++++++++++++
 5 files changed, 65 insertions(+), 117 deletions(-)
 create mode 100644 kubernetes/doc.go

diff --git a/agent/job_runner.go b/agent/job_runner.go
index 829988e813..f4c6e3cd20 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -78,7 +78,11 @@ type JobRunnerConfig struct {
 	// Whether to set debug HTTP Requests in the job
 	DebugHTTP bool
 
-	// Whether the job is executing as a k8s pod
+	// KubernetesExec enables Kubernetes execution mode. When true, the job runner
+	// creates a kubernetes.Runner that listens on a UNIX socket for other agent containers
+	// to connect, rather than spawning a local bootstrap subprocess. The other agent containers
+	// containers run `kubernetes-bootstrap` which connects to this socket, receives
+	// environment variables, and executes the bootstrap phases.
 	KubernetesExec bool
 
 	// Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection.
@@ -593,10 +597,6 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS", strconv.Itoa(int(r.conf.AgentConfiguration.SignalGracePeriod/time.Second)))
 	setEnv("BUILDKITE_TRACE_CONTEXT_ENCODING", r.conf.AgentConfiguration.TraceContextEncoding)
 
-	if r.conf.KubernetesExec {
-		setEnv("BUILDKITE_KUBERNETES_EXEC", "true")
-	}
-
 	if !r.conf.AgentConfiguration.AllowMultipartArtifactUpload {
 		setEnv("BUILDKITE_NO_MULTIPART_ARTIFACT_UPLOAD", "true")
 	}
diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go
index fbef2a745d..40c6efbda5 100644
--- a/clicommand/bootstrap.go
+++ b/clicommand/bootstrap.go
@@ -107,8 +107,6 @@ type BootstrapConfig struct {
 	TraceContextEncoding         string   `cli:"trace-context-encoding"`
 	NoJobAPI                     bool     `cli:"no-job-api"`
 	DisableWarningsFor           []string `cli:"disable-warnings-for" normalize:"list"`
-	KubernetesExec               bool     `cli:"kubernetes-exec"`
-	KubernetesContainerID        int      `cli:"kubernetes-container-id"`
 }
 
 var BootstrapCommand = cli.Command{
@@ -398,7 +396,6 @@ var BootstrapCommand = cli.Command{
 			Usage:  "A list of warning IDs to disable",
 			EnvVar: "BUILDKITE_AGENT_DISABLE_WARNINGS_FOR",
 		},
-		KubernetesContainerIDFlag,
 		cancelSignalFlag,
 		cancelGracePeriodFlag,
 		signalGracePeriodSecondsFlag,
@@ -410,7 +407,6 @@ var BootstrapCommand = cli.Command{
 		ProfileFlag,
 		RedactedVars,
 		StrictSingleHooksFlag,
-		KubernetesExecFlag,
 		TraceContextEncodingFlag,
 	},
 	Action: func(c *cli.Context) error {
@@ -517,8 +513,6 @@ var BootstrapCommand = cli.Command{
 			TracingPropagateTraceparent:  cfg.TracingPropagateTraceparent,
 			JobAPI:                       !cfg.NoJobAPI,
 			DisabledWarnings:             cfg.DisableWarningsFor,
-			KubernetesExec:               cfg.KubernetesExec,
-			KubernetesContainerID:        cfg.KubernetesContainerID,
 			Secrets:                      cfg.Secrets,
 		})
 
diff --git a/internal/job/config.go b/internal/job/config.go
index 24e1ffb426..8e5d374edd 100644
--- a/internal/job/config.go
+++ b/internal/job/config.go
@@ -184,10 +184,6 @@ type ExecutorConfig struct {
 	// Whether to start the JobAPI
 	JobAPI bool
 
-	// Whether to enable Kubernetes support, and which container we're running in
-	KubernetesExec        bool
-	KubernetesContainerID int
-
 	// The warnings that have been disabled by the user
 	DisabledWarnings []string
 
diff --git a/internal/job/executor.go b/internal/job/executor.go
index f4a51ddd0e..4010db97bb 100644
--- a/internal/job/executor.go
+++ b/internal/job/executor.go
@@ -34,7 +34,6 @@ import (
 	"github.com/buildkite/agent/v3/internal/shell"
 	"github.com/buildkite/agent/v3/internal/shellscript"
 	"github.com/buildkite/agent/v3/internal/tempfile"
-	"github.com/buildkite/agent/v3/kubernetes"
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/process"
 	"github.com/buildkite/agent/v3/tracetools"
@@ -95,8 +94,7 @@ func (e *Executor) Run(ctx context.Context) (exitCode int) {
 	// Start with stdout and stderr as their usual selves.
 	stdout, stderr := io.Writer(os.Stdout), io.Writer(os.Stderr)
 
-	// The shell environment is initially the current environment.
-	// It is mutated by kubernetesSetup and needed for setupRedactors.
+	// The shell environment is initially the current environment, needed for setupRedactors.
 	environ := env.FromSlice(os.Environ())
 
 	// Create a logger to stderr that can be used for things prior to the
@@ -104,26 +102,6 @@ func (e *Executor) Run(ctx context.Context) (exitCode int) {
 	// Be careful not to log customer secrets here!
 	tempLog := shell.NewWriterLogger(stderr, true, e.DisabledWarnings)
 
-	if e.KubernetesExec {
-		tempLog.Commentf("Using Kubernetes support")
-
-		socket := &kubernetes.Client{ID: e.KubernetesContainerID}
-		if err := e.kubernetesSetup(ctx, environ, socket); err != nil {
-			tempLog.Errorf("Failed to start kubernetes socket client: %v", err)
-			return 1
-		}
-
-		// Tee both stdout and stderr to the k8s socket client, so that the
-		// logs are shipped to the agent container and then to Buildkite, but
-		// are also visible as container logs.
-		stdout = io.MultiWriter(stdout, socket)
-		stderr = io.MultiWriter(stderr, socket)
-
-		defer func() {
-			_ = socket.Exit(exitCode)
-		}()
-	}
-
 	// setup the redactors here once and for the life of the executor
 	// they will be flushed at the end of each hook
 	preRedactedStdout, preRedactedLogger := e.setupRedactors(tempLog, environ, stdout, stderr)
@@ -1364,82 +1342,3 @@ func (e *Executor) setupRedactors(log shell.Logger, environ *env.Environment, st
 	logger := shell.NewWriterLogger(loggerRedactor, true, e.DisabledWarnings)
 	return stdoutRedactor, logger
 }
-
-func (e *Executor) kubernetesSetup(ctx context.Context, environ *env.Environment, k8sAgentSocket *kubernetes.Client) error {
-	rtr := roko.NewRetrier(
-		roko.WithMaxAttempts(7),
-		roko.WithStrategy(roko.Exponential(2*time.Second, 0)),
-	)
-	regResp, err := roko.DoFunc(ctx, rtr, func(rtr *roko.Retrier) (*kubernetes.RegisterResponse, error) {
-		return k8sAgentSocket.Connect(ctx)
-	})
-	if err != nil {
-		return fmt.Errorf("error connecting to kubernetes runner: %w", err)
-	}
-
-	// Set our environment vars based on the registration response.
-	// But note that the k8s stack interprets the job definition itself,
-	// and sets a variety of env vars (e.g. BUILDKITE_COMMAND) that
-	// *could* be different to the ones the agent normally supplies.
-	// Examples:
-	// * The command container could be passed a specific
-	//   BUILDKITE_COMMAND that is computed from the command+args
-	//   podSpec attributes (in the kubernetes "plugin"), instead of the
-	//   "command" attribute of the step.
-	// * BUILDKITE_PLUGINS is pre-processed by the k8s stack to remove
-	//   the kubernetes "plugin". If we used the agent's default
-	//   BUILDKITE_PLUGINS, we'd be trying to find a kubernetes plugin
-	//   that doesn't exist.
-	// So we should skip setting any vars that are already set, and
-	// specifically any that could be deliberately *unset* by the
-	// k8s stack (BUILDKITE_PLUGINS could be unset if kubernetes is
-	// the only "plugin" in the step).
-	// (Maybe we could move some of the k8s stack processing in here?)
-	//
-	// To think about: how to obtain the env vars early enough to set
-	// them in ExecutorConfig (because of how urfave/cli works, it
-	// must happen before App.Run, which is before the program even knows
-	// which subcommand is running).
-	for n, v := range env.FromSlice(regResp.Env).Dump() {
-		// Skip these ones specifically.
-		// See agent-stack-k8s/internal/controller/scheduler/scheduler.go#(*jobWrapper).Build
-		switch n {
-		case "BUILDKITE_COMMAND", "BUILDKITE_ARTIFACT_PATHS", "BUILDKITE_PLUGINS":
-			continue
-
-		case "BUILDKITE_AGENT_ACCESS_TOKEN":
-			// Just in case someone has tried to fiddle with this, set it
-			// unconditionally (to be compatible with pre-v3.74.1 / PR 2851
-			// behavior).
-			environ.Set(n, v)
-			if err := os.Setenv(n, v); err != nil {
-				return err
-			}
-			continue
-		}
-		// Skip any that are already set.
-		if environ.Exists(n) {
-			continue
-		}
-		// Set it!
-		environ.Set(n, v)
-		if err := os.Setenv(n, v); err != nil {
-			return err
-		}
-	}
-
-	// So that the agent doesn't exit early thinking the client is lost, we want
-	// to continue talking to the agent container for as long as possible (after
-	// Interrupt). Hence detach the StatusLoop context from cancellation using
-	// [context.WithoutCancel]. The goroutine will exit with the process.
-	// (Why even have a context arg? Testing and possible future value-passing)
-	return k8sAgentSocket.StatusLoop(context.WithoutCancel(ctx), func(err error) {
-		// If the k8s client is interrupted for any reason (either the server
-		// is in state interrupted or the connection died or ...), we should
-		// cancel the job.
-		if err != nil {
-			e.shell.Errorf("Error waiting for client interrupt: %v", err)
-		}
-		e.Cancel()
-	})
-}
diff --git a/kubernetes/doc.go b/kubernetes/doc.go
new file mode 100644
index 0000000000..169e049c4a
--- /dev/null
+++ b/kubernetes/doc.go
@@ -0,0 +1,59 @@
+// Package kubernetes provides coordination between containers in a Kubernetes
+// pod when running Buildkite jobs.
+//
+// # Architecture
+//
+// In Kubernetes, a job runs across multiple containers within a single pod:
+//
+//   - Agent container: runs `buildkite-agent start --kubernetes-exec`, receives
+//     jobs from Buildkite, and acts as a coordinator.
+//   - Checkout container: clones the repository.
+//   - Command container(s): execute the build commands.
+//
+// These containers need coordination because:
+//
+//  1. Environment sharing: The agent container receives job details (API tokens,
+//     build metadata, plugin configs) from Buildkite. Other containers need this
+//     information but Kubernetes doesn't share environment variables between
+//     containers.
+//
+//  2. Sequential execution: The checkout container must complete before command
+//     containers start. Kubernetes starts all containers simultaneously by default.
+//
+//  3. Log aggregation: All container output must be collected and streamed to
+//     Buildkite as a single job log.
+//
+//  4. Cancellation: When a job is cancelled in Buildkite, all containers must
+//     be notified to gracefully shut down.
+//
+//  5. Exit status: The agent must collect exit statuses from all containers to
+//     report the final job result.
+//
+// # Components
+//
+// [Runner] implements the server side, running in the agent container. It
+// creates a Unix socket and exposes an RPC API for other containers to connect.
+//
+// [Client] implements the client side, used by `kubernetes-bootstrap` running
+// in checkout and command containers. It connects to the socket, receives
+// environment variables, and reports logs and exit status.
+//
+// # Flow
+//
+// 1. Agent container starts [Runner], which listens on a Unix socket.
+//
+// 2. Each container runs `kubernetes-bootstrap`, which creates a [Client] and
+// calls [Client.Connect] to register with the runner and receive environment
+// variables.
+//
+// 3. The client calls [Client.StatusLoop], which blocks until the runner
+// signals it can start (ensuring sequential execution).
+//
+// 4. The container executes `buildkite-agent bootstrap`, streaming logs through
+// the socket via [Client.Write].
+//
+// 5. On completion, the container calls [Client.Exit] to report its exit status.
+//
+// 6. If the job is cancelled, the runner broadcasts an interrupt to all
+// connected clients via the status polling mechanism.
+package kubernetes

From 69d839bb00b4fa51b1295b03b62024e137cfcf83 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Fri, 12 Dec 2025 08:09:25 +1100
Subject: [PATCH 119/242] chore(deps): update zstash to v0.6.0 and update
 progress callback

---
 go.mod                  | 12 ++++++------
 go.sum                  | 24 ++++++++++++------------
 internal/cache/cache.go |  5 ++---
 3 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/go.mod b/go.mod
index df11dc3089..caa6b19644 100644
--- a/go.mod
+++ b/go.mod
@@ -12,12 +12,12 @@ require (
 	github.com/DataDog/datadog-go/v5 v5.8.1
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go-v2 v1.41.0
-	github.com/aws/aws-sdk-go-v2/config v1.32.4
+	github.com/aws/aws-sdk-go-v2/config v1.32.5
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.14
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.15
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.0
 	github.com/aws/aws-sdk-go-v2/service/kms v1.49.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
@@ -26,7 +26,7 @@ require (
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
-	github.com/buildkite/zstash v0.5.1
+	github.com/buildkite/zstash v0.6.0
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
@@ -96,7 +96,7 @@ require (
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.4 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.5 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -108,7 +108,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
diff --git a/go.sum b/go.sum
index 20b48aec3c..4b4535d008 100644
--- a/go.sum
+++ b/go.sum
@@ -80,14 +80,14 @@ github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgP
 github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.4 h1:gl+DxVuadpkYoaDcWllZqLkhGEbvwyqgNVRTmlaf5PI=
-github.com/aws/aws-sdk-go-v2/config v1.32.4/go.mod h1:MBUp9Og/bzMmQHjMwace4aJfyvJeadzXjoTcR/SxLV0=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.4 h1:KeIZxHVbGWRLhPvhdPbbi/DtFBHNKm6OsVDuiuFefdQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.4/go.mod h1:Smw5n0nCZE9PeFEguofdXyt8kUC4JNrkDTfBOioPhFA=
+github.com/aws/aws-sdk-go-v2/config v1.32.5 h1:pz3duhAfUgnxbtVhIK39PGF/AHYyrzGEyRD9Og0QrE8=
+github.com/aws/aws-sdk-go-v2/config v1.32.5/go.mod h1:xmDjzSUs/d0BB7ClzYPAZMmgQdrodNjPPhd6bGASwoE=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.14 h1:Ml4JmbZDi48OiAQx7CUst0ZO48ftbfNsWMEYiuhu06Q=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.14/go.mod h1:aBxw6KN/hqD598VrxXR6RXgNSWC3q0/aT14VXHD/MSo=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.15 h1:Zn4SfxkULorRqLg/VhxQ5cg9bi8Qhq7Y8W9RUew15oI=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.15/go.mod h1:uFphWOp8hzgUQ6ORHAw2WUf2xeqOWHjhgCDSdAVxzp0=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
@@ -108,16 +108,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lu
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
 github.com/aws/aws-sdk-go-v2/service/kms v1.49.3 h1:FtjNR2BaMf/Wu3Um8RNSAfi9An5bEYxJU6f4rLOqSLU=
 github.com/aws/aws-sdk-go-v2/service/kms v1.49.3/go.mod h1:HO31s0qt0lso/ADvZQyzKs8js/ku0fMHsfyXW8OPVYc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1 h1:5FhzzN6JmlGQF6c04kDIb5KNGm6KnNdLISNrfivIhHg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.93.1/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2 h1:U3ygWUhCpiSPYSHOrRhb3gOl9T5Y3kB8k5Vjs//57bE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.4 h1:YCu/iAhQer8WZ66lldyKkpvMyv+HkPufMa4dyT6wils=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.4/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
 github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
 github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -144,8 +144,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
-github.com/buildkite/zstash v0.5.1 h1:0/ujF6VzcuZIOyE7O7+kUclEKce+WKtlft//OiA3MZY=
-github.com/buildkite/zstash v0.5.1/go.mod h1:zAX0o9prXpjxsPAiPxi3sK4DNIC8TTHJ2j5Cp00ARLQ=
+github.com/buildkite/zstash v0.6.0 h1:yGUHomJjxnn++xZvfk5RlNTUxHyz3Pol2ct1s/K7xAw=
+github.com/buildkite/zstash v0.6.0/go.mod h1:BGBukIwUaM3r/ypdQWyaDIP3izQj1PSauoAS6fOaYwg=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf h1:yxlp0s+Sge9UsKEK0Bsvjiopb9XRk+vxylmZ9eGBfm8=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index 6a1cb75bcd..178876eb73 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -114,12 +114,11 @@ func setupCacheClient(ctx context.Context, l logger.Logger, cfg Config) (*zstash
 		Pipeline:     cfg.Pipeline,
 		Organization: cfg.Organization,
 		Caches:       fileConfig.Dependencies,
-		OnProgress: func(stage, message string, current, total int) {
+		OnProgress: func(cacheID, stage, message string, _, _ int) {
 			l.WithFields(
+				logger.StringField("cache_id", cacheID),
 				logger.StringField("stage", stage),
 				logger.StringField("message", message),
-				logger.IntField("current", current),
-				logger.IntField("total", total),
 			).Info("Cache progress")
 		},
 	})

From 09db586f4760cc279a8563b49508c90a1a696442 Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Thu, 11 Dec 2025 15:33:44 +1100
Subject: [PATCH 120/242] PS-1491: Fix double retry issue for k8s mode
 bootstrap

---
 clicommand/kubernetes_bootstrap.go | 17 +++++------------
 kubernetes/client.go               | 14 ++++++++++----
 kubernetes/kubernetes_test.go      |  2 ++
 3 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/clicommand/kubernetes_bootstrap.go b/clicommand/kubernetes_bootstrap.go
index 4858cdb2b2..ebe42ae60b 100644
--- a/clicommand/kubernetes_bootstrap.go
+++ b/clicommand/kubernetes_bootstrap.go
@@ -14,7 +14,6 @@ import (
 	"github.com/buildkite/agent/v3/env"
 	"github.com/buildkite/agent/v3/kubernetes"
 	"github.com/buildkite/agent/v3/process"
-	"github.com/buildkite/roko"
 	"github.com/urfave/cli"
 )
 
@@ -76,19 +75,13 @@ var KubernetesBootstrapCommand = cli.Command{
 		// Registration passes down the env vars the agent normally sets on the
 		// subprocess, but in this case the bootstrap is in a separate
 		// container.
-		timeoutDuration := 120 * time.Second
+		connectionTimeout := 120 * time.Second
 		if cfg.KubernetesBootstrapConnectionTimeout > 0 {
-			timeoutDuration = cfg.KubernetesBootstrapConnectionTimeout
+			connectionTimeout = cfg.KubernetesBootstrapConnectionTimeout
 		}
-		interval := 3 * time.Second
-		maxAttempt := max(int(timeoutDuration.Seconds())/int(interval.Seconds()), 1)
-		rtr := roko.NewRetrier(
-			roko.WithMaxAttempts(maxAttempt),
-			roko.WithStrategy(roko.Constant(interval)),
-		)
-		regResp, err := roko.DoFunc(ctx, rtr, func(rtr *roko.Retrier) (*kubernetes.RegisterResponse, error) {
-			return socket.Connect(ctx)
-		})
+		connectCtx, connectCancel := context.WithTimeout(ctx, connectionTimeout)
+		defer connectCancel()
+		regResp, err := socket.Connect(connectCtx)
 		if err != nil {
 			return fmt.Errorf("error connecting to kubernetes runner: %w", err)
 		}
diff --git a/kubernetes/client.go b/kubernetes/client.go
index 94a722124e..5266173940 100644
--- a/kubernetes/client.go
+++ b/kubernetes/client.go
@@ -23,16 +23,22 @@ type Client struct {
 
 var errNotConnected = errors.New("client not connected")
 
+// Connect establishes a connection to the Agent container in the same k8s pod and registers the client.
+// Because k8s might run the containers "out of order", the server socket might not exist yet,
+// so this method retries the connection with a 1-second interval until the context is cancelled.
+// Callers should use context.WithTimeout to control the connection timeout.
 func (c *Client) Connect(ctx context.Context) (*RegisterResponse, error) {
 	if c.SocketPath == "" {
 		c.SocketPath = defaultSocketPath
 	}
 
-	// Because k8s might run the containers "out of order", the server socket
-	// might not exist yet. Try to connect several times.
+	// Retry until the context is cancelled. The high maxAttempts is a safety net
+	// in case the caller forgets to set a context deadline - in practice the
+	// context deadline should be the limiting factor.
+	const retryInterval = time.Second
 	r := roko.NewRetrier(
-		roko.WithMaxAttempts(30),
-		roko.WithStrategy(roko.Constant(time.Second)),
+		roko.WithMaxAttempts(3600),
+		roko.WithStrategy(roko.Constant(retryInterval)),
 	)
 	client, err := roko.DoFunc(ctx, r, func(*roko.Retrier) (*rpc.Client, error) {
 		return rpc.DialHTTP("unix", c.SocketPath)
diff --git a/kubernetes/kubernetes_test.go b/kubernetes/kubernetes_test.go
index 65b4b9e318..2e2ef697c4 100644
--- a/kubernetes/kubernetes_test.go
+++ b/kubernetes/kubernetes_test.go
@@ -309,6 +309,8 @@ func init() {
 
 // helper for ignoring the response from regular client.Connect
 func connect(ctx context.Context, c *Client) error {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
 	_, err := c.Connect(ctx)
 	return err
 }

From 3a545ad325c059092cb1c11107b2fa5c4f422d7a Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Fri, 12 Dec 2025 10:15:53 +1100
Subject: [PATCH 121/242] release: 3.115.1

---
 CHANGELOG.md    | 13 ++++++++++++-
 version/VERSION |  2 +-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95a985bfd8..2d797ef57b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.115.1](https://github.com/buildkite/agent/tree/v3.115.1) (2025-12-12)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.115.0...v3.115.1)
+
+### Fixes
+- PS-1491: Fix double retry issue for k8s mode bootstrap [#3628](https://github.com/buildkite/agent/pull/3628) (@zhming0)
+
+### Internal
+- PB-1023: remove old kubernetes bootstrap setup [#3629](https://github.com/buildkite/agent/pull/3629) (@zhming0)
+- chore(deps): update zstash to v0.6.0 and update progress callback [#3630](https://github.com/buildkite/agent/pull/3630) (@wolfeidau)
+- feat: add support for concurrent save and restore operations [#3627](https://github.com/buildkite/agent/pull/3627) (@wolfeidau)
+
 ## [v3.115.0](https://github.com/buildkite/agent/tree/v3.115.0) (2025-12-10)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.114.1...v3.115.0)
 
@@ -14,7 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Fixes
 - Further fixes to custom bucket artifact uploads/downloads [#3615](https://github.com/buildkite/agent/pull/3615) (@moskyb)
 
-## Internal
+### Internal
 - Dependabot updates [#3618](https://github.com/buildkite/agent/pull/3618) [#3619](https://github.com/buildkite/agent/pull/3619) [#3622](https://github.com/buildkite/agent/pull/3622) [#3623](https://github.com/buildkite/agent/pull/3623) [#3621](https://github.com/buildkite/agent/pull/3621) (@dependabot[bot])
 
 ## [v3.114.1](https://github.com/buildkite/agent/tree/v3.114.1) (2025-12-05)
diff --git a/version/VERSION b/version/VERSION
index 2975c1ac0c..f5563946b1 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.115.0
+3.115.1

From 89f2087809aeff18c91849ef4555a590e27d3584 Mon Sep 17 00:00:00 2001
From: Mark Wolfe <mark.wolfe@buildkite.com>
Date: Fri, 12 Dec 2025 12:08:35 +1100
Subject: [PATCH 122/242] chore(deps): bump zstash to v0.7.0

This release adds copy in place to ensure local s3 objects which are read frequently don't get removed by s3 lifecycle policies.
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index caa6b19644..1eb9528aa2 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
-	github.com/buildkite/zstash v0.6.0
+	github.com/buildkite/zstash v0.7.0
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
diff --git a/go.sum b/go.sum
index 4b4535d008..f9fafa5e28 100644
--- a/go.sum
+++ b/go.sum
@@ -144,8 +144,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
-github.com/buildkite/zstash v0.6.0 h1:yGUHomJjxnn++xZvfk5RlNTUxHyz3Pol2ct1s/K7xAw=
-github.com/buildkite/zstash v0.6.0/go.mod h1:BGBukIwUaM3r/ypdQWyaDIP3izQj1PSauoAS6fOaYwg=
+github.com/buildkite/zstash v0.7.0 h1:lKDGLaRjdjh7M0ItRfpy9HIUv02r1bszKC/8fupDjXY=
+github.com/buildkite/zstash v0.7.0/go.mod h1:BGBukIwUaM3r/ypdQWyaDIP3izQj1PSauoAS6fOaYwg=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf h1:yxlp0s+Sge9UsKEK0Bsvjiopb9XRk+vxylmZ9eGBfm8=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=

From 684e0d280e46f553cb2dd79ba0e888c9591ab81f Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Fri, 12 Dec 2025 15:39:39 +1100
Subject: [PATCH 123/242] PB-1025: improve e2e test DevEX

---
 bin/e2e-local            | 27 +++++++++++++++++++++++++++
 internal/e2e/testcase.go | 11 +++++++++++
 2 files changed, 38 insertions(+)
 create mode 100755 bin/e2e-local

diff --git a/bin/e2e-local b/bin/e2e-local
new file mode 100755
index 0000000000..142909522d
--- /dev/null
+++ b/bin/e2e-local
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run e2e tests locally against a freshly compiled buildkite-agent.
+#
+# NOTE: Tests run against a production Buildkite org, not a test environment.
+#
+# Required environment variables:
+#   CI_E2E_TESTS_BUILDKITE_API_TOKEN - Buildkite API token
+#   CI_E2E_TESTS_AGENT_TOKEN         - Buildkite agent token
+#
+# Optional environment variables:
+#   CI_E2E_TESTS_PRINT_JOB_LOGS=true     - Print job logs after each test
+#
+# Usage:
+#   bin/e2e-local                     # run all e2e tests
+#   bin/e2e-local -run TestBasicE2E   # run a specific test
+#   bin/e2e-local -v                  # verbose output
+
+# Compile buildkite-agent
+AGENT_BINARY="$(pwd)/pkg/buildkite-agent-$(go env GOOS)-$(go env GOARCH)"
+echo "Compiling buildkite-agent to ${AGENT_BINARY}..."
+go build -o "${AGENT_BINARY}" .
+
+# Run e2e tests
+export CI_E2E_TESTS_AGENT_PATH="${AGENT_BINARY}"
+exec go test -tags e2e ./internal/e2e/... "$@"
diff --git a/internal/e2e/testcase.go b/internal/e2e/testcase.go
index fcb868fb74..b0556545dc 100644
--- a/internal/e2e/testcase.go
+++ b/internal/e2e/testcase.go
@@ -6,6 +6,7 @@ import (
 	"cmp"
 	"context"
 	"embed"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -32,6 +33,7 @@ var (
 
 	// E2E testing config
 	agentPath = os.Getenv("CI_E2E_TESTS_AGENT_PATH")
+	printLogs = os.Getenv("CI_E2E_TESTS_PRINT_JOB_LOGS") == "true"
 
 	// Obtained from agentToken in main_test.go
 	targetOrg     string
@@ -191,6 +193,7 @@ func (tc *testCase) triggerBuild() *buildkite.Build {
 // Note that the build pointed to by build is updated with the latest state
 // after each poll. It calls t.Fatal if there was an error fetching the build
 // or the context ends.
+// If CI_E2E_TESTS_PRINT_JOB_LOGS=true, it fetches and prints the build logs.
 func (tc *testCase) waitForBuild(ctx context.Context, build *buildkite.Build) string {
 	tick := time.Tick(time.Second)
 	for {
@@ -204,6 +207,10 @@ func (tc *testCase) waitForBuild(ctx context.Context, build *buildkite.Build) st
 		*build = state
 		switch state.State {
 		case "passed", "failed", "canceled", "canceling":
+			if printLogs {
+				logs := tc.fetchLogs(ctx, build)
+				tc.Logf("Build logs:\n%s", logs)
+			}
 			return state.State
 
 		case "scheduled", "running":
@@ -265,6 +272,10 @@ func createPipeline(ctx context.Context, client *buildkite.Client, name, config
 		ClusterID:     targetCluster,
 	})
 	if err != nil {
+		var errResp *buildkite.ErrorResponse
+		if errors.As(err, &errResp) && len(errResp.RawBody) > 0 {
+			return nil, nopCleanup, fmt.Errorf("%w: %s", err, errResp.RawBody)
+		}
 		return nil, nopCleanup, err
 	}
 

From 1e33509a0694a3775b38ad95288c147c3e448967 Mon Sep 17 00:00:00 2001
From: Mykematt <michael.omoge@buildkite.com>
Date: Fri, 12 Dec 2025 18:07:40 -0700
Subject: [PATCH 124/242] Remove experiment from 'env' command since it has
 been promoted since agent version v3.64.0

---
 clicommand/env_get.go   | 5 +----
 clicommand/env_set.go   | 2 --
 clicommand/env_unset.go | 2 --
 3 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/clicommand/env_get.go b/clicommand/env_get.go
index 48104cbf87..bec61123a1 100644
--- a/clicommand/env_get.go
+++ b/clicommand/env_get.go
@@ -12,7 +12,7 @@ import (
 
 const envClientErrMessage = `Could not create Job API client: %w
 This command can only be used from hooks or plugins running under a job executor
-where the "job-api" experiment is enabled.`
+where the Job API (available in agent v3.64.0 and later) is available.`
 
 const envGetHelpDescription = `Usage:
 
@@ -23,9 +23,6 @@ Description:
 Retrieves environment variables and their current values from the current job
 execution environment.
 
-Note that this subcommand is only available from within the job executor with
-the ′job-api′ experiment enabled.
-
 Changes to the job environment only apply to the environments of subsequent
 phases of the job. However, ′env get′ can be used to inspect the changes made
 with ′env set′ and ′env unset′.
diff --git a/clicommand/env_set.go b/clicommand/env_set.go
index 0a2d01c539..ed1b99339e 100644
--- a/clicommand/env_set.go
+++ b/clicommand/env_set.go
@@ -24,8 +24,6 @@ This command cannot unset Buildkite read-only variables.
 
 To read the new values of variables from within the current phase, use ′env get′.
 
-Note that this subcommand is only available from within the job executor with the job-api experiment enabled.
-
 Examples:
 
 Setting the variables ′LLAMA′ and ′ALPACA′:
diff --git a/clicommand/env_unset.go b/clicommand/env_unset.go
index 59d4d9dd4e..4f57e5e99f 100644
--- a/clicommand/env_unset.go
+++ b/clicommand/env_unset.go
@@ -23,8 +23,6 @@ This command cannot unset Buildkite read-only variables.
 
 To read the new values of variables from within the current phase, use ′env get′.
 
-Note that this subcommand is only available from within the job executor with the job-api experiment enabled.
-
 Examples:
 
 Unsetting the variables ′LLAMA′ and ′ALPACA′:

From c8809b9b6b92744fdfd2c810c607cfe633d3f0dc Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Fri, 12 Dec 2025 14:32:14 +1100
Subject: [PATCH 125/242] PB-1007: add e2e test for gcs artifact
 upload/download

---
 internal/e2e/artifact_test.go                 | 20 ++++++++++++++++
 .../fixtures/artifact_custom_gcs_bucket.yaml  | 23 +++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 internal/e2e/fixtures/artifact_custom_gcs_bucket.yaml

diff --git a/internal/e2e/artifact_test.go b/internal/e2e/artifact_test.go
index b7f44c2b3a..39013ccf77 100644
--- a/internal/e2e/artifact_test.go
+++ b/internal/e2e/artifact_test.go
@@ -1,5 +1,10 @@
 //go:build e2e
 
+// Note to external contributors: Many test cases in this file require
+// access to specific Buildkite organization resources and may not work
+// in your local environment. These tests can be safely skipped during
+// local development.
+
 package e2e
 
 import (
@@ -33,3 +38,18 @@ func TestArtifactUploadDownload_CustomBucket(t *testing.T) {
 		t.Errorf("Build state = %q, want %q", got, want)
 	}
 }
+
+// Test that we can upload/downdload artifact using a custom GCS bucket.
+// Everything that gets uploaded here gets auto removed in 30 days.
+func TestArtifactUploadDownload_GCS(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, "artifact_custom_gcs_bucket.yaml")
+
+	tc.startAgent()
+	build := tc.triggerBuild()
+	state := tc.waitForBuild(ctx, build)
+
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+}
diff --git a/internal/e2e/fixtures/artifact_custom_gcs_bucket.yaml b/internal/e2e/fixtures/artifact_custom_gcs_bucket.yaml
new file mode 100644
index 0000000000..fcaa703263
--- /dev/null
+++ b/internal/e2e/fixtures/artifact_custom_gcs_bucket.yaml
@@ -0,0 +1,23 @@
+env:
+  BUILDKITE_ARTIFACT_UPLOAD_DESTINATION: "gs://buildkite-agent-e2e-testing/$BUILDKITE_PIPELINE_ID"
+
+agents:
+  queue: {{ .queue }}
+
+secrets:
+  - GCP_E2E_TEST_CREDENTIALS_JSON
+
+steps:
+  - key: upload
+    commands:
+      - set -e
+      - echo "hello world" > artifact.txt
+      - export BUILDKITE_GS_APPLICATION_CREDENTIALS_JSON="$$GCP_E2E_TEST_CREDENTIALS_JSON"
+      - {{ .buildkite_agent_binary }} artifact upload artifact.txt
+  - key: download
+    depends_on: upload
+    commands:
+      - set -e
+      - export BUILDKITE_GS_APPLICATION_CREDENTIALS_JSON="$$GCP_E2E_TEST_CREDENTIALS_JSON"
+      - {{ .buildkite_agent_binary }} artifact download artifact.txt .
+      - if [[ $(cat artifact.txt) == "hello world" ]]; then exit 0; else exit 1; fi

From 9b908c3732bebe736066e5df4796ca290b98f763 Mon Sep 17 00:00:00 2001
From: Giles Gas <gilesgas@gmail.com>
Date: Mon, 15 Dec 2025 18:29:15 +1100
Subject: [PATCH 126/242] Update clicommand/env_get.go

---
 clicommand/env_get.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/env_get.go b/clicommand/env_get.go
index bec61123a1..4b7bc3a519 100644
--- a/clicommand/env_get.go
+++ b/clicommand/env_get.go
@@ -12,7 +12,7 @@ import (
 
 const envClientErrMessage = `Could not create Job API client: %w
 This command can only be used from hooks or plugins running under a job executor
-where the Job API (available in agent v3.64.0 and later) is available.`
+where the agent's job API is available (in version v3.64.0 and later of the Buildkite Agent).`
 
 const envGetHelpDescription = `Usage:
 

From 009af31a94115db6d2b08625ab04fae4dd342480 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Dec 2025 15:03:20 +0000
Subject: [PATCH 127/242] build(deps): bump the otel group with 5 updates

Bumps the otel group with 5 updates:

| Package | From | To |
| --- | --- | --- |
| [go.opentelemetry.io/contrib/propagators/aws](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.38.0` | `1.39.0` |
| [go.opentelemetry.io/contrib/propagators/b3](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.38.0` | `1.39.0` |
| [go.opentelemetry.io/contrib/propagators/jaeger](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.38.0` | `1.39.0` |
| [go.opentelemetry.io/contrib/propagators/ot](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.38.0` | `1.39.0` |
| [go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp](https://github.com/open-telemetry/opentelemetry-go) | `1.38.0` | `1.39.0` |


Updates `go.opentelemetry.io/contrib/propagators/aws` from 1.38.0 to 1.39.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.38.0...v1.39.0)

Updates `go.opentelemetry.io/contrib/propagators/b3` from 1.38.0 to 1.39.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.38.0...v1.39.0)

Updates `go.opentelemetry.io/contrib/propagators/jaeger` from 1.38.0 to 1.39.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.38.0...v1.39.0)

Updates `go.opentelemetry.io/contrib/propagators/ot` from 1.38.0 to 1.39.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.38.0...v1.39.0)

Updates `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` from 1.38.0 to 1.39.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.38.0...v1.39.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/contrib/propagators/aws
  dependency-version: 1.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/contrib/propagators/b3
  dependency-version: 1.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/contrib/propagators/jaeger
  dependency-version: 1.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/contrib/propagators/ot
  dependency-version: 1.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp
  dependency-version: 1.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 10 +++++-----
 go.sum | 20 ++++++++++----------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/go.mod b/go.mod
index 1eb9528aa2..c599d92b42 100644
--- a/go.mod
+++ b/go.mod
@@ -47,13 +47,13 @@ require (
 	github.com/qri-io/jsonschema v0.2.1
 	github.com/stretchr/testify v1.11.1
 	github.com/urfave/cli v1.22.17
-	go.opentelemetry.io/contrib/propagators/aws v1.38.0
-	go.opentelemetry.io/contrib/propagators/b3 v1.38.0
-	go.opentelemetry.io/contrib/propagators/jaeger v1.38.0
-	go.opentelemetry.io/contrib/propagators/ot v1.38.0
+	go.opentelemetry.io/contrib/propagators/aws v1.39.0
+	go.opentelemetry.io/contrib/propagators/b3 v1.39.0
+	go.opentelemetry.io/contrib/propagators/jaeger v1.39.0
+	go.opentelemetry.io/contrib/propagators/ot v1.39.0
 	go.opentelemetry.io/otel v1.39.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0
 	go.opentelemetry.io/otel/sdk v1.39.0
 	go.opentelemetry.io/otel/trace v1.39.0
 	golang.org/x/crypto v0.46.0
diff --git a/go.sum b/go.sum
index f9fafa5e28..c757e70f27 100644
--- a/go.sum
+++ b/go.sum
@@ -449,22 +449,22 @@ go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpo
 go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY=
-go.opentelemetry.io/contrib/propagators/aws v1.38.0 h1:eRZ7asSbLc5dH7+TBzL6hFKb1dabz0IV51uUUwYRZts=
-go.opentelemetry.io/contrib/propagators/aws v1.38.0/go.mod h1:wXqc9NTGcXapBExHBDVLEZlByu6quiQL8w7Tjgv8TCg=
-go.opentelemetry.io/contrib/propagators/b3 v1.38.0 h1:uHsCCOSKl0kLrV2dLkFK+8Ywk9iKa/fptkytc6aFFEo=
-go.opentelemetry.io/contrib/propagators/b3 v1.38.0/go.mod h1:wMRSZJZcY8ya9mApLLhwIMjqmApy2o/Ml+62lhvxyHU=
-go.opentelemetry.io/contrib/propagators/jaeger v1.38.0 h1:nXGeLvT1QtCAhkASkP/ksjkTKZALIaQBIW+JSIw1KIc=
-go.opentelemetry.io/contrib/propagators/jaeger v1.38.0/go.mod h1:oMvOXk78ZR3KEuPMBgp/ThAMDy9ku/eyUVztr+3G6Wo=
-go.opentelemetry.io/contrib/propagators/ot v1.38.0 h1:k4gSyyohaDXI8F9BDXYC3uO2vr5sRNeQFMsN9Zn0EoI=
-go.opentelemetry.io/contrib/propagators/ot v1.38.0/go.mod h1:2hDsuiHRO39SRUMhYGqmj64z/IuMRoxE4bBSFR82Lo8=
+go.opentelemetry.io/contrib/propagators/aws v1.39.0 h1:IvNR8pAVGpkK1CHMjU/YE6B6TlnAPGFvogkMWRWU6wo=
+go.opentelemetry.io/contrib/propagators/aws v1.39.0/go.mod h1:TUsFCERuGM4IGhJG9w+9l0nzmHUKHuaDYYNF6mtNgjY=
+go.opentelemetry.io/contrib/propagators/b3 v1.39.0 h1:PI7pt9pkSnimWcp5sQhUA9OzLbc3Ba4sL+VEUTNsxrk=
+go.opentelemetry.io/contrib/propagators/b3 v1.39.0/go.mod h1:5gV/EzPnfYIwjzj+6y8tbGW2PKWhcsz5e/7twptRVQY=
+go.opentelemetry.io/contrib/propagators/jaeger v1.39.0 h1:Gz3yKzfMSEFzF0Vy5eIpu9ndpo4DhXMCxsLMF0OOApo=
+go.opentelemetry.io/contrib/propagators/jaeger v1.39.0/go.mod h1:2D/cxxCqTlrday0rZrPujjg5aoAdqk1NaNyoXn8FJn8=
+go.opentelemetry.io/contrib/propagators/ot v1.39.0 h1:vKTve1W/WKPVp1fzJamhCDDECt+5upJJ65bPyWoddGg=
+go.opentelemetry.io/contrib/propagators/ot v1.39.0/go.mod h1:FH5VB2N19duNzh1Q8ks6CsZFyu3LFhNLiA9lPxyEkvU=
 go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
 go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 h1:in9O8ESIOlwJAEGTkkf34DesGRAc/Pn8qJ7k3r/42LM=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0/go.mod h1:Rp0EXBm5tfnv0WL+ARyO/PHBEaEAT8UUHQ6AGJcSq6c=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU=
 go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y=
 go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc=
 go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=

From 7e20834bafd2056ccf96e0098b69d7586f24cc5f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Dec 2025 15:05:49 +0000
Subject: [PATCH 128/242] build(deps): bump the cloud-providers group with 2
 updates

Bumps the cloud-providers group with 2 updates: [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2).


Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.276.0 to 1.276.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.276.0...service/ec2/v1.276.1)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.49.3 to 1.49.4
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ssm/v1.49.3...service/ssm/v1.49.4)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.276.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.49.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 1eb9528aa2..d33431975b 100644
--- a/go.mod
+++ b/go.mod
@@ -15,8 +15,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.32.5
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.15
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.0
-	github.com/aws/aws-sdk-go-v2/service/kms v1.49.3
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1
+	github.com/aws/aws-sdk-go-v2/service/kms v1.49.4
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
diff --git a/go.sum b/go.sum
index f9fafa5e28..cdbd0e13b6 100644
--- a/go.sum
+++ b/go.sum
@@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.0 h1:EXwbpkq/tsz1lHI5QRoXjnkZRKgW0Xa+mPSv6Dz/9N0=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.0/go.mod h1:Wg68QRgy2gEGGdmTPU/UbVpdv8sM14bUZmF64KFwAsY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1 h1:P7db/Z55pXvwnueLuHUuVlxnqjbAtiadm01+QIC42OA=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1/go.mod h1:Wg68QRgy2gEGGdmTPU/UbVpdv8sM14bUZmF64KFwAsY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
@@ -106,8 +106,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.3 h1:FtjNR2BaMf/Wu3Um8RNSAfi9An5bEYxJU6f4rLOqSLU=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.3/go.mod h1:HO31s0qt0lso/ADvZQyzKs8js/ku0fMHsfyXW8OPVYc=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.4 h1:2gom8MohxN0SnhHZBYAC4S8jHG+ENEnXjyJ5xKe3vLc=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.4/go.mod h1:HO31s0qt0lso/ADvZQyzKs8js/ku0fMHsfyXW8OPVYc=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2 h1:U3ygWUhCpiSPYSHOrRhb3gOl9T5Y3kB8k5Vjs//57bE=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=

From 2661b9797158e9e3a571141ece068e041baef9c2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Dec 2025 15:05:53 +0000
Subject: [PATCH 129/242] build(deps): bump github.com/DataDog/datadog-go/v5
 from 5.8.1 to 5.8.2

Bumps [github.com/DataDog/datadog-go/v5](https://github.com/DataDog/datadog-go) from 5.8.1 to 5.8.2.
- [Release notes](https://github.com/DataDog/datadog-go/releases)
- [Changelog](https://github.com/DataDog/datadog-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/DataDog/datadog-go/compare/v5.8.1...v5.8.2)

---
updated-dependencies:
- dependency-name: github.com/DataDog/datadog-go/v5
  dependency-version: 5.8.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 1eb9528aa2..af206d9ef4 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	drjosh.dev/zzglob v0.4.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
-	github.com/DataDog/datadog-go/v5 v5.8.1
+	github.com/DataDog/datadog-go/v5 v5.8.2
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go-v2 v1.41.0
 	github.com/aws/aws-sdk-go-v2/config v1.32.5
diff --git a/go.sum b/go.sum
index f9fafa5e28..ac196633fd 100644
--- a/go.sum
+++ b/go.sum
@@ -39,8 +39,8 @@ github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0 h1:aIWF85OKxXGo7rVyqJ
 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0/go.mod h1:Lfap5FuM4b/Pw9IrTuAvWBWZEmXOvZhCya3dYv4G8O0=
 github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyLGxp8E6j2f2Mh5IPbYmQ=
 github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8=
-github.com/DataDog/datadog-go/v5 v5.8.1 h1:+GOES5W9zpKlhwHptZVW2C0NLVf7ilr7pHkDcbNvpIc=
-github.com/DataDog/datadog-go/v5 v5.8.1/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/DataDog/datadog-go/v5 v5.8.2 h1:9IEfH1Mw9AjWwhAMqCAkhbxjuJeMxm2ARX2VdgL+ols=
+github.com/DataDog/datadog-go/v5 v5.8.2/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/DataDog/dd-trace-go/v2 v2.3.0 h1:0Y5kx+Wbod0z8moY0vUbKl6OM0oIV4zAynsVmsq+XT8=
 github.com/DataDog/dd-trace-go/v2 v2.3.0/go.mod h1:yFomJ/rqKNLDbS9ohIDibdz8q9GK0MUSSkBdVDCibGA=
 github.com/DataDog/go-libddwaf/v4 v4.3.2 h1:YGvW2Of1C4e1yU+p7iibmhN2zEOgi9XEchbhQjBxb/A=

From 61350326e94542a40dbe85ce3e3959dfa7759a54 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Dec 2025 15:33:11 +0000
Subject: [PATCH 130/242] build(deps): bump the container-images group across 5
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `02a60dc` to `de2827c`

Updates `buildkite/agent-base` from `ee66745` to `d9a7458`

Updates `buildkite/agent-base` from `057dadc` to `a21105e`

Updates `buildkite/agent-base` from `9fd3c14` to `03e588a`

Updates `buildkite/agent-base` from `6c39f3f` to `b3d073a`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 902af97120..247609a752 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:ee6674501b40b7fbfd1ffc847d9da104f773db019242fcd016c571929506d967
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:d9a74585f8b3621f0b8948215992922bed8aa992e02091ab123c57b0cef0c8fe
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 01ffc239b5..73909e265d 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:02a60dc48f278c8ca70f3eba8b603a4c93ec0acbdfea7202b50f208a954dd831
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:de2827cfa8a68424fef9029130f106cb1c0fdb16a9e193cc4e2ba24a229cc510
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 10edf18fd6..39bc8a7ef6 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:057dadcc269f55136b78ca6c98a5c291eb4c5aadb742e6903fe7dfc973005895
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:a21105ea46eab7b173d423f3b1583707c277dc1537b7d9ff1b150982ada6989f
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index ebbfa66fb6..a588529231 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:9fd3c14e80076afa8576816007e07526fceaf6e9a03e4eae14aea9de8c0b90cd
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:03e588ac87c55c3191c254be767b3b2e57fc499db3f603a93b4b5857e93a46fc
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index 4335439d3e..bfb5ea99af 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:6c39f3f4e0b17045c5f35d1a20154198705d3134418a44cf6fbb12753663438c
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:b3d073a3013d5cd3be46ba01c4a083da92f6dcf5de7622365fd50b5ec624bb04
 ARG TARGETOS
 ARG TARGETARCH
 

From f327a3cf9f2cbb62bb2c0b32dd54508c1345b996 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Dec 2025 15:33:27 +0000
Subject: [PATCH 131/242] build(deps): bump docker/library/golang

Bumps the container-images group with 1 update in the /.buildkite directory: docker/library/golang.


Updates `docker/library/golang` from `cf1272d` to `54528d1`

---
updated-dependencies:
- dependency-name: docker/library/golang
  dependency-version: 1.24.11
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-compile | 2 +-
 .buildkite/Dockerfile-e2e     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index 9a07789ae0..e215645152 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:cf1272dbf972a94f39a81dcb9dc243a8d2f981e5dd3b5a5c965f6d9ab9268b26
+FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:54528d189affe7ce60840cad093d53360705d47d71ea74d62eef72c476242fe9
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest
diff --git a/.buildkite/Dockerfile-e2e b/.buildkite/Dockerfile-e2e
index ae3d728402..fa23d09f6a 100644
--- a/.buildkite/Dockerfile-e2e
+++ b/.buildkite/Dockerfile-e2e
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:cf1272dbf972a94f39a81dcb9dc243a8d2f981e5dd3b5a5c965f6d9ab9268b26
+FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:54528d189affe7ce60840cad093d53360705d47d71ea74d62eef72c476242fe9
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     unzip \

From 887d593e9ef2360997c98cb3a6feda7a9e3af370 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 16 Dec 2025 15:03:35 +1100
Subject: [PATCH 132/242] Add E2E test for Azure Blob storage

---
 internal/e2e/artifact_test.go                 | 16 ++++++++++++++
 .../artifact_custom_azure_storage.yaml        | 21 +++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 internal/e2e/fixtures/artifact_custom_azure_storage.yaml

diff --git a/internal/e2e/artifact_test.go b/internal/e2e/artifact_test.go
index 39013ccf77..636d5bbf37 100644
--- a/internal/e2e/artifact_test.go
+++ b/internal/e2e/artifact_test.go
@@ -53,3 +53,19 @@ func TestArtifactUploadDownload_GCS(t *testing.T) {
 		t.Errorf("Build state = %q, want %q", got, want)
 	}
 }
+
+// Test that we can upload/downdload artifact using a custom Azure Blob storage
+// container.
+// Everything that gets uploaded here gets auto removed in 30 days.
+func TestArtifactUploadDownload_Azure(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, "artifact_custom_azure_storage.yaml")
+
+	tc.startAgent()
+	build := tc.triggerBuild()
+	state := tc.waitForBuild(ctx, build)
+
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+}
diff --git a/internal/e2e/fixtures/artifact_custom_azure_storage.yaml b/internal/e2e/fixtures/artifact_custom_azure_storage.yaml
new file mode 100644
index 0000000000..0f7010b96d
--- /dev/null
+++ b/internal/e2e/fixtures/artifact_custom_azure_storage.yaml
@@ -0,0 +1,21 @@
+env:
+  BUILDKITE_ARTIFACT_UPLOAD_DESTINATION: "https://l0srwtxpsx7s9h26qra8a40j.blob.core.windows.net/buildkite-agent-e2e-test/${BUILDKITE_PIPELINE_ID}"
+
+agents:
+  queue: {{ .queue }}
+
+secrets:
+  - AZURE_E2E_TEST_KEY
+
+steps:
+  - key: upload
+    commands: |
+      set -e
+      echo "hello world" > artifact.txt
+      BUILDKITE_AZURE_BLOB_ACCESS_KEY="$${AZURE_E2E_TEST_KEY}" {{ .buildkite_agent_binary }} artifact upload artifact.txt
+  - key: download
+    depends_on: upload
+    commands: |
+      set -e
+      BUILDKITE_AZURE_BLOB_ACCESS_KEY="$${AZURE_E2E_TEST_KEY}" {{ .buildkite_agent_binary }} artifact download artifact.txt .
+      [[ $(cat artifact.txt) == "hello world" ]]

From 93d0904612783bdcd673e12fb8dac8bf1a42f2d9 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 17 Dec 2025 11:52:08 +1100
Subject: [PATCH 133/242] Avoid overriding BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH

---
 agent/job_runner.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index f4c6e3cd20..d71a7d974c 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -579,7 +579,11 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
 	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
 	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
-	setEnv("BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", fmt.Sprint(r.conf.AgentConfiguration.PluginsAlwaysCloneFresh))
+	// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config
+	// or by pipeline/step env.
+	if r.conf.AgentConfiguration.PluginsAlwaysCloneFresh {
+		setEnv("BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", "true")
+	}
 	setEnv("BUILDKITE_LOCAL_HOOKS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled))
 
 	setEnv("BUILDKITE_GIT_CHECKOUT_FLAGS", r.conf.AgentConfiguration.GitCheckoutFlags)

From de9ade1365d48cc3e80bef16fa29654b0b601bbc Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 17 Dec 2025 11:52:08 +1100
Subject: [PATCH 134/242] Nested-loop jitter structure for log processing

---
 agent/run_job.go | 105 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 75 insertions(+), 30 deletions(-)

diff --git a/agent/run_job.go b/agent/run_job.go
index e28ea52aa6..09296ac3be 100644
--- a/agent/run_job.go
+++ b/agent/run_job.go
@@ -453,26 +453,46 @@ func (r *JobRunner) streamJobLogsAfterProcessStart(ctx context.Context, wg *sync
 	if r.conf.Job.ChunksIntervalSeconds > 0 {
 		processInterval = time.Duration(r.conf.Job.ChunksIntervalSeconds) * time.Second
 	}
-	intervalTicker := time.NewTicker(processInterval)
-	defer intervalTicker.Stop()
-	first := make(chan struct{}, 1)
-	first <- struct{}{}
 
+	// We want log chunks to be uploaded regularly. If there were only a few
+	// agents in the world, a simple ticker loop would be fine. But there are a
+	// lot of agents, and we want to spread the requests over time. Some things
+	// to consider when designing such a loop:
+	//
+	// - Large groups of agents all starting the loop at the same time
+	// - Clock drift or backend issues causing the loops among agents to start
+	//   synchronising
+	// - Statistical reasons for agents tending to synchronise
+	// - Having loose or tight bounds on the time between requests
+	//
+	// In this case we want both a lower bound, because fewer larger chunks are
+	// more efficient than lots of smaller chunks, but also we probably want an
+	// upper bound, to improve the experience of tailing a log in the UI.
+	//
+	// Below is a loop within a loop. The inner loop is just a plain ticker loop
+	// that processes chunks once per interval pretty much exactly.
+	// The outer loop periodically restarts the inner loop at a random offset
+	// in time (jitter).
+	// Periodically applying jitter prevents large numbers of agents from
+	// synchronising, but only doing so every so often makes the time between
+	// requests regular (most of the time).
+	//
+	// 32 intervals is an arbitrarily chosen amount of time between jittering.
+	// Increasing it will make it more susceptible to spontaneous
+	// synchronsiation problems like clock drift,
+	// decreasing it will add more variability but also add more "large gaps"
+	// at the point where jitter is added.
+	//
+	// The outer loop repeats every 33 intervals, in order to fit both the inner
+	// loop (32 intervals) and the jitter (between 0 and 1 interval):
+	//   32 intervals < (jitter + inner loop) < 33 intervals
+	// So the gap between requests will usually be 1 interval, but occasionally
+	// be longer (up to 2 intervals).
+
+	const runLength = 32
+	rejitterTicker := time.Tick((runLength + 1) * processInterval)
 	for {
-		setStat("😴 Waiting for next log processing interval tick")
-		select {
-		case <-first:
-			// continue below
-		case <-intervalTicker.C:
-			// continue below
-		case <-ctx.Done():
-			return
-		case <-r.process.Done():
-			return
-		}
-
-		// Within the interval, wait a random amount of time to avoid
-		// spontaneous synchronisation across agents.
+		// The inner loop starts at a random offset within an interval.
 		jitter := rand.N(processInterval)
 		setStat(fmt.Sprintf("🫨 Jittering for %v", jitter))
 		select {
@@ -484,19 +504,44 @@ func (r *JobRunner) streamJobLogsAfterProcessStart(ctx context.Context, wg *sync
 			return
 		}
 
-		setStat("📨 Sending process output to log streamer")
-
-		// Send the output of the process to the log streamer for processing
-		if err := r.logStreamer.Process(ctx, r.output.ReadAndTruncate()); err != nil {
-			r.agentLogger.Error("Could not stream the log output: %v", err)
-			// LogStreamer.Process only returns an error when it can no longer
-			// accept logs (maybe Stop was called, or a hard limit was reached).
-			// Since we can no longer send logs, Close the buffer, which causes
-			// future Writes to return io.ErrClosedPipe, typically SIGPIPE-ing
-			// the running process (if it is still running).
-			if err := r.output.Close(); err != nil && err != process.ErrAlreadyClosed {
-				r.agentLogger.Error("Process output buffer could not be closed: %v", err)
+		// The inner loop processes once per interval pretty much exactly.
+		intervalTicker := time.Tick(processInterval)
+		for range runLength {
+			setStat("📨 Sending process output to log streamer")
+
+			// Send the output of the process to the log streamer for processing
+			if err := r.logStreamer.Process(ctx, r.output.ReadAndTruncate()); err != nil {
+				r.agentLogger.Error("Could not stream the log output: %v", err)
+				// LogStreamer.Process only returns an error when it can no longer
+				// accept logs (maybe Stop was called, or a hard limit was reached).
+				// Since we can no longer send logs, Close the buffer, which causes
+				// future Writes to return io.ErrClosedPipe, typically SIGPIPE-ing
+				// the running process (if it is still running).
+				if err := r.output.Close(); err != nil && err != process.ErrAlreadyClosed {
+					r.agentLogger.Error("Process output buffer could not be closed: %v", err)
+				}
+				return
 			}
+
+			setStat("😴 Waiting for next log processing interval tick")
+			select {
+			case <-intervalTicker:
+				// continue next loop iteration
+			case <-ctx.Done():
+				return
+			case <-r.process.Done():
+				return
+			}
+		}
+
+		// Start the next run on a fixed schedule (for statistical reasons).
+		setStat("😴 Waiting for next re-jittering interval tick")
+		select {
+		case <-rejitterTicker:
+			// continue next loop iteration
+		case <-ctx.Done():
+			return
+		case <-r.process.Done():
 			return
 		}
 	}

From 8bbf552752e7f692f8a4923cda46fd92dcdf60bf Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 18 Dec 2025 13:58:11 +1100
Subject: [PATCH 135/242] Bump version and changelog for v3.115.2

---
 CHANGELOG.md    | 22 ++++++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2d797ef57b..4f041cc21c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.115.2](https://github.com/buildkite/agent/tree/v3.115.2) (2025-12-18)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.115.1...v3.115.2)
+
+### Fixed
+- Try to avoid overriding BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH with false [#3644](https://github.com/buildkite/agent/pull/3644) (@DrJosh9000)
+- SUP-5826: Remove experiment from 'env' command [#3635](https://github.com/buildkite/agent/pull/3635) (@Mykematt)
+
+### Internal
+- Nested-loop jitter structure for log processing [#3645](https://github.com/buildkite/agent/pull/3645) (@DrJosh9000)
+- Add E2E test for Azure Blob storage [#3642](https://github.com/buildkite/agent/pull/3642) (@DrJosh9000)
+- PB-1007: add e2e test for gcs artifact upload/download [#3633](https://github.com/buildkite/agent/pull/3633) (@zhming0)
+- PB-1025: improve e2e test DevEX [#3634](https://github.com/buildkite/agent/pull/3634) (@zhming0)
+
+### Dependency updates
+- chore(deps): bump zstash to v0.7.0 [#3632](https://github.com/buildkite/agent/pull/3632) (@wolfeidau)
+- build(deps): bump the cloud-providers group with 2 updates [#3638](https://github.com/buildkite/agent/pull/3638) (@dependabot[bot])
+- build(deps): bump the otel group with 5 updates [#3637](https://github.com/buildkite/agent/pull/3637) (@dependabot[bot])
+- build(deps): bump github.com/DataDog/datadog-go/v5 from 5.8.1 to 5.8.2 [#3639](https://github.com/buildkite/agent/pull/3639) (@dependabot[bot])
+- build(deps): bump the container-images group across 5 directories with 1 update [#3640](https://github.com/buildkite/agent/pull/3640) (@dependabot[bot])
+- build(deps): bump docker/library/golang from `cf1272d` to `54528d1` in /.buildkite in the container-images group across 1 directory [#3641](https://github.com/buildkite/agent/pull/3641) (@dependabot[bot])
+
+
 ## [v3.115.1](https://github.com/buildkite/agent/tree/v3.115.1) (2025-12-12)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.115.0...v3.115.1)
 
diff --git a/version/VERSION b/version/VERSION
index f5563946b1..b9d0f99561 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.115.1
+3.115.2

From 0165053920bb6ca9cb844120305af99f342e0244 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Dec 2025 15:07:11 +0000
Subject: [PATCH 136/242] build(deps): bump the cloud-providers group with 5
 updates

Bumps the cloud-providers group with 5 updates:

| Package | From | To |
| --- | --- | --- |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.5` | `1.32.6` |
| [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.20.15` | `1.20.17` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.276.1` | `1.278.0` |
| [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.93.2` | `1.94.0` |
| [google.golang.org/api](https://github.com/googleapis/google-api-go-client) | `0.257.0` | `0.258.0` |


Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.5 to 1.32.6
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.32.5...v1.32.6)

Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.20.15 to 1.20.17
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/emr/v1.20.15...service/emr/v1.20.17)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.276.1 to 1.278.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.276.1...service/ec2/v1.278.0)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.93.2 to 1.94.0
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.93.2...service/s3/v1.94.0)

Updates `google.golang.org/api` from 0.257.0 to 0.258.0
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.257.0...v0.258.0)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.32.6
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-version: 1.20.17
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.278.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-version: 1.94.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: google.golang.org/api
  dependency-version: 0.258.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 18 +++++++++---------
 go.sum | 36 ++++++++++++++++++------------------
 2 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/go.mod b/go.mod
index b39ee5712b..2acfa82cb4 100644
--- a/go.mod
+++ b/go.mod
@@ -12,12 +12,12 @@ require (
 	github.com/DataDog/datadog-go/v5 v5.8.2
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go-v2 v1.41.0
-	github.com/aws/aws-sdk-go-v2/config v1.32.5
+	github.com/aws/aws-sdk-go-v2/config v1.32.6
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.15
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.278.0
 	github.com/aws/aws-sdk-go-v2/service/kms v1.49.4
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
@@ -62,7 +62,7 @@ require (
 	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.39.0
 	golang.org/x/term v0.38.0
-	google.golang.org/api v0.257.0
+	google.golang.org/api v0.258.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
@@ -96,7 +96,7 @@ require (
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.5 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -106,7 +106,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -204,9 +204,9 @@ require (
 	golang.org/x/tools v0.39.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
 	google.golang.org/grpc v1.77.0 // indirect
-	google.golang.org/protobuf v1.36.10 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/gotestsum v1.13.0 // indirect
diff --git a/go.sum b/go.sum
index 6dcf374860..c025f07505 100644
--- a/go.sum
+++ b/go.sum
@@ -80,14 +80,14 @@ github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgP
 github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.5 h1:pz3duhAfUgnxbtVhIK39PGF/AHYyrzGEyRD9Og0QrE8=
-github.com/aws/aws-sdk-go-v2/config v1.32.5/go.mod h1:xmDjzSUs/d0BB7ClzYPAZMmgQdrodNjPPhd6bGASwoE=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk=
+github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
+github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.15 h1:Zn4SfxkULorRqLg/VhxQ5cg9bi8Qhq7Y8W9RUew15oI=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.15/go.mod h1:uFphWOp8hzgUQ6ORHAw2WUf2xeqOWHjhgCDSdAVxzp0=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17 h1:fODjlj9c1zIfZYFxdC6Z4GX/plrZUYI/5EklgA/24Hw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17/go.mod h1:CEyBu8kavY5Tc8i/8A810DuKydd19Lrx2/TmcNdjOAk=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
@@ -96,8 +96,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEG
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1 h1:P7db/Z55pXvwnueLuHUuVlxnqjbAtiadm01+QIC42OA=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.276.1/go.mod h1:Wg68QRgy2gEGGdmTPU/UbVpdv8sM14bUZmF64KFwAsY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.278.0 h1:Cx/Rs2zaG30Dn4QMvUGC5rCAZagA8heta0TWAdBE/Xc=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.278.0/go.mod h1:Wg68QRgy2gEGGdmTPU/UbVpdv8sM14bUZmF64KFwAsY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
@@ -108,12 +108,12 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lu
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
 github.com/aws/aws-sdk-go-v2/service/kms v1.49.4 h1:2gom8MohxN0SnhHZBYAC4S8jHG+ENEnXjyJ5xKe3vLc=
 github.com/aws/aws-sdk-go-v2/service/kms v1.49.4/go.mod h1:HO31s0qt0lso/ADvZQyzKs8js/ku0fMHsfyXW8OPVYc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2 h1:U3ygWUhCpiSPYSHOrRhb3gOl9T5Y3kB8k5Vjs//57bE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.93.2/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 h1:SWTxh/EcUCDVqi/0s26V6pVUq0BBG7kx0tDTmF/hCgA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
@@ -554,20 +554,20 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.257.0 h1:8Y0lzvHlZps53PEaw+G29SsQIkuKrumGWs9puiexNAA=
-google.golang.org/api v0.257.0/go.mod h1:4eJrr+vbVaZSqs7vovFd1Jb/A6ml6iw2e6FBYf3GAO4=
+google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
+google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
 google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
 google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
 google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
 google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 h1:h96ji92t9eXbPvSWhJ+lrPWetHiQNYlt48JKRO09NFA=
 gopkg.in/DataDog/dd-trace-go.v1 v1.74.8/go.mod h1:LpHbtHsCZBlm1HWrlVOUQcEXwMWZnU6yMvmtd1GvSDI=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

From 8a56f1f882ee76d0167af7d58c6fa9ea2baa86b7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Dec 2025 15:07:25 +0000
Subject: [PATCH 137/242] build(deps): bump
 github.com/buildkite/go-buildkite/v4

Bumps [github.com/buildkite/go-buildkite/v4](https://github.com/buildkite/go-buildkite) from 4.11.0 to 4.13.1.
- [Release notes](https://github.com/buildkite/go-buildkite/releases)
- [Changelog](https://github.com/buildkite/go-buildkite/blob/main/CHANGELOG.md)
- [Commits](https://github.com/buildkite/go-buildkite/compare/v4.11.0...v4.13.1)

---
updated-dependencies:
- dependency-name: github.com/buildkite/go-buildkite/v4
  dependency-version: 4.13.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index b39ee5712b..168cf71b9b 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
-	github.com/buildkite/go-buildkite/v4 v4.11.0
+	github.com/buildkite/go-buildkite/v4 v4.13.1
 	github.com/buildkite/go-pipeline v0.16.0
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
diff --git a/go.sum b/go.sum
index 6dcf374860..ac573b66b8 100644
--- a/go.sum
+++ b/go.sum
@@ -132,8 +132,8 @@ github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf/go.mod h1:CeKhh8xSs3WZAc50xABMxu+FlfAAd5PNumo7NfOv7EE=
 github.com/buildkite/bintest/v3 v3.3.0 h1:RTWcSaJRlOT6t/K311ejPf+0J3LE/QEODzVG3vlLnWo=
 github.com/buildkite/bintest/v3 v3.3.0/go.mod h1:btqpTsVODiJcb0NMdkkmtMQ6xoFc2W/nY5yy+3I0zcs=
-github.com/buildkite/go-buildkite/v4 v4.11.0 h1:rEvvUwITrqv433W9JWf6mj+NkkcM45s+ObhNs6C17i4=
-github.com/buildkite/go-buildkite/v4 v4.11.0/go.mod h1:DlebrRJqpZttXDjCW+MJ1QyW9AN++ZWt/UbPtKdbSSk=
+github.com/buildkite/go-buildkite/v4 v4.13.1 h1:3PhrShdQwlZ2+OIWy0QbxbbjP6M97NCvarlIB6Oye+k=
+github.com/buildkite/go-buildkite/v4 v4.13.1/go.mod h1:DlebrRJqpZttXDjCW+MJ1QyW9AN++ZWt/UbPtKdbSSk=
 github.com/buildkite/go-pipeline v0.16.0 h1:wEgWUMRAgSg1ZnWOoA3AovtYYdTvN0dLY1zwUWmPP+4=
 github.com/buildkite/go-pipeline v0.16.0/go.mod h1:VE37qY3X5pmAKKUMoDZvPsHOQuyakB9cmXj9Qn6QasA=
 github.com/buildkite/interpolate v0.1.5 h1:v2Ji3voik69UZlbfoqzx+qfcsOKLA61nHdU79VV+tPU=

From 06b4064e7ebdd203e5b86b0433aa9519c9ada440 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Dec 2025 15:24:41 +0000
Subject: [PATCH 138/242] build(deps): bump the container-images group across 4
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `de2827c` to `dc83378`

Updates `buildkite/agent-base` from `d9a7458` to `12dcb2d`

Updates `buildkite/agent-base` from `a21105e` to `dc768a6`

Updates `buildkite/agent-base` from `03e588a` to `a0ba149`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 247609a752..56c1bb1968 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:d9a74585f8b3621f0b8948215992922bed8aa992e02091ab123c57b0cef0c8fe
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:12dcb2dbc09a70210babdf0e221a4448127009cfb136e41b9a34fb4a79e59b84
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 73909e265d..0c66dab44a 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:de2827cfa8a68424fef9029130f106cb1c0fdb16a9e193cc4e2ba24a229cc510
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:dc8337889813c0787bda5fd4119fdcdb1a9fb9151af16e896a7c2ab31eab141f
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 39bc8a7ef6..8414859520 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:a21105ea46eab7b173d423f3b1583707c277dc1537b7d9ff1b150982ada6989f
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:dc768a639d18edb9ef2722881af6b59938f818b6594b2cf14996a0add2edc850
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index a588529231..e9fc21ca6b 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:03e588ac87c55c3191c254be767b3b2e57fc499db3f603a93b4b5857e93a46fc
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:a0ba149ebf4828cdd6a643f97cc1e6a61ce44698bba4ae84ed8a38170cd9bc8f
 ARG TARGETOS
 ARG TARGETARCH
 

From a52f5c0da59a7601169a4a39b4d6c7136c52b8fe Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Dec 2025 23:21:44 +0000
Subject: [PATCH 139/242] build(deps): bump github.com/buildkite/zstash from
 0.7.0 to 0.8.0

Bumps [github.com/buildkite/zstash](https://github.com/buildkite/zstash) from 0.7.0 to 0.8.0.
- [Release notes](https://github.com/buildkite/zstash/releases)
- [Commits](https://github.com/buildkite/zstash/compare/v0.7.0...v0.8.0)

---
updated-dependencies:
- dependency-name: github.com/buildkite/zstash
  dependency-version: 0.8.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 6d54215ddb..f8e6df72a2 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
-	github.com/buildkite/zstash v0.7.0
+	github.com/buildkite/zstash v0.8.0
 	github.com/creack/pty v1.1.19
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
diff --git a/go.sum b/go.sum
index e003a4f0b5..452e1bec59 100644
--- a/go.sum
+++ b/go.sum
@@ -144,8 +144,8 @@ github.com/buildkite/shellwords v1.0.1 h1:88OjMbEBf+EliVB0tizXJynpAM2CKOvYwepg5n
 github.com/buildkite/shellwords v1.0.1/go.mod h1:so0eQnTxgbo58CTYX+4BCx5UuMzvRha9dcKdCKl6NV4=
 github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI/sx4XP1xmFesE=
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
-github.com/buildkite/zstash v0.7.0 h1:lKDGLaRjdjh7M0ItRfpy9HIUv02r1bszKC/8fupDjXY=
-github.com/buildkite/zstash v0.7.0/go.mod h1:BGBukIwUaM3r/ypdQWyaDIP3izQj1PSauoAS6fOaYwg=
+github.com/buildkite/zstash v0.8.0 h1:z8hBGGKIaN/UX0MZh51P8s28mWaDBkgD3uMoOu6Q96g=
+github.com/buildkite/zstash v0.8.0/go.mod h1:60v4pl8PnsVA6b09MC47aTR8Z1QNJoeQqOGp2jkEww8=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf h1:yxlp0s+Sge9UsKEK0Bsvjiopb9XRk+vxylmZ9eGBfm8=
 github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=

From 54b2e6f99e3d49b6f353156b33e374295cc549b9 Mon Sep 17 00:00:00 2001
From: Christian Navarrete <33426000+chrisnavar@users.noreply.github.com>
Date: Wed, 7 Jan 2026 09:32:25 +0100
Subject: [PATCH 140/242] chore: Modified mktemp command for tarball extraction
 on MacStadium (#6)

On MacStadium VMs, the default mktemp is the BSD/macOS version, which does not support the -p option. Even on MacStadium, unless you have specifically installed GNU coreutils and are using gmktemp, mktemp -p will fail. This approach is safe and works in both systems, as long as $DESTINATION exists and is writable. tmp.XXXXXX will create a unique temp directory inside $DESTINATION and avoid the error when the folder already exists.
---
 install.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install.sh b/install.sh
index de9467bc48..6edc7c6496 100755
--- a/install.sh
+++ b/install.sh
@@ -166,7 +166,7 @@ fi
 
 mkdir -p "${DESTINATION}/bin"   # for the binary
 mkdir -p "${DESTINATION}/hooks" # for hooks
-INSTALL_TMP="$(mktemp -d -p "${DESTINATION}")" # for tarball extraction
+INSTALL_TMP="$(mktemp -d "${DESTINATION}/tmp.XXXXXX")" # for tarball extraction
 trap 'rm -fr ${INSTALL_TMP}' EXIT
 
 echo -e "Destination: \033[35m${DESTINATION}\033[0m"

From 630dfd0d308a86be29485929a583c91c1696cbc4 Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Thu, 8 Jan 2026 09:51:19 +1100
Subject: [PATCH 141/242] PS-1525: keep BUILDKITE_KUBERNETES_EXEC true for k8s
 bootstrap

---
 clicommand/agent_start.go          | 10 +++++++++-
 clicommand/global.go               |  8 --------
 clicommand/kubernetes_bootstrap.go |  8 +++++---
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 571ef31407..f2a4ce7e45 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -747,10 +747,18 @@ var AgentStartCommand = cli.Command{
 		DebugHTTPFlag,
 		TraceHTTPFlag,
 
+		// Kubernetes
+		cli.BoolFlag{
+			Name: "kubernetes-exec",
+			Usage: "This is intended to be used only by the Buildkite k8s stack " +
+				"(github.com/buildkite/agent-stack-k8s); it enables a Unix socket for transporting " +
+				"logs and exit statuses between containers in a pod",
+			EnvVar: "BUILDKITE_KUBERNETES_EXEC",
+		},
+
 		// Other shared flags
 		RedactedVars,
 		StrictSingleHooksFlag,
-		KubernetesExecFlag,
 		TraceContextEncodingFlag,
 		NoMultipartArtifactUploadFlag,
 
diff --git a/clicommand/global.go b/clicommand/global.go
index c33231b421..559aa9eab5 100644
--- a/clicommand/global.go
+++ b/clicommand/global.go
@@ -100,14 +100,6 @@ var (
 		EnvVar: "BUILDKITE_SOCKETS_PATH",
 	}
 
-	KubernetesExecFlag = cli.BoolFlag{
-		Name: "kubernetes-exec",
-		Usage: "This is intended to be used only by the Buildkite k8s stack " +
-			"(github.com/buildkite/agent-stack-k8s); it enables a Unix socket for transporting " +
-			"logs and exit statuses between containers in a pod",
-		EnvVar: "BUILDKITE_KUBERNETES_EXEC",
-	}
-
 	KubernetesContainerIDFlag = cli.IntFlag{
 		Name: "kubernetes-container-id",
 		Usage: "This is intended to be used only by the Buildkite k8s stack " +
diff --git a/clicommand/kubernetes_bootstrap.go b/clicommand/kubernetes_bootstrap.go
index ebe42ae60b..d9b5bac67e 100644
--- a/clicommand/kubernetes_bootstrap.go
+++ b/clicommand/kubernetes_bootstrap.go
@@ -123,9 +123,11 @@ var KubernetesBootstrapCommand = cli.Command{
 			return err
 		}
 
-		// Ensure the Kubernetes socket setup is disabled in the subprocess
-		// (we're doing all that here).
-		environ.Set("BUILDKITE_KUBERNETES_EXEC", "false")
+		// BUILDKITE_KUBERNETES_EXEC is a legacy environment variable. It was used to activate the socket
+		// on the bootstrap command, and to activate the socket server on `buildkite-agent start`.
+		// The former has been superseded by this `kubernetes-bootstrap` command.
+		// We keep this env var because some users depend on it as a k8s environment detection mechanism.
+		environ.Set("BUILDKITE_KUBERNETES_EXEC", "true")
 
 		if _, exists := environ.Get("BUILDKITE_BUILD_CHECKOUT_PATH"); !exists {
 			// The OG agent runs as a long-live worker, therefore it set a checkout path dynamically to cater

From 0df1c3ffcda4b2c954ef41d7e0756794d47ff8cd Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Thu, 8 Jan 2026 16:09:55 +1100
Subject: [PATCH 142/242] release 3.115.3

---
 CHANGELOG.md    | 10 ++++++++++
 version/VERSION |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4f041cc21c..37a8ce01ec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.115.3](https://github.com/buildkite/agent/tree/v3.115.3) (2026-01-08)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.115.2...v3.115.3)
+
+### Changed
+- PS-1525: keep BUILDKITE_KUBERNETES_EXEC true for k8s bootstrap [#3658](https://github.com/buildkite/agent/pull/3658) (@zhming0)
+
+### Internal
+- Dependencies updates: [#3649](https://github.com/buildkite/agent/pull/3649), [#3651](https://github.com/buildkite/agent/pull/3651), [#3650](https://github.com/buildkite/agent/pull/3650), [#3648](https://github.com/buildkite/agent/pull/3648) (@dependabot[bot])
+
+
 ## [v3.115.2](https://github.com/buildkite/agent/tree/v3.115.2) (2025-12-18)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.115.1...v3.115.2)
 
diff --git a/version/VERSION b/version/VERSION
index b9d0f99561..d516e3a4d8 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.115.2
+3.115.3

From 52d860124610e07fbd4405cb7532642647f523e6 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Fri, 2 Jan 2026 16:38:48 +1100
Subject: [PATCH 143/242] Refactor plugin config -> envar generation

I was in this code investigating an escalation, and found it quite hard to parse out what was going on, so i did a couple of light refactors to make it a little easier to follow
---
 agent/plugin/plugin.go | 86 +++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 48 deletions(-)

diff --git a/agent/plugin/plugin.go b/agent/plugin/plugin.go
index 78bd705638..6a95055e15 100644
--- a/agent/plugin/plugin.go
+++ b/agent/plugin/plugin.go
@@ -222,17 +222,17 @@ func formatEnvKey(key string) string {
 	return hypenOrSpaceRE.ReplaceAllString(newKey, "_")
 }
 
-func walkConfigValues(prefix string, v any, into *[]string) error {
+func flattenConfigToEnvMap(into map[string]string, v any, envPrefix string) error {
 	switch vv := v.(type) {
 	// handles all of our primitive types, golang provides a good string representation
 	case string, bool, json.Number:
-		*into = append(*into, fmt.Sprintf("%s=%v", prefix, vv))
+		into[envPrefix] = fmt.Sprintf("%v", vv)
 		return nil
 
 	// handle lists of things, which get a KEY_N prefix depending on the index
 	case []any:
-		for i := range vv {
-			if err := walkConfigValues(fmt.Sprintf("%s_%d", prefix, i), vv[i], into); err != nil {
+		for i, item := range vv {
+			if err := flattenConfigToEnvMap(into, item, fmt.Sprintf("%s_%d", envPrefix, i)); err != nil {
 				return err
 			}
 		}
@@ -241,72 +241,62 @@ func walkConfigValues(prefix string, v any, into *[]string) error {
 	// handle maps of things, which get a KEY_SUBKEY prefix depending on the map keys
 	case map[string]any:
 		for k, vvv := range vv {
-			if err := walkConfigValues(fmt.Sprintf("%s_%s", prefix, formatEnvKey(k)), vvv, into); err != nil {
+			if err := flattenConfigToEnvMap(into, vvv, fmt.Sprintf("%s_%s", envPrefix, formatEnvKey(k))); err != nil {
 				return err
 			}
 		}
 		return nil
-	}
 
-	return fmt.Errorf("Unknown type %T %v", v, v)
+	default:
+		return fmt.Errorf("Unknown type %T %v", v, v)
+	}
 }
 
-// The input should be a slice of Env Variables of the form `k=v` where `k` is the variable name
-// and `v` is its value. If it can be determined that any of the env variables names is part of a
-// deprecation, either as the deprecated variable name or its replacement, append both to the
-// returned slice, and also append a deprecation error to the error value. If there are no
-// deprecations, the error value is guaranteed to be nil.
-func fanOutDeprecatedEnvVarNames(envSlice []string) ([]string, error) {
-	envSliceAfter := envSlice
-
-	var dnerrs *DeprecatedNameErrors
-	for _, kv := range envSlice {
-		k, v, ok := strings.Cut(kv, "=")
-		if !ok { // this is impossible if the precondition is met
-			continue
-		}
-
+// addDeprecatedEnvVarAliases provides backward compatibility for environment variable names.
+//
+// Before v3.48.0 (https://github.com/buildkite/agent/pull/2116), consecutive underscores in
+// derived env var names were collapsed (e.g., "some--key__name" → SOME_KEY_NAME).
+// Since v3.48.0, consecutive underscores are preserved (e.g., SOME__KEY__NAME).
+//
+// For compatibility, this function adds the collapsed form for any key containing consecutive
+// underscores and returns deprecation errors listing the affected variables.
+func addDeprecatedEnvVarAliases(envMap map[string]string) error {
+	var errs *DeprecatedNameErrors
+	for k, v := range envMap {
 		// the form with consecutive underscores is replacing the form without, but the replacement
-		// is what is expected to be in input slice
-		noConsecutiveUnderScoreKey := consecutiveUnderscoreRE.ReplaceAllString(k, "_")
-		if k != noConsecutiveUnderScoreKey {
-			envSliceAfter = append(envSliceAfter, fmt.Sprintf("%s=%s", noConsecutiveUnderScoreKey, v))
-			dnerrs = dnerrs.Append(DeprecatedNameError{old: noConsecutiveUnderScoreKey, new: k})
+		// is what is expected to be in input map
+		withoutConsecutiveUnderscores := consecutiveUnderscoreRE.ReplaceAllString(k, "_")
+		if k != withoutConsecutiveUnderscores {
+			envMap[withoutConsecutiveUnderscores] = v
+			errs = errs.Append(DeprecatedNameError{old: withoutConsecutiveUnderscores, new: k})
 		}
 	}
 
-	// guarantee that the error value is nil if there are no deprecations
-	if !dnerrs.IsEmpty() {
-		return envSliceAfter, dnerrs
+	if !errs.IsEmpty() {
+		return errs
 	}
-	return envSliceAfter, nil
+
+	return nil
 }
 
-// ConfigurationToEnvironment converts the plugin configuration values to
-// environment variables.
+// ConfigurationToEnvironment converts the plugin configuration values to environment variables.
 func (p *Plugin) ConfigurationToEnvironment() (*env.Environment, error) {
-	envSlice := []string{}
-	envPrefix := fmt.Sprintf("BUILDKITE_PLUGIN_%s", formatEnvKey(p.Name()))
-
-	// Append current plugin name
-	envSlice = append(envSlice, fmt.Sprintf("BUILDKITE_PLUGIN_NAME=%s", formatEnvKey(p.Name())))
-
-	// Append current plugin configuration as JSON
 	configJSON, err := json.Marshal(p.Configuration)
 	if err != nil {
 		return env.New(), err
 	}
-	envSlice = append(envSlice, fmt.Sprintf("BUILDKITE_PLUGIN_CONFIGURATION=%s", configJSON))
 
-	for k, v := range p.Configuration {
-		configPrefix := fmt.Sprintf("%s_%s", envPrefix, formatEnvKey(k))
-		if err := walkConfigValues(configPrefix, v, &envSlice); err != nil {
-			return env.New(), err
-		}
+	envMap := make(map[string]string)
+	envMap["BUILDKITE_PLUGIN_NAME"] = formatEnvKey(p.Name())
+	envMap["BUILDKITE_PLUGIN_CONFIGURATION"] = string(configJSON)
+
+	envPrefix := fmt.Sprintf("BUILDKITE_PLUGIN_%s", formatEnvKey(p.Name()))
+	if err := flattenConfigToEnvMap(envMap, p.Configuration, envPrefix); err != nil {
+		return env.New(), err
 	}
 
-	envSlice, err = fanOutDeprecatedEnvVarNames(envSlice)
-	return env.FromSlice(envSlice), err
+	err = addDeprecatedEnvVarAliases(envMap)
+	return env.FromMap(envMap), err
 }
 
 // Label returns a pretty name for the plugin.

From b30d3e118510e5e937028d16a5c05ab186e5f0c6 Mon Sep 17 00:00:00 2001
From: Ryan Sundberg <ryan@arctype.co>
Date: Sun, 11 Jan 2026 17:23:59 -0800
Subject: [PATCH 144/242] clicommand/agent_start.go: Use `/usr/bin/env bash` as
 default shell

Use `/usr/bin/env bash` rather than `/bin/bash` to locate bash for
the default shell in a Linux agent.

This improves portability for running agents on NixOS and GNU Guix
hosts, where /bin/bash is not present, but /usr/bin/env is present.
---
 clicommand/agent_start.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index f2a4ce7e45..debfbee80f 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -289,7 +289,7 @@ func DefaultShell() string {
 	case "netbsd":
 		return "/usr/pkg/bin/bash -e -c"
 	default:
-		return "/bin/bash -e -c"
+		return "/usr/bin/env bash -e -c"
 	}
 }
 

From d445b5b213af2f947e456f64dee55ca0ecbf4e54 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Jan 2026 19:44:13 +0000
Subject: [PATCH 145/242] build(deps): bump the container-images group across 4
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `dc83378` to `63c4bd6`

Updates `buildkite/agent-base` from `12dcb2d` to `c6da479`

Updates `buildkite/agent-base` from `dc768a6` to `5a5d5fa`

Updates `buildkite/agent-base` from `a0ba149` to `5b09281`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 56c1bb1968..c45516c498 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:12dcb2dbc09a70210babdf0e221a4448127009cfb136e41b9a34fb4a79e59b84
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:c6da47972412bd43486d25b53fe290654982a1e0e15d491821d7cbeb7f966eb9
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 0c66dab44a..29bc7b6517 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:dc8337889813c0787bda5fd4119fdcdb1a9fb9151af16e896a7c2ab31eab141f
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:63c4bd695b2b3d6ce6c907155ce0b5c0bf98f61b05bd3475c56b51bc51c2e8ad
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 8414859520..0fd2f988aa 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:dc768a639d18edb9ef2722881af6b59938f818b6594b2cf14996a0add2edc850
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:5a5d5faf2ec27bde427a9b542bce1152f470e68cd64ca1cdb6899593621f2aaa
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index e9fc21ca6b..5a203b0a8b 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:a0ba149ebf4828cdd6a643f97cc1e6a61ce44698bba4ae84ed8a38170cd9bc8f
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:5b0928128147b9e4619f50d0a98d06bbd9d7a1db1bdb3ce471cf123228ea1514
 ARG TARGETOS
 ARG TARGETARCH
 

From e204703adf5ab5e2a4bfa154bc1dca92a121200e Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Tue, 13 Jan 2026 13:38:20 +1100
Subject: [PATCH 146/242] adjust agent_start default shell

---
 clicommand/agent_start.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index debfbee80f..45be848a37 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -289,6 +289,13 @@ func DefaultShell() string {
 	case "netbsd":
 		return "/usr/pkg/bin/bash -e -c"
 	default:
+		// On most Unix-like systems, bash is at /bin/bash and we prefer to use it
+		// directly to avoid PATH manipulation concerns with /usr/bin/env.
+		// However, some systems like NixOS or GNU Guix don't have /bin/bash.
+		// In those cases, fall back to /usr/bin/env bash which will find bash in PATH.
+		if _, err := os.Stat("/bin/bash"); err == nil {
+			return "/bin/bash -e -c"
+		}
 		return "/usr/bin/env bash -e -c"
 	}
 }

From ad38f46acb1ec0f0422f1d21337f44aa97b8a926 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 13 Jan 2026 04:16:51 +0000
Subject: [PATCH 147/242] build(deps): bump the container-images group across 4
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `63c4bd6` to `174a2a9`

Updates `buildkite/agent-base` from `c6da479` to `7ca08c1`

Updates `buildkite/agent-base` from `5a5d5fa` to `6cb7bd9`

Updates `buildkite/agent-base` from `5b09281` to `14db4ab`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index c45516c498..b1b14ac6b2 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:c6da47972412bd43486d25b53fe290654982a1e0e15d491821d7cbeb7f966eb9
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:7ca08c1e474eb971088deea6d356d6a1f17be62d4ac7a0ad383b437868f47d26
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 29bc7b6517..12c765b1e3 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:63c4bd695b2b3d6ce6c907155ce0b5c0bf98f61b05bd3475c56b51bc51c2e8ad
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:174a2a911c8949efafbecb5374b17f5b4a9341a097e24f2c13396969a2f865df
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 0fd2f988aa..bbcb9342e3 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:5a5d5faf2ec27bde427a9b542bce1152f470e68cd64ca1cdb6899593621f2aaa
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:6cb7bd99b2b774cae9b30bf3af1c5581ff5d4b0fa34cb37279cb04270dea7781
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index 5a203b0a8b..9ff5571499 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:5b0928128147b9e4619f50d0a98d06bbd9d7a1db1bdb3ce471cf123228ea1514
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:14db4ab933b71023690acbceca9365d8f538dc6f3d3bb7c41c42c810ccc67463
 ARG TARGETOS
 ARG TARGETARCH
 

From c96737f5e0046e77b1926e5273c892fb2a28a7b0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 13 Jan 2026 04:44:26 +0000
Subject: [PATCH 148/242] build(deps): bump buildkite/agent-base

Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `b3d073a` to `797f9d7`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index bfb5ea99af..ef64c82ad7 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:b3d073a3013d5cd3be46ba01c4a083da92f6dcf5de7622365fd50b5ec624bb04
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:797f9d79ef9f8ed8fcbe4bc22dbeafd58d4dc109369e40b8b770efb275006bad
 ARG TARGETOS
 ARG TARGETARCH
 

From dc60107bf235e9ee6577aa9ad145a7f143d22b9b Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Tue, 13 Jan 2026 16:24:49 +1100
Subject: [PATCH 149/242] release 3.115.4

---
 CHANGELOG.md    | 10 ++++++++++
 version/VERSION |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 37a8ce01ec..8db63317fc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.115.4](https://github.com/buildkite/agent/tree/v3.115.4) (2026-01-13)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.115.3...v3.115.4)
+
+### Changed
+
+- Fallback to `/usr/bin/env bash`, when `/bin/bash` does not exist [#3661](https://github.com/buildkite/agent/pull/3661) (@sundbry), [#3667](https://github.com/buildkite/agent/pull/3667) (@zhming0)
+
+### Internal
+- Bump various container base image version. [#3669](https://github.com/buildkite/agent/pull/3669), [#3668](https://github.com/buildkite/agent/pull/3668),  [#3667](https://github.com/buildkite/agent/pull/3667) (@dependabot[bot])
+
 ## [v3.115.3](https://github.com/buildkite/agent/tree/v3.115.3) (2026-01-08)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.115.2...v3.115.3)
 
diff --git a/version/VERSION b/version/VERSION
index d516e3a4d8..390434eef0 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.115.3
+3.115.4

From ea92ca4f70e3f5ef2f3ee6089490c26f72ebcbcb Mon Sep 17 00:00:00 2001
From: Ben McNicholl <git@benmcnicholl.com>
Date: Wed, 14 Jan 2026 22:41:41 +1100
Subject: [PATCH 150/242] add tests for expected skip checkout behaviour

Signed-off-by: Ben McNicholl <git@benmcnicholl.com>
---
 internal/job/checkout_test.go                 | 20 ++++++++++++++++++
 .../integration/checkout_integration_test.go  | 21 +++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/internal/job/checkout_test.go b/internal/job/checkout_test.go
index fb59c88457..01ee9bec89 100644
--- a/internal/job/checkout_test.go
+++ b/internal/job/checkout_test.go
@@ -147,6 +147,26 @@ func TestDefaultCheckoutPhase(t *testing.T) {
 	}
 }
 
+func TestSkipCheckout(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	sh, err := shell.New()
+	require.NoError(t, err)
+
+	executor := &Executor{
+		shell: sh,
+		ExecutorConfig: ExecutorConfig{
+			Repository:   "https://github.com/buildkite/agent.git",
+			SkipCheckout: true,
+		},
+	}
+
+	err = executor.checkout(ctx)
+	require.NoError(t, err)
+}
+
 func TestDefaultCheckoutPhase_DelayedRefCreation(t *testing.T) {
 	if race.IsRaceTest {
 		t.Skip("this test simulates the agent recovering from a race condition, and needs to create one to test it.")
diff --git a/internal/job/integration/checkout_integration_test.go b/internal/job/integration/checkout_integration_test.go
index e1fd8f9791..ebf42051cc 100644
--- a/internal/job/integration/checkout_integration_test.go
+++ b/internal/job/integration/checkout_integration_test.go
@@ -849,6 +849,27 @@ func TestForcingACleanCheckout(t *testing.T) {
 	}
 }
 
+func TestSkippingCheckout(t *testing.T) {
+	t.Parallel()
+
+	tester, err := NewExecutorTester(mainCtx)
+	if err != nil {
+		t.Fatalf("NewExecutorTester() error = %v", err)
+	}
+	defer tester.Close()
+
+	tester.RunAndCheck(t, "BUILDKITE_SKIP_CHECKOUT=true")
+
+	if !strings.Contains(tester.Output, "Skipping checkout") {
+		t.Fatal(`tester.Output does not contain "Skipping checkout"`)
+	}
+
+	// Verify no git commands were run (no clone, fetch, checkout)
+	if strings.Contains(tester.Output, "git clone") {
+		t.Fatal(`tester.Output should not contain "git clone" when checkout is skipped`)
+	}
+}
+
 func TestCheckoutOnAnExistingRepositoryWithoutAGitFolder(t *testing.T) {
 	t.Parallel()
 

From 4028391989d62d5a2610ddd3593aacd172747182 Mon Sep 17 00:00:00 2001
From: Ben McNicholl <git@benmcnicholl.com>
Date: Wed, 14 Jan 2026 22:42:53 +1100
Subject: [PATCH 151/242] add SkipCheckout as agent start option

Signed-off-by: Ben McNicholl <git@benmcnicholl.com>
---
 clicommand/agent_start.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 45be848a37..f0afc45bf3 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -150,6 +150,7 @@ type AgentStartConfig struct {
 	GitMirrorsLockTimeout int    `cli:"git-mirrors-lock-timeout"`
 	GitMirrorsSkipUpdate  bool   `cli:"git-mirrors-skip-update"`
 	NoGitSubmodules       bool   `cli:"no-git-submodules"`
+	SkipCheckout          bool   `cli:"skip-checkout"`
 
 	NoSSHKeyscan            bool     `cli:"no-ssh-keyscan"`
 	NoCommandEval           bool     `cli:"no-command-eval"`
@@ -624,6 +625,11 @@ var AgentStartCommand = cli.Command{
 			Usage:  "Don't automatically checkout git submodules",
 			EnvVar: "BUILDKITE_NO_GIT_SUBMODULES,BUILDKITE_DISABLE_GIT_SUBMODULES",
 		},
+		cli.BoolFlag{
+			Name:   "skip-checkout",
+			Usage:  "Skip the git checkout phase entirely",
+			EnvVar: "BUILDKITE_SKIP_CHECKOUT",
+		},
 		cli.BoolFlag{
 			Name:   "no-feature-reporting",
 			Usage:  "Disables sending a list of enabled features back to the Buildkite mothership. We use this information to measure feature usage, but if you're not comfortable sharing that information then that's totally okay :)",
@@ -1040,6 +1046,7 @@ var AgentStartCommand = cli.Command{
 			GitCleanFlags:                cfg.GitCleanFlags,
 			GitFetchFlags:                cfg.GitFetchFlags,
 			GitSubmodules:                !cfg.NoGitSubmodules,
+			SkipCheckout:                 cfg.SkipCheckout,
 			SSHKeyscan:                   !cfg.NoSSHKeyscan,
 			CommandEval:                  !cfg.NoCommandEval,
 			PluginsEnabled:               !cfg.NoPlugins,

From a39bcc5e40e750b79c1774711102966bc75e1e25 Mon Sep 17 00:00:00 2001
From: Ben McNicholl <git@benmcnicholl.com>
Date: Wed, 14 Jan 2026 22:43:25 +1100
Subject: [PATCH 152/242] add cli command to bootstrap

Signed-off-by: Ben McNicholl <git@benmcnicholl.com>
---
 clicommand/bootstrap.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go
index 40c6efbda5..b5b1acdd2a 100644
--- a/clicommand/bootstrap.go
+++ b/clicommand/bootstrap.go
@@ -68,6 +68,7 @@ type BootstrapConfig struct {
 	AutomaticArtifactUploadPaths string   `cli:"artifact-upload-paths"`
 	ArtifactUploadDestination    string   `cli:"artifact-upload-destination"`
 	CleanCheckout                bool     `cli:"clean-checkout"`
+	SkipCheckout                 bool     `cli:"skip-checkout"`
 	GitCheckoutFlags             string   `cli:"git-checkout-flags"`
 	GitCloneFlags                string   `cli:"git-clone-flags"`
 	GitFetchFlags                string   `cli:"git-fetch-flags"`
@@ -227,6 +228,11 @@ var BootstrapCommand = cli.Command{
 			Usage:  "Whether or not the bootstrap should remove the existing repository before running the command",
 			EnvVar: "BUILDKITE_CLEAN_CHECKOUT",
 		},
+		cli.BoolFlag{
+			Name:   "skip-checkout",
+			Usage:  "Skip the git checkout phase entirely",
+			EnvVar: "BUILDKITE_SKIP_CHECKOUT",
+		},
 		cli.StringFlag{
 			Name:   "git-checkout-flags",
 			Value:  "-f",
@@ -468,6 +474,7 @@ var BootstrapCommand = cli.Command{
 			CancelSignal:                 cancelSig,
 			SignalGracePeriod:            signalGracePeriod,
 			CleanCheckout:                cfg.CleanCheckout,
+			SkipCheckout:                 cfg.SkipCheckout,
 			Command:                      cfg.Command,
 			CommandEval:                  cfg.CommandEval,
 			Commit:                       cfg.Commit,

From a1dd61aec1400f95cc94725c4006f8494324d717 Mon Sep 17 00:00:00 2001
From: Ben McNicholl <git@benmcnicholl.com>
Date: Wed, 14 Jan 2026 22:44:04 +1100
Subject: [PATCH 153/242] add skip checkout to config struct

Signed-off-by: Ben McNicholl <git@benmcnicholl.com>
---
 agent/agent_configuration.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go
index b467bc51b6..a9b53a3ddd 100644
--- a/agent/agent_configuration.go
+++ b/agent/agent_configuration.go
@@ -24,6 +24,7 @@ type AgentConfiguration struct {
 	GitCleanFlags               string
 	GitFetchFlags               string
 	GitSubmodules               bool
+	SkipCheckout                bool
 	AllowedRepositories         []*regexp.Regexp
 	AllowedPlugins              []*regexp.Regexp
 	AllowedEnvironmentVariables []*regexp.Regexp

From 6b60e72b21beb5e7acd27d98927a760131a743e7 Mon Sep 17 00:00:00 2001
From: Ben McNicholl <git@benmcnicholl.com>
Date: Wed, 14 Jan 2026 22:44:24 +1100
Subject: [PATCH 154/242] set env in job runner

Signed-off-by: Ben McNicholl <git@benmcnicholl.com>
---
 agent/job_runner.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index d71a7d974c..d01ed22627 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -577,6 +577,7 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_PLUGINS_PATH", r.conf.AgentConfiguration.PluginsPath)
 	setEnv("BUILDKITE_SSH_KEYSCAN", fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan))
 	setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
+	setEnv("BUILDKITE_SKIP_CHECKOUT", fmt.Sprint(r.conf.AgentConfiguration.SkipCheckout))
 	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
 	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
 	// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config

From 0f06aa3af1c1fd23bbf588bb57ed95056d56271d Mon Sep 17 00:00:00 2001
From: Ben McNicholl <git@benmcnicholl.com>
Date: Wed, 14 Jan 2026 22:45:07 +1100
Subject: [PATCH 155/242] support skip checkout as an agent flag

---
 agent/job_runner.go      | 7 ++++++-
 internal/job/checkout.go | 5 +++++
 internal/job/config.go   | 3 +++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index d01ed22627..e24d9fca03 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -577,7 +577,12 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_PLUGINS_PATH", r.conf.AgentConfiguration.PluginsPath)
 	setEnv("BUILDKITE_SSH_KEYSCAN", fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan))
 	setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
-	setEnv("BUILDKITE_SKIP_CHECKOUT", fmt.Sprint(r.conf.AgentConfiguration.SkipCheckout))
+	// Allow BUILDKITE_SKIP_CHECKOUT to be enabled either by agent config
+	// or by pipeline/step env
+	// This is here now to make it ready for if/when we add skip_checkout to the core app
+	if r.conf.AgentConfiguration.SkipCheckout {
+		setEnv("BUILDKITE_SKIP_CHECKOUT", "true")
+	}
 	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
 	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
 	// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config
diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index 1836c0b948..e59c3c29fc 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -217,6 +217,11 @@ func (e *Executor) CheckoutPhase(ctx context.Context) error {
 
 // checkout runs checkout hook or default checkout logic
 func (e *Executor) checkout(ctx context.Context) error {
+	if e.SkipCheckout {
+		e.shell.Commentf("Skipping checkout, BUILDKITE_SKIP_CHECKOUT is set")
+		return nil
+	}
+
 	// There can only be one checkout hook, either plugin or global, in that order
 	switch {
 	case e.hasPluginHook("checkout"):
diff --git a/internal/job/config.go b/internal/job/config.go
index 8e5d374edd..abcbbd200e 100644
--- a/internal/job/config.go
+++ b/internal/job/config.go
@@ -75,6 +75,9 @@ type ExecutorConfig struct {
 	// Should the executor remove an existing checkout before running the job
 	CleanCheckout bool `env:"BUILDKITE_CLEAN_CHECKOUT"`
 
+	// Skip the checkout phase entirely
+	SkipCheckout bool `env:"BUILDKITE_SKIP_CHECKOUT"`
+
 	// Flags to pass to "git checkout" command
 	GitCheckoutFlags string `env:"BUILDKITE_GIT_CHECKOUT_FLAGS"`
 

From 0c4b8cf7719b0cc155e509f7be98a41cb8e74907 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Jan 2026 17:40:51 +0000
Subject: [PATCH 156/242] build(deps): bump github.com/go-chi/chi/v5 from 5.2.3
 to 5.2.4

Bumps [github.com/go-chi/chi/v5](https://github.com/go-chi/chi) from 5.2.3 to 5.2.4.
- [Release notes](https://github.com/go-chi/chi/releases)
- [Changelog](https://github.com/go-chi/chi/blob/master/CHANGELOG.md)
- [Commits](https://github.com/go-chi/chi/compare/v5.2.3...v5.2.4)

---
updated-dependencies:
- dependency-name: github.com/go-chi/chi/v5
  dependency-version: 5.2.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index f8e6df72a2..a2e1a5a2b5 100644
--- a/go.mod
+++ b/go.mod
@@ -32,7 +32,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2
 	github.com/gliderlabs/ssh v0.3.8
-	github.com/go-chi/chi/v5 v5.2.3
+	github.com/go-chi/chi/v5 v5.2.4
 	github.com/gofrs/flock v0.13.0
 	github.com/google/go-cmp v0.7.0
 	github.com/google/go-querystring v1.1.0
diff --git a/go.sum b/go.sum
index 452e1bec59..ffe9da4946 100644
--- a/go.sum
+++ b/go.sum
@@ -191,8 +191,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
-github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
-github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
+github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
+github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

From 1473333de33ed49a9b8ada9813e5855e8205234a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Jan 2026 17:49:16 +0000
Subject: [PATCH 157/242] build(deps): bump the cloud-providers group across 1
 directory with 9 updates

Bumps the cloud-providers group with 7 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [github.com/Azure/azure-sdk-for-go/sdk/storage/azblob](https://github.com/Azure/azure-sdk-for-go) | `1.6.3` | `1.6.4` |
| [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.0` | `1.41.1` |
| [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.6` | `1.32.7` |
| [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.20.17` | `1.20.19` |
| [github.com/aws/aws-sdk-go-v2/service/ec2](https://github.com/aws/aws-sdk-go-v2) | `1.278.0` | `1.279.2` |
| [github.com/aws/aws-sdk-go-v2/service/kms](https://github.com/aws/aws-sdk-go-v2) | `1.49.4` | `1.49.5` |
| [google.golang.org/api](https://github.com/googleapis/google-api-go-client) | `0.258.0` | `0.260.0` |



Updates `github.com/Azure/azure-sdk-for-go/sdk/storage/azblob` from 1.6.3 to 1.6.4
- [Release notes](https://github.com/Azure/azure-sdk-for-go/releases)
- [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/storage/azblob/v1.6.3...sdk/storage/azblob/v1.6.4)

Updates `github.com/aws/aws-sdk-go-v2` from 1.41.0 to 1.41.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.0...v1.41.1)

Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.6 to 1.32.7
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.32.6...v1.32.7)

Updates `github.com/aws/aws-sdk-go-v2/feature/ec2/imds` from 1.18.16 to 1.18.17
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.16...config/v1.18.17)

Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.20.17 to 1.20.19
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/emr/v1.20.17...feature/s3/manager/v1.20.19)

Updates `github.com/aws/aws-sdk-go-v2/service/ec2` from 1.278.0 to 1.279.2
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ec2/v1.278.0...service/ec2/v1.279.2)

Updates `github.com/aws/aws-sdk-go-v2/service/kms` from 1.49.4 to 1.49.5
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ssm/v1.49.4...service/fsx/v1.49.5)

Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.94.0 to 1.95.1
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.94.0...service/s3/v1.95.1)

Updates `google.golang.org/api` from 0.258.0 to 0.260.0
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.258.0...v0.260.0)

---
updated-dependencies:
- dependency-name: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob
  dependency-version: 1.6.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2
  dependency-version: 1.41.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-version: 1.32.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/ec2/imds
  dependency-version: 1.18.17
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager
  dependency-version: 1.20.19
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/ec2
  dependency-version: 1.279.2
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/kms
  dependency-version: 1.49.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cloud-providers
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-version: 1.95.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
- dependency-name: google.golang.org/api
  dependency-version: 0.260.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: cloud-providers
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod |  50 +++++++++++++--------------
 go.sum | 104 ++++++++++++++++++++++++++++-----------------------------
 2 files changed, 77 insertions(+), 77 deletions(-)

diff --git a/go.mod b/go.mod
index f8e6df72a2..cf534d72ea 100644
--- a/go.mod
+++ b/go.mod
@@ -8,16 +8,16 @@ require (
 	cloud.google.com/go/compute/metadata v0.9.0
 	drjosh.dev/zzglob v0.4.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
 	github.com/DataDog/datadog-go/v5 v5.8.2
 	github.com/Khan/genqlient v0.8.1
-	github.com/aws/aws-sdk-go-v2 v1.41.0
-	github.com/aws/aws-sdk-go-v2/config v1.32.6
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.278.0
-	github.com/aws/aws-sdk-go-v2/service/kms v1.49.4
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0
+	github.com/aws/aws-sdk-go-v2 v1.41.1
+	github.com/aws/aws-sdk-go-v2/config v1.32.7
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.279.2
+	github.com/aws/aws-sdk-go-v2/service/kms v1.49.5
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
@@ -62,14 +62,14 @@ require (
 	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.39.0
 	golang.org/x/term v0.38.0
-	google.golang.org/api v0.258.0
+	google.golang.org/api v0.260.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
 )
 
 require (
-	cloud.google.com/go/auth v0.17.0 // indirect
+	cloud.google.com/go/auth v0.18.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
@@ -96,19 +96,19 @@ require (
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
@@ -135,8 +135,8 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
-	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.9 // indirect
+	github.com/googleapis/gax-go/v2 v2.16.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -204,8 +204,8 @@ require (
 	golang.org/x/tools v0.39.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
-	google.golang.org/grpc v1.77.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
+	google.golang.org/grpc v1.78.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
diff --git a/go.sum b/go.sum
index 452e1bec59..5d3ddb8ea4 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
-cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
+cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0=
+cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
@@ -16,8 +16,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDo
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
@@ -76,48 +76,48 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
-github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
+github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
-github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17 h1:fODjlj9c1zIfZYFxdC6Z4GX/plrZUYI/5EklgA/24Hw=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17/go.mod h1:CEyBu8kavY5Tc8i/8A810DuKydd19Lrx2/TmcNdjOAk=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
+github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
+github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19 h1:Gxj3kAlmM+a/VVO4YNsmgHGVUZhSxs0tuVwLIxZBCtM=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19/go.mod h1:XGq5kImVqQT4HUNbbG+0Y8O74URsPNH7CGPg1s1HW5E=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.278.0 h1:Cx/Rs2zaG30Dn4QMvUGC5rCAZagA8heta0TWAdBE/Xc=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.278.0/go.mod h1:Wg68QRgy2gEGGdmTPU/UbVpdv8sM14bUZmF64KFwAsY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.279.2 h1:MG12Z/W1zzJLkw2gCU2gKZ872rqLM0pi9LdkZ/z3FHc=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.279.2/go.mod h1:Uy+C+Sc58jozdoL1McQr8bDsEvNFx+/nBY+vpO1HVUY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.4 h1:2gom8MohxN0SnhHZBYAC4S8jHG+ENEnXjyJ5xKe3vLc=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.4/go.mod h1:HO31s0qt0lso/ADvZQyzKs8js/ku0fMHsfyXW8OPVYc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 h1:SWTxh/EcUCDVqi/0s26V6pVUq0BBG7kx0tDTmF/hCgA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.5 h1:DKibav4XF66XSeaXcrn9GlWGHos6D/vJ4r7jsK7z5CE=
+github.com/aws/aws-sdk-go-v2/service/kms v1.49.5/go.mod h1:1SdcmEGUEQE1mrU2sIgeHtcMSxHuybhPvuEPANzIDfI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1 h1:C2dUPSnEpy4voWFIq3JNd8gN0Y5vYGDo44eUE58a/p8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
 github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
 github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -235,10 +235,10 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
-github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
-github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
+github.com/googleapis/enterprise-certificate-proxy v0.3.9 h1:TOpi/QG8iDcZlkQlGlFUti/ZtyLkliXvHDcyUIMuFrU=
+github.com/googleapis/enterprise-certificate-proxy v0.3.9/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
+github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y=
+github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU=
@@ -554,18 +554,18 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
-google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
+google.golang.org/api v0.260.0 h1:XbNi5E6bOVEj/uLXQRlt6TKuEzMD7zvW/6tNwltE4P4=
+google.golang.org/api v0.260.0/go.mod h1:Shj1j0Phr/9sloYrKomICzdYgsSDImpTxME8rGLaZ/o=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
-google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
-google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
+google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934=
+google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0=
 google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
 google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
-google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
-google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
+google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 h1:h96ji92t9eXbPvSWhJ+lrPWetHiQNYlt48JKRO09NFA=

From fa9e2c85f62bb6319968bb451135a0d69f8227ec Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Fri, 23 Jan 2026 17:14:44 +0000
Subject: [PATCH 158/242] feat: enable BUILDKITE_GIT_SUBMODULES via environment
 variable * defaults to true for backwards compatibility

Signed-off-by: Tom Watt <tom@buildkite.com>
---
 agent/job_runner.go    | 6 +++++-
 internal/job/config.go | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index d71a7d974c..b4a28fcd0f 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -576,7 +576,11 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_ADDITIONAL_HOOKS_PATHS", strings.Join(r.conf.AgentConfiguration.AdditionalHooksPaths, ","))
 	setEnv("BUILDKITE_PLUGINS_PATH", r.conf.AgentConfiguration.PluginsPath)
 	setEnv("BUILDKITE_SSH_KEYSCAN", fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan))
-	setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
+	// Allow BUILDKITE_GIT_SUBMODULES to be enabled either by config
+	// or by pipeline/step environment.
+	if !r.conf.AgentConfiguration.GitSubmodules {
+		setEnv("BUILDKITE_GIT_SUBMODULES", "true")
+	}
 	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
 	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
 	// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config
diff --git a/internal/job/config.go b/internal/job/config.go
index 8e5d374edd..21bb439214 100644
--- a/internal/job/config.go
+++ b/internal/job/config.go
@@ -49,7 +49,7 @@ type ExecutorConfig struct {
 	Plugins string
 
 	// Should git submodules be checked out
-	GitSubmodules bool
+	GitSubmodules bool `env:"BUILDKITE_GIT_SUBMODULES"`
 
 	// If the commit was part of a pull request, this will container the PR number
 	PullRequest string

From 55af1fabffdab8db3fe8bffb7ae3db1e5e8de08a Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Fri, 23 Jan 2026 17:15:12 +0000
Subject: [PATCH 159/242] chore: update warning message for disabled submodules

Signed-off-by: Tom Watt <tom@buildkite.com>
---
 internal/job/checkout.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index 1836c0b948..27f7b332a2 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -751,7 +751,7 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 			e.shell.Commentf("Git submodules detected")
 			gitSubmodules = true
 		} else {
-			e.shell.OptionalWarningf("submodules-disabled", "This repository has submodules, but submodules are disabled at an agent level")
+			e.shell.OptionalWarningf("submodules-disabled", "This repository has submodules, but submodules are disabled")
 		}
 	}
 

From aa3b22458ad92c5cb1b5a2505c069997385f56be Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Fri, 23 Jan 2026 18:31:42 +0000
Subject: [PATCH 160/242] revert: BUILDKITE_GIT_SUBMODULES setting for
 bidirectional overrides

Signed-off-by: Tom Watt <tom@buildkite.com>
---
 agent/job_runner.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index b4a28fcd0f..d71a7d974c 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -576,11 +576,7 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_ADDITIONAL_HOOKS_PATHS", strings.Join(r.conf.AgentConfiguration.AdditionalHooksPaths, ","))
 	setEnv("BUILDKITE_PLUGINS_PATH", r.conf.AgentConfiguration.PluginsPath)
 	setEnv("BUILDKITE_SSH_KEYSCAN", fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan))
-	// Allow BUILDKITE_GIT_SUBMODULES to be enabled either by config
-	// or by pipeline/step environment.
-	if !r.conf.AgentConfiguration.GitSubmodules {
-		setEnv("BUILDKITE_GIT_SUBMODULES", "true")
-	}
+	setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
 	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
 	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
 	// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config

From 01f552ccf12fe61b8d02cacac3d6be0eaac0e42e Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Fri, 23 Jan 2026 18:35:01 +0000
Subject: [PATCH 161/242] test: add GitSubmodules to environment variable
 mapping tests

Signed-off-by: Tom Watt <tom@buildkite.com>
---
 internal/job/config_test.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/internal/job/config_test.go b/internal/job/config_test.go
index f0ead3cc4c..ff7d811a17 100644
--- a/internal/job/config_test.go
+++ b/internal/job/config_test.go
@@ -18,6 +18,7 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) {
 		AgentName:                    "myAgent",
 		CleanCheckout:                false,
 		PluginsAlwaysCloneFresh:      false,
+		GitSubmodules:                false,
 	}
 
 	environ := env.FromSlice([]string{
@@ -27,6 +28,7 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) {
 		"BUILDKITE_REPO=https://my.mirror/repo.git",
 		"BUILDKITE_CLEAN_CHECKOUT=true",
 		"BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH=true",
+		"BUILDKITE_GIT_SUBMODULES=true",
 	})
 
 	changes := config.ReadFromEnvironment(environ)
@@ -36,6 +38,7 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) {
 		"BUILDKITE_REPO":                       "https://my.mirror/repo.git",
 		"BUILDKITE_CLEAN_CHECKOUT":             "true",
 		"BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH": "true",
+		"BUILDKITE_GIT_SUBMODULES":             "true",
 	}
 
 	if diff := cmp.Diff(changes, wantChanges); diff != "" {
@@ -57,6 +60,10 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) {
 	if got, want := config.PluginsAlwaysCloneFresh, true; got != want {
 		t.Errorf("config.PluginsAlwaysCloneFresh = %t, want %t", got, want)
 	}
+
+	if got, want := config.GitSubmodules, true; got != want {
+		t.Errorf("config.GitSubmodules = %t, want %t", got, want)
+	}
 }
 
 func TestReadFromEnvironmentIgnoresMalformedBooleans(t *testing.T) {
@@ -64,10 +71,12 @@ func TestReadFromEnvironmentIgnoresMalformedBooleans(t *testing.T) {
 	config := &ExecutorConfig{
 		CleanCheckout:           true,
 		PluginsAlwaysCloneFresh: false,
+		GitSubmodules:           true,
 	}
 	environ := env.FromSlice([]string{
 		"BUILDKITE_CLEAN_CHECKOUT=blarg",
 		"BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH=grarg",
+		"BUILDKITE_GIT_SUBMODULES=notabool",
 	})
 	changes := config.ReadFromEnvironment(environ)
 	if len(changes) != 0 {
@@ -79,4 +88,7 @@ func TestReadFromEnvironmentIgnoresMalformedBooleans(t *testing.T) {
 	if got, want := config.PluginsAlwaysCloneFresh, false; got != want {
 		t.Errorf("config.PluginsAlwaysCloneFresh = %t, want %t", got, want)
 	}
+	if got, want := config.GitSubmodules, true; got != want {
+		t.Errorf("config.GitSubmodules = %t, want %t", got, want)
+	}
 }

From d21e8888974f11eb62a8579d0d392c2ffb5e6a98 Mon Sep 17 00:00:00 2001
From: Pete Tomasik <senate6268@gmail.com>
Date: Fri, 23 Jan 2026 16:09:11 -0500
Subject: [PATCH 162/242] Add default BoolFlag, BoolTFlag values to
 descriptions

---
 clicommand/agent_start.go        | 54 ++++++++++++++++----------------
 clicommand/agent_stop.go         |  2 +-
 clicommand/annotate.go           |  2 +-
 clicommand/artifact_download.go  |  2 +-
 clicommand/artifact_search.go    |  4 +--
 clicommand/artifact_shasum.go    |  4 +--
 clicommand/artifact_upload.go    |  8 ++---
 clicommand/bootstrap.go          | 26 +++++++--------
 clicommand/global.go             | 14 ++++-----
 clicommand/oidc_request_token.go |  2 +-
 clicommand/pipeline_upload.go    | 12 +++----
 clicommand/secret_get.go         |  2 +-
 clicommand/step_cancel.go        |  2 +-
 clicommand/step_update.go        |  2 +-
 clicommand/tool_sign.go          |  6 ++--
 15 files changed, 71 insertions(+), 71 deletions(-)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 45be848a37..72a81f4b1b 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -366,12 +366,12 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "reflect-exit-status",
-			Usage:  "When used with --acquire-job, causes the agent to exit with the same exit status as the job",
+			Usage:  "When used with --acquire-job, causes the agent to exit with the same exit status as the job (default: false)",
 			EnvVar: "BUILDKITE_AGENT_REFLECT_EXIT_STATUS",
 		},
 		cli.BoolFlag{
 			Name:   "disconnect-after-job",
-			Usage:  "Disconnect the agent after running exactly one job. When used in conjunction with the ′--spawn′ flag, each worker booted will run exactly one job",
+			Usage:  "Disconnect the agent after running exactly one job. When used in conjunction with the ′--spawn′ flag, each worker booted will run exactly one job (default: false)",
 			EnvVar: "BUILDKITE_AGENT_DISCONNECT_AFTER_JOB",
 		},
 		cli.IntFlag{
@@ -389,7 +389,7 @@ var AgentStartCommand = cli.Command{
 		cancelGracePeriodFlag,
 		cli.BoolFlag{
 			Name:   "enable-job-log-tmpfile",
-			Usage:  "Store the job logs in a temporary file ′BUILDKITE_JOB_LOG_TMPFILE′ that is accessible during the job and removed at the end of the job",
+			Usage:  "Store the job logs in a temporary file ′BUILDKITE_JOB_LOG_TMPFILE′ that is accessible during the job and removed at the end of the job (default: false)",
 			EnvVar: "BUILDKITE_ENABLE_JOB_LOG_TMPFILE",
 		},
 		cli.StringFlag{
@@ -399,7 +399,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "write-job-logs-to-stdout",
-			Usage:  "Writes job logs to the agent process' stdout. This simplifies log collection if running agents in Docker.",
+			Usage:  "Writes job logs to the agent process' stdout. This simplifies log collection if running agents in Docker (default: false)",
 			EnvVar: "BUILDKITE_WRITE_JOB_LOGS_TO_STDOUT",
 		},
 		cli.StringFlag{
@@ -421,7 +421,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "tags-from-host",
-			Usage:  "Include tags from the host (hostname, machine-id, os)",
+			Usage:  "Include tags from the host (hostname, machine-id, os) (default: false)",
 			EnvVar: "BUILDKITE_AGENT_TAGS_FROM_HOST",
 		},
 		cli.StringSliceFlag{
@@ -438,12 +438,12 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "tags-from-ec2-tags",
-			Usage:  "Include the host's EC2 tags as tags",
+			Usage:  "Include the host's EC2 tags as tags (default: false)",
 			EnvVar: "BUILDKITE_AGENT_TAGS_FROM_EC2_TAGS",
 		},
 		cli.BoolFlag{
 			Name:   "tags-from-ecs-meta-data",
-			Usage:  "Include the host's ECS meta-data as tags (container-name, image, and task-arn)",
+			Usage:  "Include the host's ECS meta-data as tags (container-name, image, and task-arn) (default: false)",
 			EnvVar: "BUILDKITE_AGENT_TAGS_FROM_ECS_META_DATA",
 		},
 		cli.StringSliceFlag{
@@ -460,7 +460,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "tags-from-gcp-labels",
-			Usage:  "Include the host's Google Cloud instance labels as tags",
+			Usage:  "Include the host's Google Cloud instance labels as tags (default: false)",
 			EnvVar: "BUILDKITE_AGENT_TAGS_FROM_GCP_LABELS",
 		},
 		cli.DurationFlag{
@@ -535,7 +535,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "git-mirrors-skip-update",
-			Usage:  "Skip updating the Git mirror",
+			Usage:  "Skip updating the Git mirror (default: false)",
 			EnvVar: "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE",
 		},
 		cli.StringFlag{
@@ -571,12 +571,12 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "no-ansi-timestamps",
-			Usage:  "Do not insert ANSI timestamp codes at the start of each line of job output",
+			Usage:  "Do not insert ANSI timestamp codes at the start of each line of job output (default: false)",
 			EnvVar: "BUILDKITE_NO_ANSI_TIMESTAMPS",
 		},
 		cli.BoolFlag{
 			Name:   "timestamp-lines",
-			Usage:  "Prepend timestamps on each line of job output. Has no effect unless --no-ansi-timestamps is also used",
+			Usage:  "Prepend timestamps on each line of job output. Has no effect unless --no-ansi-timestamps is also used (default: false)",
 			EnvVar: "BUILDKITE_TIMESTAMP_LINES",
 		},
 		cli.StringFlag{
@@ -586,47 +586,47 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "no-pty",
-			Usage:  "Do not run jobs within a pseudo terminal",
+			Usage:  "Do not run jobs within a pseudo terminal (default: false)",
 			EnvVar: "BUILDKITE_NO_PTY",
 		},
 		cli.BoolFlag{
 			Name:   "no-ssh-keyscan",
-			Usage:  "Don't automatically run ssh-keyscan before checkout",
+			Usage:  "Don't automatically run ssh-keyscan before checkout (default: false)",
 			EnvVar: "BUILDKITE_NO_SSH_KEYSCAN",
 		},
 		cli.BoolFlag{
 			Name:   "no-command-eval",
-			Usage:  "Don't allow this agent to run arbitrary console commands, including plugins",
+			Usage:  "Don't allow this agent to run arbitrary console commands, including plugins (default: false)",
 			EnvVar: "BUILDKITE_NO_COMMAND_EVAL",
 		},
 		cli.BoolFlag{
 			Name:   "no-plugins",
-			Usage:  "Don't allow this agent to load plugins",
+			Usage:  "Don't allow this agent to load plugins (default: false)",
 			EnvVar: "BUILDKITE_NO_PLUGINS",
 		},
 		cli.BoolTFlag{
 			Name:   "no-plugin-validation",
-			Usage:  "Don't validate plugin configuration and requirements",
+			Usage:  "Don't validate plugin configuration and requirements (default: true)",
 			EnvVar: "BUILDKITE_NO_PLUGIN_VALIDATION",
 		},
 		cli.BoolFlag{
 			Name:   "plugins-always-clone-fresh",
-			Usage:  "Always make a new clone of plugin source, even if already present",
+			Usage:  "Always make a new clone of plugin source, even if already present (default: false)",
 			EnvVar: "BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH",
 		},
 		cli.BoolFlag{
 			Name:   "no-local-hooks",
-			Usage:  "Don't allow local hooks to be run from checked out repositories",
+			Usage:  "Don't allow local hooks to be run from checked out repositories (default: false)",
 			EnvVar: "BUILDKITE_NO_LOCAL_HOOKS",
 		},
 		cli.BoolFlag{
 			Name:   "no-git-submodules",
-			Usage:  "Don't automatically checkout git submodules",
+			Usage:  "Don't automatically checkout git submodules (default: false)",
 			EnvVar: "BUILDKITE_NO_GIT_SUBMODULES,BUILDKITE_DISABLE_GIT_SUBMODULES",
 		},
 		cli.BoolFlag{
 			Name:   "no-feature-reporting",
-			Usage:  "Disables sending a list of enabled features back to the Buildkite mothership. We use this information to measure feature usage, but if you're not comfortable sharing that information then that's totally okay :)",
+			Usage:  "Disables sending a list of enabled features back to the Buildkite mothership. We use this information to measure feature usage, but if you're not comfortable sharing that information then that's totally okay :) (default: false)",
 			EnvVar: "BUILDKITE_AGENT_NO_FEATURE_REPORTING",
 		},
 		cli.StringSliceFlag{
@@ -637,7 +637,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "enable-environment-variable-allowlist",
-			Usage:  "Only run jobs where all environment variables are allowed by the allowed-environment-variables option, or have been set by Buildkite",
+			Usage:  "Only run jobs where all environment variables are allowed by the allowed-environment-variables option, or have been set by Buildkite (default: false)",
 			EnvVar: "BUILDKITE_ENABLE_ENVIRONMENT_VARIABLE_ALLOWLIST",
 		},
 		cli.StringSliceFlag{
@@ -654,7 +654,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "metrics-datadog",
-			Usage:  "Send metrics to DogStatsD for Datadog",
+			Usage:  "Send metrics to DogStatsD for Datadog (default: false)",
 			EnvVar: "BUILDKITE_METRICS_DATADOG",
 		},
 		cli.StringFlag{
@@ -665,7 +665,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "metrics-datadog-distributions",
-			Usage:  "Use Datadog Distributions for Timing metrics",
+			Usage:  "Use Datadog Distributions for Timing metrics (default: false)",
 			EnvVar: "BUILDKITE_METRICS_DATADOG_DISTRIBUTIONS",
 		},
 		cli.StringFlag{
@@ -688,7 +688,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "spawn-with-priority",
-			Usage:  "Assign priorities to every spawned agent (when using --spawn or --spawn-per-cpu) equal to the agent's index",
+			Usage:  "Assign priorities to every spawned agent (when using --spawn or --spawn-per-cpu) equal to the agent's index (default: false)",
 			EnvVar: "BUILDKITE_AGENT_SPAWN_WITH_PRIORITY",
 		},
 		cancelSignalFlag,
@@ -701,7 +701,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "tracing-propagate-traceparent",
-			Usage:  `Enable accepting traceparent context from Buildkite control plane (only supported for OpenTelemetry backend)`,
+			Usage:  `Enable accepting traceparent context from Buildkite control plane (only supported for OpenTelemetry backend) (default: false)`,
 			EnvVar: "BUILDKITE_TRACING_PROPAGATE_TRACEPARENT",
 		},
 		cli.StringFlag{
@@ -732,7 +732,7 @@ var AgentStartCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "debug-signing",
-			Usage:  "Enable debug logging for pipeline signing. This can potentially leak secrets to the logs as it prints each step in full before signing. Requires debug logging to be enabled",
+			Usage:  "Enable debug logging for pipeline signing. This can potentially leak secrets to the logs as it prints each step in full before signing. Requires debug logging to be enabled (default: false)",
 			EnvVar: "BUILDKITE_AGENT_DEBUG_SIGNING",
 		},
 		cli.StringFlag{
@@ -759,7 +759,7 @@ var AgentStartCommand = cli.Command{
 			Name: "kubernetes-exec",
 			Usage: "This is intended to be used only by the Buildkite k8s stack " +
 				"(github.com/buildkite/agent-stack-k8s); it enables a Unix socket for transporting " +
-				"logs and exit statuses between containers in a pod",
+				"logs and exit statuses between containers in a pod (default: false)",
 			EnvVar: "BUILDKITE_KUBERNETES_EXEC",
 		},
 
diff --git a/clicommand/agent_stop.go b/clicommand/agent_stop.go
index 7186aa4069..8c0b918fb2 100644
--- a/clicommand/agent_stop.go
+++ b/clicommand/agent_stop.go
@@ -43,7 +43,7 @@ var AgentStopCommand = cli.Command{
 	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
 		cli.BoolFlag{
 			Name:  "force",
-			Usage: "Cancel any currently running job",
+			Usage: "Cancel any currently running job (default: false)",
 		},
 	}),
 	Action: func(c *cli.Context) error {
diff --git a/clicommand/annotate.go b/clicommand/annotate.go
index 7f8a58d558..458f1a949f 100644
--- a/clicommand/annotate.go
+++ b/clicommand/annotate.go
@@ -87,7 +87,7 @@ var AnnotateCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "append",
-			Usage:  "Append to the body of an existing annotation",
+			Usage:  "Append to the body of an existing annotation (default: false)",
 			EnvVar: "BUILDKITE_ANNOTATION_APPEND",
 		},
 		cli.IntFlag{
diff --git a/clicommand/artifact_download.go b/clicommand/artifact_download.go
index 6bd8b5f811..fff027bee9 100644
--- a/clicommand/artifact_download.go
+++ b/clicommand/artifact_download.go
@@ -77,7 +77,7 @@ var ArtifactDownloadCommand = cli.Command{
 		cli.BoolFlag{
 			Name:   "include-retried-jobs",
 			EnvVar: "BUILDKITE_AGENT_INCLUDE_RETRIED_JOBS",
-			Usage:  "Include artifacts from retried jobs in the search",
+			Usage:  "Include artifacts from retried jobs in the search (default: false)",
 		},
 	}),
 	Action: func(c *cli.Context) error {
diff --git a/clicommand/artifact_search.go b/clicommand/artifact_search.go
index 39ded799b6..f89e863897 100644
--- a/clicommand/artifact_search.go
+++ b/clicommand/artifact_search.go
@@ -101,11 +101,11 @@ var ArtifactSearchCommand = cli.Command{
 		cli.BoolFlag{
 			Name:   "include-retried-jobs",
 			EnvVar: "BUILDKITE_AGENT_INCLUDE_RETRIED_JOBS",
-			Usage:  "Include artifacts from retried jobs in the search",
+			Usage:  "Include artifacts from retried jobs in the search (default: false)",
 		},
 		cli.BoolFlag{
 			Name:  "allow-empty-results",
-			Usage: "By default, searches exit 1 if there are no results. If this flag is set, searches will exit 0 with an empty set",
+			Usage: "By default, searches exit 1 if there are no results. If this flag is set, searches will exit 0 with an empty set (default: false)",
 		},
 		cli.StringFlag{
 			Name:  "format",
diff --git a/clicommand/artifact_shasum.go b/clicommand/artifact_shasum.go
index 1288ed6383..d59ba7a103 100644
--- a/clicommand/artifact_shasum.go
+++ b/clicommand/artifact_shasum.go
@@ -68,7 +68,7 @@ var ArtifactShasumCommand = cli.Command{
 	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
 		cli.BoolFlag{
 			Name:  "sha256",
-			Usage: "Request SHA-256 instead of SHA-1, errors if SHA-256 not available",
+			Usage: "Request SHA-256 instead of SHA-1, errors if SHA-256 not available (default: false)",
 		},
 		cli.StringFlag{
 			Name:  "step",
@@ -84,7 +84,7 @@ var ArtifactShasumCommand = cli.Command{
 		cli.BoolFlag{
 			Name:   "include-retried-jobs",
 			EnvVar: "BUILDKITE_AGENT_INCLUDE_RETRIED_JOBS",
-			Usage:  "Include artifacts from retried jobs in the search",
+			Usage:  "Include artifacts from retried jobs in the search (default: false)",
 		},
 	}),
 	Action: func(c *cli.Context) error {
diff --git a/clicommand/artifact_upload.go b/clicommand/artifact_upload.go
index 0403ff7609..f543ac4eb6 100644
--- a/clicommand/artifact_upload.go
+++ b/clicommand/artifact_upload.go
@@ -106,7 +106,7 @@ var ArtifactUploadCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "literal",
-			Usage:  "Disables parsing of the upload paths as glob patterns; each path will be treated as a single literal file path",
+			Usage:  "Disables parsing of the upload paths as glob patterns; each path will be treated as a single literal file path (default: false)",
 			EnvVar: "BUILDKITE_AGENT_ARTIFACT_LITERAL",
 		},
 		cli.StringFlag{
@@ -117,17 +117,17 @@ var ArtifactUploadCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "glob-resolve-follow-symlinks",
-			Usage:  "Follow symbolic links to directories while resolving globs. Note: this will not prevent symlinks to files from being uploaded. Use --upload-skip-symlinks to do that",
+			Usage:  "Follow symbolic links to directories while resolving globs. Note: this will not prevent symlinks to files from being uploaded. Use --upload-skip-symlinks to do that (default: false)",
 			EnvVar: "BUILDKITE_AGENT_ARTIFACT_GLOB_RESOLVE_FOLLOW_SYMLINKS",
 		},
 		cli.BoolFlag{
 			Name:   "upload-skip-symlinks",
-			Usage:  "After the glob has been resolved to a list of files to upload, skip uploading those that are symlinks to files",
+			Usage:  "After the glob has been resolved to a list of files to upload, skip uploading those that are symlinks to files (default: false)",
 			EnvVar: "BUILDKITE_ARTIFACT_UPLOAD_SKIP_SYMLINKS",
 		},
 		cli.BoolFlag{ // Deprecated
 			Name:   "follow-symlinks",
-			Usage:  "Follow symbolic links while resolving globs. Note this argument is deprecated. Use `--glob-resolve-follow-symlinks` instead",
+			Usage:  "Follow symbolic links while resolving globs. Note this argument is deprecated. Use `--glob-resolve-follow-symlinks` instead (default: false)",
 			EnvVar: "BUILDKITE_AGENT_ARTIFACT_SYMLINKS",
 		},
 		NoMultipartArtifactUploadFlag,
diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go
index 40c6efbda5..62b9912ec2 100644
--- a/clicommand/bootstrap.go
+++ b/clicommand/bootstrap.go
@@ -177,7 +177,7 @@ var BootstrapCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "pull-request-using-merge-refspec",
-			Usage:  "Whether the agent should attempt to checkout the pull request commit using the merge refspec",
+			Usage:  "Whether the agent should attempt to checkout the pull request commit using the merge refspec (default: false)",
 			EnvVar: "BUILDKITE_PULL_REQUEST_USING_MERGE_REFSPEC",
 		},
 		cli.StringFlag{
@@ -224,7 +224,7 @@ var BootstrapCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "clean-checkout",
-			Usage:  "Whether or not the bootstrap should remove the existing repository before running the command",
+			Usage:  "Whether or not the bootstrap should remove the existing repository before running the command (default: false)",
 			EnvVar: "BUILDKITE_CLEAN_CHECKOUT",
 		},
 		cli.StringFlag{
@@ -277,7 +277,7 @@ var BootstrapCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "git-mirrors-skip-update",
-			Usage:  "Skip updating the Git mirror",
+			Usage:  "Skip updating the Git mirror (default: false)",
 			EnvVar: "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE",
 		},
 		cli.StringFlag{
@@ -313,42 +313,42 @@ var BootstrapCommand = cli.Command{
 		},
 		cli.BoolTFlag{
 			Name:   "command-eval",
-			Usage:  "Allow running of arbitrary commands",
+			Usage:  "Allow running of arbitrary commands (default: true)",
 			EnvVar: "BUILDKITE_COMMAND_EVAL",
 		},
 		cli.BoolTFlag{
 			Name:   "plugins-enabled",
-			Usage:  "Allow plugins to be run",
+			Usage:  "Allow plugins to be run (default: true)",
 			EnvVar: "BUILDKITE_PLUGINS_ENABLED",
 		},
 		cli.BoolFlag{
 			Name:   "plugin-validation",
-			Usage:  "Validate plugin configuration",
+			Usage:  "Validate plugin configuration (default: false)",
 			EnvVar: "BUILDKITE_PLUGIN_VALIDATION",
 		},
 		cli.BoolFlag{
 			Name:   "plugins-always-clone-fresh",
-			Usage:  "Always make a new clone of plugin source, even if already present",
+			Usage:  "Always make a new clone of plugin source, even if already present (default: false)",
 			EnvVar: "BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH",
 		},
 		cli.BoolTFlag{
 			Name:   "local-hooks-enabled",
-			Usage:  "Allow local hooks to be run",
+			Usage:  "Allow local hooks to be run (default: true)",
 			EnvVar: "BUILDKITE_LOCAL_HOOKS_ENABLED",
 		},
 		cli.BoolTFlag{
 			Name:   "ssh-keyscan",
-			Usage:  "Automatically run ssh-keyscan before checkout",
+			Usage:  "Automatically run ssh-keyscan before checkout (default: true)",
 			EnvVar: "BUILDKITE_SSH_KEYSCAN",
 		},
 		cli.BoolTFlag{
 			Name:   "git-submodules",
-			Usage:  "Enable git submodules",
+			Usage:  "Enable git submodules (default: true)",
 			EnvVar: "BUILDKITE_GIT_SUBMODULES",
 		},
 		cli.BoolTFlag{
 			Name:   "pty",
-			Usage:  "Run jobs within a pseudo terminal",
+			Usage:  "Run jobs within a pseudo terminal (default: true)",
 			EnvVar: "BUILDKITE_PTY",
 		},
 		cli.StringFlag{
@@ -382,13 +382,13 @@ var BootstrapCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "tracing-propagate-traceparent",
-			Usage:  "Accept traceparent from Buildkite control plane",
+			Usage:  "Accept traceparent from Buildkite control plane (default: false)",
 			EnvVar: "BUILDKITE_TRACING_PROPAGATE_TRACEPARENT",
 		},
 
 		cli.BoolFlag{
 			Name:   "no-job-api",
-			Usage:  "Disables the Job API, which gives commands in jobs some abilities to introspect and mutate the state of the job.",
+			Usage:  "Disables the Job API, which gives commands in jobs some abilities to introspect and mutate the state of the job (default: false)",
 			EnvVar: "BUILDKITE_AGENT_NO_JOB_API",
 		},
 		cli.StringSliceFlag{
diff --git a/clicommand/global.go b/clicommand/global.go
index 559aa9eab5..f8de89eb15 100644
--- a/clicommand/global.go
+++ b/clicommand/global.go
@@ -46,13 +46,13 @@ var (
 
 	NoHTTP2Flag = cli.BoolFlag{
 		Name:   "no-http2",
-		Usage:  "Disable HTTP2 when communicating with the Agent API.",
+		Usage:  "Disable HTTP2 when communicating with the Agent API (default: false)",
 		EnvVar: "BUILDKITE_NO_HTTP2",
 	}
 
 	DebugFlag = cli.BoolFlag{
 		Name:   "debug",
-		Usage:  "Enable debug mode. Synonym for ′--log-level debug′. Takes precedence over ′--log-level′",
+		Usage:  "Enable debug mode. Synonym for ′--log-level debug′. Takes precedence over ′--log-level′ (default: false)",
 		EnvVar: "BUILDKITE_AGENT_DEBUG",
 	}
 
@@ -71,25 +71,25 @@ var (
 
 	DebugHTTPFlag = cli.BoolFlag{
 		Name:   "debug-http",
-		Usage:  "Enable HTTP debug mode, which dumps all request and response bodies to the log",
+		Usage:  "Enable HTTP debug mode, which dumps all request and response bodies to the log (default: false)",
 		EnvVar: "BUILDKITE_AGENT_DEBUG_HTTP",
 	}
 
 	TraceHTTPFlag = cli.BoolFlag{
 		Name:   "trace-http",
-		Usage:  "Enable HTTP trace mode, which logs timings for each HTTP request. Timings are logged at the debug level unless a request fails at the network level in which case they are logged at the error level",
+		Usage:  "Enable HTTP trace mode, which logs timings for each HTTP request. Timings are logged at the debug level unless a request fails at the network level in which case they are logged at the error level (default: false)",
 		EnvVar: "BUILDKITE_AGENT_TRACE_HTTP",
 	}
 
 	NoColorFlag = cli.BoolFlag{
 		Name:   "no-color",
-		Usage:  "Don't show colors in logging",
+		Usage:  "Don't show colors in logging (default: false)",
 		EnvVar: "BUILDKITE_AGENT_NO_COLOR",
 	}
 
 	StrictSingleHooksFlag = cli.BoolFlag{
 		Name:   "strict-single-hooks",
-		Usage:  "Enforces that only one checkout hook, and only one command hook, can be run",
+		Usage:  "Enforces that only one checkout hook, and only one command hook, can be run (default: false)",
 		EnvVar: "BUILDKITE_STRICT_SINGLE_HOOKS",
 	}
 
@@ -117,7 +117,7 @@ var (
 
 	NoMultipartArtifactUploadFlag = cli.BoolFlag{
 		Name:   "no-multipart-artifact-upload",
-		Usage:  "For Buildkite-hosted artifacts, disables the use of multipart uploads. Has no effect on uploads to other destinations such as custom cloud buckets",
+		Usage:  "For Buildkite-hosted artifacts, disables the use of multipart uploads. Has no effect on uploads to other destinations such as custom cloud buckets (default: false)",
 		EnvVar: "BUILDKITE_NO_MULTIPART_ARTIFACT_UPLOAD",
 	}
 
diff --git a/clicommand/oidc_request_token.go b/clicommand/oidc_request_token.go
index bf506e9c47..1a28f6c997 100644
--- a/clicommand/oidc_request_token.go
+++ b/clicommand/oidc_request_token.go
@@ -84,7 +84,7 @@ var OIDCRequestTokenCommand = cli.Command{
 
 		cli.BoolFlag{
 			Name:   "skip-redaction",
-			Usage:  "Skip redacting the OIDC token from the logs. Then, the command will print the token to the Job's logs if called directly.",
+			Usage:  "Skip redacting the OIDC token from the logs. Then, the command will print the token to the Job's logs if called directly (default: false)",
 			EnvVar: "BUILDKITE_AGENT_OIDC_REQUEST_TOKEN_SKIP_TOKEN_REDACTION",
 		},
 		cli.StringFlag{
diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index 31dd51875d..54f04d3182 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -106,7 +106,7 @@ var PipelineUploadCommand = cli.Command{
 	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
 		cli.BoolFlag{
 			Name:   "replace",
-			Usage:  "Replace the rest of the existing pipeline with the steps uploaded. Jobs that are already running are not removed.",
+			Usage:  "Replace the rest of the existing pipeline with the steps uploaded. Jobs that are already running are not removed (default: false)",
 			EnvVar: "BUILDKITE_PIPELINE_REPLACE",
 		},
 		cli.StringFlag{
@@ -117,7 +117,7 @@ var PipelineUploadCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "dry-run",
-			Usage:  "Rather than uploading the pipeline, it will be echoed to stdout",
+			Usage:  "Rather than uploading the pipeline, it will be echoed to stdout (default: false)",
 			EnvVar: "BUILDKITE_PIPELINE_UPLOAD_DRY_RUN",
 		},
 		cli.StringFlag{
@@ -128,17 +128,17 @@ var PipelineUploadCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "no-interpolation",
-			Usage:  "Skip variable interpolation into the pipeline prior to upload",
+			Usage:  "Skip variable interpolation into the pipeline prior to upload (default: false)",
 			EnvVar: "BUILDKITE_PIPELINE_NO_INTERPOLATION",
 		},
 		cli.BoolFlag{
 			Name:   "reject-secrets",
-			Usage:  "When true, fail the pipeline upload early if the pipeline contains secrets",
+			Usage:  "When true, fail the pipeline upload early if the pipeline contains secrets (default: false)",
 			EnvVar: "BUILDKITE_AGENT_PIPELINE_UPLOAD_REJECT_SECRETS",
 		},
 		cli.BoolTFlag{
 			Name:   "apply-if-changed",
-			Usage:  "When enabled, steps containing an ′if_changed′ key are evaluated against the git diff. If the ′if_changed′ glob pattern match no files changed in the build, the step is skipped. Minimum Buildkite Agent version: v3.99 (with --apply-if-changed flag), v3.103.0 (enabled by default)",
+			Usage:  "When enabled, steps containing an ′if_changed′ key are evaluated against the git diff. If the ′if_changed′ glob pattern match no files changed in the build, the step is skipped. Minimum Buildkite Agent version: v3.99 (with --apply-if-changed flag), v3.103.0 (enabled by default) (default: true)",
 			EnvVar: "BUILDKITE_AGENT_APPLY_IF_CHANGED,BUILDKITE_AGENT_APPLY_SKIP_IF_UNCHANGED",
 		},
 		cli.StringFlag{
@@ -171,7 +171,7 @@ var PipelineUploadCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "debug-signing",
-			Usage:  "Enable debug logging for pipeline signing. This can potentially leak secrets to the logs as it prints each step in full before signing. Requires debug logging to be enabled",
+			Usage:  "Enable debug logging for pipeline signing. This can potentially leak secrets to the logs as it prints each step in full before signing. Requires debug logging to be enabled (default: false)",
 			EnvVar: "BUILDKITE_AGENT_DEBUG_SIGNING",
 		},
 		RedactedVars,
diff --git a/clicommand/secret_get.go b/clicommand/secret_get.go
index 40a1ef68e8..aab5228f10 100644
--- a/clicommand/secret_get.go
+++ b/clicommand/secret_get.go
@@ -76,7 +76,7 @@ Examples:
 		},
 		cli.BoolFlag{
 			Name:   "skip-redaction",
-			Usage:  "Skip redacting the retrieved secret from the logs. Then, the command will print the secret to the Job's logs if called directly.",
+			Usage:  "Skip redacting the retrieved secret from the logs. Then, the command will print the secret to the Job's logs if called directly (default: false)",
 			EnvVar: "BUILDKITE_AGENT_SECRET_GET_SKIP_SECRET_REDACTION",
 		},
 	}),
diff --git a/clicommand/step_cancel.go b/clicommand/step_cancel.go
index 96e97bf0a5..ede19eafae 100644
--- a/clicommand/step_cancel.go
+++ b/clicommand/step_cancel.go
@@ -57,7 +57,7 @@ var StepCancelCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "force",
-			Usage:  "Transition unfinished jobs to a canceled state instead of waiting for jobs to finish uploading artifacts",
+			Usage:  "Transition unfinished jobs to a canceled state instead of waiting for jobs to finish uploading artifacts (default: false)",
 			EnvVar: "BUILDKITE_STEP_CANCEL_FORCE",
 		},
 
diff --git a/clicommand/step_update.go b/clicommand/step_update.go
index e8693db203..e455544b12 100644
--- a/clicommand/step_update.go
+++ b/clicommand/step_update.go
@@ -74,7 +74,7 @@ var StepUpdateCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "append",
-			Usage:  "Append to current attribute instead of replacing it",
+			Usage:  "Append to current attribute instead of replacing it (default: false)",
 			EnvVar: "BUILDKITE_STEP_UPDATE_APPEND",
 		},
 		RedactedVars,
diff --git a/clicommand/tool_sign.go b/clicommand/tool_sign.go
index 5dc9d984b5..3bbb455b58 100644
--- a/clicommand/tool_sign.go
+++ b/clicommand/tool_sign.go
@@ -109,12 +109,12 @@ Signing a pipeline from a file:
 		},
 		cli.BoolFlag{
 			Name:   "update",
-			Usage:  "Update the pipeline using the GraphQL API after signing it. This can only be used if ′graphql-token′ is provided.",
+			Usage:  "Update the pipeline using the GraphQL API after signing it. This can only be used if ′graphql-token′ is provided (default: false)",
 			EnvVar: "BUILDKITE_TOOL_SIGN_UPDATE",
 		},
 		cli.BoolFlag{
 			Name:   "no-confirm",
-			Usage:  "Show confirmation prompts before updating the pipeline with the GraphQL API.",
+			Usage:  "Show confirmation prompts before updating the pipeline with the GraphQL API (default: false)",
 			EnvVar: "BUILDKITE_TOOL_SIGN_NO_CONFIRM",
 		},
 
@@ -136,7 +136,7 @@ Signing a pipeline from a file:
 		},
 		cli.BoolFlag{
 			Name:   "debug-signing",
-			Usage:  "Enable debug logging for pipeline signing. This can potentially leak secrets to the logs as it prints each step in full before signing. Requires debug logging to be enabled",
+			Usage:  "Enable debug logging for pipeline signing. This can potentially leak secrets to the logs as it prints each step in full before signing. Requires debug logging to be enabled (default: false)",
 			EnvVar: "BUILDKITE_AGENT_DEBUG_SIGNING",
 		},
 

From 4365de25efdaf67777e8beda3b867d871f85f7db Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 26 Jan 2026 17:31:19 +0000
Subject: [PATCH 163/242] build(deps): bump the container-images group across 1
 directory with 2 updates

Bumps the container-images group with 2 updates in the /.buildkite directory: docker/library/golang and golangci/golangci-lint.


Updates `docker/library/golang` from 1.24.11 to 1.24.12

Updates `golangci/golangci-lint` from v2.7-alpine to v2.8-alpine

---
updated-dependencies:
- dependency-name: docker/library/golang
  dependency-version: 1.24.12
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: container-images
- dependency-name: golangci/golangci-lint
  dependency-version: v2.8-alpine
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-compile | 2 +-
 .buildkite/Dockerfile-e2e     | 2 +-
 .buildkite/Dockerfile-lint    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index e215645152..c7b5f2815b 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:54528d189affe7ce60840cad093d53360705d47d71ea74d62eef72c476242fe9
+FROM public.ecr.aws/docker/library/golang:1.24.12@sha256:c2131140c7c29ff277b1c412d524b7f56289513f49672c57a3d992247dd146f8
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest
diff --git a/.buildkite/Dockerfile-e2e b/.buildkite/Dockerfile-e2e
index fa23d09f6a..61225cc883 100644
--- a/.buildkite/Dockerfile-e2e
+++ b/.buildkite/Dockerfile-e2e
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.11@sha256:54528d189affe7ce60840cad093d53360705d47d71ea74d62eef72c476242fe9
+FROM public.ecr.aws/docker/library/golang:1.24.12@sha256:c2131140c7c29ff277b1c412d524b7f56289513f49672c57a3d992247dd146f8
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     unzip \
diff --git a/.buildkite/Dockerfile-lint b/.buildkite/Dockerfile-lint
index 0ae92eaf0a..cf846803b6 100644
--- a/.buildkite/Dockerfile-lint
+++ b/.buildkite/Dockerfile-lint
@@ -1 +1 @@
-FROM golangci/golangci-lint:v2.7-alpine@sha256:1e1851102b736971267400e08b3e4b2e7799c73976a998820f6f6b6b86b48343
\ No newline at end of file
+FROM golangci/golangci-lint:v2.8-alpine@sha256:1194f3bfcbaeeb92d8d159fdfbe2a79d18ec0a222d9d984b1438906bca416b51
\ No newline at end of file

From eb2d1f8e060d3498b1c53cf829a1e3fac1443b03 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 26 Jan 2026 17:31:42 +0000
Subject: [PATCH 164/242] build(deps): bump the container-images group across 4
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `174a2a9` to `8e5b3e9`

Updates `buildkite/agent-base` from `7ca08c1` to `f8be5e4`

Updates `buildkite/agent-base` from `6cb7bd9` to `558fc38`

Updates `buildkite/agent-base` from `797f9d7` to `480db1d`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index b1b14ac6b2..b52389eea0 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:7ca08c1e474eb971088deea6d356d6a1f17be62d4ac7a0ad383b437868f47d26
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:f8be5e489c117a7c99d43cf328ae602172b5261e5d898c8103782c8852d09d41
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 12c765b1e3..f9f51e0e79 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:174a2a911c8949efafbecb5374b17f5b4a9341a097e24f2c13396969a2f865df
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:8e5b3e97c62fce1249625e10bafaba2d5ce2bc135c0a983322a0d12d257ae625
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index bbcb9342e3..86b70848b1 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:6cb7bd99b2b774cae9b30bf3af1c5581ff5d4b0fa34cb37279cb04270dea7781
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:558fc38e756f454a6856e4ff120dc21b4e8df9f259447d85abf5093057703ac2
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index ef64c82ad7..39823a00b9 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:797f9d79ef9f8ed8fcbe4bc22dbeafd58d4dc109369e40b8b770efb275006bad
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:480db1d597c391371ffe7d5f77c9a455b86190816aff5a8a37deeee41dbe88f6
 ARG TARGETOS
 ARG TARGETARCH
 

From ca89b3f8d4613a0564859080f673f517ffdc3cb8 Mon Sep 17 00:00:00 2001
From: Lachlan <lachlan@buildkite.com>
Date: Tue, 27 Jan 2026 11:30:57 +1100
Subject: [PATCH 165/242] fix: check for nil response before accessing
 StatusCode in meta-data get

When the API request times out or fails with a network error, the response
is nil. Accessing resp.StatusCode without a nil check causes a panic.

Amp-Thread-ID: https://ampcode.com/threads/T-019bfcda-b7d4-726d-a58e-1190729c8b6d
---
 clicommand/meta_data_get.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/meta_data_get.go b/clicommand/meta_data_get.go
index aa4346f66a..a2be79352e 100644
--- a/clicommand/meta_data_get.go
+++ b/clicommand/meta_data_get.go
@@ -98,7 +98,7 @@ var MetaDataGetCommand = cli.Command{
 			//
 			// We also use `IsSet` instead of `cfg.Default != ""`
 			// to allow people to use a default of a blank string.
-			if resp.StatusCode == 404 && c.IsSet("default") {
+			if resp != nil && resp.StatusCode == 404 && c.IsSet("default") {
 				l.Warn(
 					"No meta-data value exists with key %q, returning the supplied default %q",
 					cfg.Key,

From a7b8aef2396d1d8685499ec5ba7dc23c2b002218 Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Tue, 27 Jan 2026 14:41:20 +1100
Subject: [PATCH 166/242] PS-1575: in k8s mode, write BUILDKITE_ENV_FILE to
 /workspace

---
 agent/job_runner.go | 55 ++++++++++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 21 deletions(-)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index d71a7d974c..fde113983e 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -199,29 +199,10 @@ func NewJobRunner(ctx context.Context, l logger.Logger, apiClient *api.Client, c
 		},
 	)
 
-	// TempDir is not guaranteed to exist
-	tempDir := os.TempDir()
-	if _, err := os.Stat(tempDir); os.IsNotExist(err) {
-		// Actual file permissions will be reduced by umask, and won't be 0o777 unless the user has manually changed the umask to 000
-		if err = os.MkdirAll(tempDir, 0o777); err != nil {
-			return nil, err
-		}
-	}
-
-	// Prepare a file to receive the given job environment
-	file, err := os.CreateTemp(tempDir, fmt.Sprintf("job-env-%s", r.conf.Job.ID))
+	r.envShellFile, r.envJSONFile, err = createJobEnvFiles(r.agentLogger, r.conf.Job.ID, conf.KubernetesExec)
 	if err != nil {
-		return r, err
-	}
-	r.agentLogger.Debug("[JobRunner] Created env file (shell format): %s", file.Name())
-	r.envShellFile = file
-
-	file, err = os.CreateTemp(tempDir, fmt.Sprintf("job-env-json-%s", r.conf.Job.ID))
-	if err != nil {
-		return r, err
+		return nil, err
 	}
-	r.agentLogger.Debug("[JobRunner] Created env file (JSON format): %s", file.Name())
-	r.envJSONFile = file
 
 	env, err := r.createEnvironment(ctx)
 	if err != nil {
@@ -891,3 +872,35 @@ func (l jobLogger) Write(data []byte) (int, error) {
 	l.log.Info(msg)
 	return len(data), nil
 }
+
+func createJobEnvFiles(l logger.Logger, jobID string, kubernetesExec bool) (shellFile, jsonFile *os.File, err error) {
+	// Use /workspace in Kubernetes mode for shared volume access between containers
+	tempDir := os.TempDir()
+	if kubernetesExec {
+		tempDir = "/workspace"
+	}
+
+	// tempDir is not guaranteed to exist
+	if _, err := os.Stat(tempDir); os.IsNotExist(err) {
+		// Actual file permissions will be reduced by umask, and won't be 0o777 unless the user has manually changed the umask to 000
+		if err = os.MkdirAll(tempDir, 0o777); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	shellFile, err = os.CreateTemp(tempDir, fmt.Sprintf("job-env-%s", jobID))
+	if err != nil {
+		return nil, nil, err
+	}
+	l.Debug("[JobRunner] Created env file (shell format): %s", shellFile.Name())
+
+	jsonFile, err = os.CreateTemp(tempDir, fmt.Sprintf("job-env-json-%s", jobID))
+	if err != nil {
+		shellFile.Close()
+		os.Remove(shellFile.Name())
+		return nil, nil, err
+	}
+	l.Debug("[JobRunner] Created env file (JSON format): %s", jsonFile.Name())
+
+	return shellFile, jsonFile, nil
+}

From 0181ea5a72f6f0f5e5cc8cd453327db3eaaa68cb Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 27 Jan 2026 16:52:13 +1100
Subject: [PATCH 167/242] [fix] Repeated plugins run correct # times with
 always-clone-fresh

---
 internal/e2e/fixtures/repeated_plugin.yaml |  9 ++++++
 internal/e2e/plugin_test.go                | 32 ++++++++++++++++++++++
 internal/job/plugin.go                     | 11 ++++++++
 3 files changed, 52 insertions(+)
 create mode 100644 internal/e2e/fixtures/repeated_plugin.yaml
 create mode 100644 internal/e2e/plugin_test.go

diff --git a/internal/e2e/fixtures/repeated_plugin.yaml b/internal/e2e/fixtures/repeated_plugin.yaml
new file mode 100644
index 0000000000..eefa7cb9d0
--- /dev/null
+++ b/internal/e2e/fixtures/repeated_plugin.yaml
@@ -0,0 +1,9 @@
+agents:
+  queue: "{{.queue}}"
+steps:
+  - command: echo 'Hello from the command'
+    env:
+      BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH: "true"
+    plugins:
+      - plugin-test#d2df1d8
+      - plugin-test#d2df1d8
diff --git a/internal/e2e/plugin_test.go b/internal/e2e/plugin_test.go
new file mode 100644
index 0000000000..c674873c51
--- /dev/null
+++ b/internal/e2e/plugin_test.go
@@ -0,0 +1,32 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func TestPluginE2E(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, "repeated_plugin.yaml")
+
+	tc.startAgent()
+	build := tc.triggerBuild()
+	state := tc.waitForBuild(ctx, build)
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+
+	logs := tc.fetchLogs(ctx, build)
+
+	hooks := []string{"environment", "pre-checkout", "post-checkout", "pre-command", "post-command"}
+	for _, h := range hooks {
+		needle := fmt.Sprintf("Hello from the plugin-test-plugin %s hook", h)
+		if got, want := strings.Count(logs, needle), 2; got != want {
+			t.Errorf("tc.fetchLogs(ctx, build %q) logs as follows, contained %d copies of %q, want %d", build.ID, got, needle, want)
+		}
+	}
+	t.Log(logs)
+}
diff --git a/internal/job/plugin.go b/internal/job/plugin.go
index 91345b8f2d..8fe722394f 100644
--- a/internal/job/plugin.go
+++ b/internal/job/plugin.go
@@ -124,6 +124,8 @@ func (e *Executor) PluginPhase(ctx context.Context) error {
 		return nil
 	}
 
+	// If the same plugin is used multiple times, only check it out once.
+	checkoutsByLabel := make(map[string]*pluginCheckout)
 	checkouts := []*pluginCheckout{}
 
 	// Checkout and validate plugins that aren't vendored
@@ -135,6 +137,14 @@ func (e *Executor) PluginPhase(ctx context.Context) error {
 			continue
 		}
 
+		// Already checked out (in this job)?
+		if checkout := checkoutsByLabel[p.Label()]; checkout != nil {
+			checkout2 := *checkout
+			checkout2.Plugin = p
+			checkouts = append(checkouts, &checkout2)
+			continue
+		}
+
 		checkout, err := e.checkoutPlugin(ctx, p)
 		if err != nil {
 			return fmt.Errorf("Failed to checkout plugin %s: %w", p.Name(), err)
@@ -145,6 +155,7 @@ func (e *Executor) PluginPhase(ctx context.Context) error {
 			return err
 		}
 
+		checkoutsByLabel[p.Label()] = checkout
 		checkouts = append(checkouts, checkout)
 	}
 

From 94ecd17659e6d8137ae42b4ea7d90e8019867ab5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 28 Jan 2026 00:04:57 +0000
Subject: [PATCH 168/242] build(deps): bump the container-images group across 5
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `dc83378` to `1ac9d5d`

Updates `buildkite/agent-base` from `12dcb2d` to `59e23a6`

Updates `buildkite/agent-base` from `dc768a6` to `e884e55`

Updates `buildkite/agent-base` from `a0ba149` to `56648a1`

Updates `buildkite/agent-base` from `b3d073a` to `67899c3`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index b52389eea0..4ab14e277b 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:f8be5e489c117a7c99d43cf328ae602172b5261e5d898c8103782c8852d09d41
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:2ffa7f782edbf46a9f283b0c326b5ec0f44c7d690db4fd50040f73013e13f29f
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index f9f51e0e79..9cac337183 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:8e5b3e97c62fce1249625e10bafaba2d5ce2bc135c0a983322a0d12d257ae625
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:2dca5c55276726181d090ab89ac9a130e2818df9159de3dc4557db1931dff84b
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 86b70848b1..5c2365a683 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:558fc38e756f454a6856e4ff120dc21b4e8df9f259447d85abf5093057703ac2
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:3851483427a45549035f219464e1d1d45f628bb1898318de218913e8adc3363d
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index 9ff5571499..b3a0608d61 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:14db4ab933b71023690acbceca9365d8f538dc6f3d3bb7c41c42c810ccc67463
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:fdf8b6dc594555da27bbea0fe37e1c9249caf0b5f4d7ecc41d844f81bdd00ec2
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index 39823a00b9..9e34e94745 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:480db1d597c391371ffe7d5f77c9a455b86190816aff5a8a37deeee41dbe88f6
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:e1bf16dc4c2c7b1fa382322c52e3edee632b4491166af33fb83be58dad5d7886
 ARG TARGETOS
 ARG TARGETARCH
 

From b9845707130565970b5edc1981b61bd528d2df76 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 28 Jan 2026 00:17:59 +0000
Subject: [PATCH 169/242] build(deps): bump the golang-x group with 4 updates

Bumps the golang-x group with 4 updates: [golang.org/x/crypto](https://github.com/golang/crypto), [golang.org/x/net](https://github.com/golang/net), [golang.org/x/sys](https://github.com/golang/sys) and [golang.org/x/term](https://github.com/golang/term).


Updates `golang.org/x/crypto` from 0.46.0 to 0.47.0
- [Commits](https://github.com/golang/crypto/compare/v0.46.0...v0.47.0)

Updates `golang.org/x/net` from 0.48.0 to 0.49.0
- [Commits](https://github.com/golang/net/compare/v0.48.0...v0.49.0)

Updates `golang.org/x/sys` from 0.39.0 to 0.40.0
- [Commits](https://github.com/golang/sys/compare/v0.39.0...v0.40.0)

Updates `golang.org/x/term` from 0.38.0 to 0.39.0
- [Commits](https://github.com/golang/term/compare/v0.38.0...v0.39.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.47.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/net
  dependency-version: 0.49.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/sys
  dependency-version: 0.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/term
  dependency-version: 0.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 14 +++++++-------
 go.sum | 28 ++++++++++++++--------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/go.mod b/go.mod
index b26f8063aa..63f0a951b8 100644
--- a/go.mod
+++ b/go.mod
@@ -56,12 +56,12 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0
 	go.opentelemetry.io/otel/sdk v1.39.0
 	go.opentelemetry.io/otel/trace v1.39.0
-	golang.org/x/crypto v0.46.0
-	golang.org/x/net v0.48.0
+	golang.org/x/crypto v0.47.0
+	golang.org/x/net v0.49.0
 	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
-	golang.org/x/sys v0.39.0
-	golang.org/x/term v0.38.0
+	golang.org/x/sys v0.40.0
+	golang.org/x/term v0.39.0
 	google.golang.org/api v0.260.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
@@ -198,10 +198,10 @@ require (
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
-	golang.org/x/mod v0.30.0 // indirect
-	golang.org/x/text v0.32.0 // indirect
+	golang.org/x/mod v0.31.0 // indirect
+	golang.org/x/text v0.33.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.39.0 // indirect
+	golang.org/x/tools v0.40.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
diff --git a/go.sum b/go.sum
index 31c347324a..ee2a9870f1 100644
--- a/go.sum
+++ b/go.sum
@@ -491,22 +491,22 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
-golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
-golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
+golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
+golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
-golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
 golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
 golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -528,15 +528,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
-golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
-golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
+golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
-golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
 golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -544,8 +544,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
-golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
+golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
+golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From e519980277a71240f8df7acb7dc668f6d2cd08a0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 28 Jan 2026 00:18:17 +0000
Subject: [PATCH 170/242] build(deps): bump github.com/google/go-querystring
 from 1.1.0 to 1.2.0

Bumps [github.com/google/go-querystring](https://github.com/google/go-querystring) from 1.1.0 to 1.2.0.
- [Release notes](https://github.com/google/go-querystring/releases)
- [Commits](https://github.com/google/go-querystring/compare/v1.1.0...v1.2.0)

---
updated-dependencies:
- dependency-name: github.com/google/go-querystring
  dependency-version: 1.2.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index b26f8063aa..62df345560 100644
--- a/go.mod
+++ b/go.mod
@@ -35,7 +35,7 @@ require (
 	github.com/go-chi/chi/v5 v5.2.4
 	github.com/gofrs/flock v0.13.0
 	github.com/google/go-cmp v0.7.0
-	github.com/google/go-querystring v1.1.0
+	github.com/google/go-querystring v1.2.0
 	github.com/google/uuid v1.6.0
 	github.com/gowebpki/jcs v1.0.1
 	github.com/lestrrat-go/jwx/v2 v2.1.6
diff --git a/go.sum b/go.sum
index 31c347324a..59bad2b568 100644
--- a/go.sum
+++ b/go.sum
@@ -218,11 +218,11 @@ github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/
 github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
-github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
+github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=

From 1f2dce231fe1831d8526e624086c859eb2e7db70 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 28 Jan 2026 15:33:33 +1100
Subject: [PATCH 171/242] Exit with non-zero code if ping or heartbeat fail
 unrecoverably

---
 agent/agent_worker.go         | 80 ++++++++++++++++++++++-------------
 agent/agent_worker_test.go    | 55 ++++++++++++++++++++++++
 agent/fake_api_server_test.go | 33 +++++++++++----
 3 files changed, 130 insertions(+), 38 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 556e15f6cf..d6af2c7ffc 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -163,9 +163,9 @@ func (e *errUnrecoverable) Error() string {
 	return fmt.Sprintf("%s failed with unrecoverable status: %s, mesage: %q", e.action, status, e.err)
 }
 
-func (e *errUnrecoverable) Is(other error) bool {
-	_, ok := other.(*errUnrecoverable)
-	return ok
+func isUnrecoverable(err error) bool {
+	var u *errUnrecoverable
+	return errors.As(err, &u)
 }
 
 func (e *errUnrecoverable) Unwrap() error {
@@ -233,11 +233,12 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	}
 	defer a.metricsCollector.Stop() //nolint:errcheck // Best-effort cleanup
 
-	// Use a context to run heartbeats for as long as the ping loop or job runs
-	heartbeatCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
+	// errCh receives 1 value from the heartbeat loop, and 1 from the ping loop.
+	errCh := make(chan error, 2)
 
-	go a.runHeartbeatLoop(heartbeatCtx)
+	go func() {
+		errCh <- a.runHeartbeatLoop(ctx)
+	}()
 
 	// If the agent is booted in acquisition mode, acquire that particular job
 	// before running the ping loop.
@@ -253,6 +254,7 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		if err := a.AcquireAndRunJob(ctx, a.agentConfiguration.AcquireJob); err != nil {
 			// If the job acquisition was rejected, we can exit with an error
 			// so that supervisor knows that the job was not acquired due to the job being rejected.
+			a.internalStop() // stop the heartbeat loop
 			if errors.Is(err, core.ErrJobAcquisitionRejected) {
 				return fmt.Errorf("Failed to acquire job %q: %w", a.agentConfiguration.AcquireJob, err)
 			}
@@ -270,10 +272,17 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		}
 	}
 
-	return a.runPingLoop(ctx, idleMon)
+	errCh <- a.runPingLoop(ctx, idleMon)
+
+	// Blocks until both heartbeat and ping loops have returned.
+	// Both loops are context aware, so no need to wait on ctx here.
+	for range 2 {
+		startErr = errors.Join(startErr, <-errCh)
+	}
+	return startErr
 }
 
-func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) {
+func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) error {
 	ctx, setStat, _ := status.AddSimpleItem(ctx, "Heartbeat loop")
 	defer setStat("💔 Heartbeat loop stopped!")
 	setStat("🏃 Starting...")
@@ -287,26 +296,32 @@ func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) {
 		case <-heartbeatTicker.C:
 			setStat("❤️ Sending heartbeat")
 			if err := a.Heartbeat(ctx); err != nil {
-				if errors.Is(err, &errUnrecoverable{}) {
+				if isUnrecoverable(err) {
 					a.logger.Error("%s", err)
-					return
+					// unrecoverable heartbeat failure also stops ping loop
+					a.StopUngracefully()
+					return err
 				}
 
 				// Get the last heartbeat time to the nearest microsecond
 				a.stats.Lock()
 				if a.stats.lastHeartbeat.IsZero() {
-					a.logger.Error("Failed to heartbeat %s. Will try again in %s. (No heartbeat yet)",
+					a.logger.Error("Failed to heartbeat %s. Will try again in %v. (No heartbeat yet)",
 						err, heartbeatInterval)
 				} else {
-					a.logger.Error("Failed to heartbeat %s. Will try again in %s. (Last successful was %v ago)",
+					a.logger.Error("Failed to heartbeat %s. Will try again in %v. (Last successful was %v ago)",
 						err, heartbeatInterval, time.Since(a.stats.lastHeartbeat))
 				}
 				a.stats.Unlock()
 			}
 
+		case <-a.stop:
+			a.logger.Debug("Stopping heartbeats due to agent stop")
+			return nil
+
 		case <-ctx.Done():
-			a.logger.Debug("Stopping heartbeats")
-			return
+			a.logger.Debug("Stopping heartbeats due to context cancel")
+			return ctx.Err()
 		}
 	}
 }
@@ -319,6 +334,9 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 	defer setStat("🛑 Ping loop stopped!")
 	setStat("🏃 Starting...")
 
+	// Everything stops once the ping loop stops
+	defer a.internalStop()
+
 	// Create the ticker
 	pingInterval := time.Second * time.Duration(a.agent.PingInterval)
 	pingTicker := time.NewTicker(pingInterval)
@@ -362,8 +380,10 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		case <-pingTicker.C:
 			// continue below
 		case <-a.stop:
+			a.logger.Debug("Stopping pings due to agent stop")
 			return nil
 		case <-ctx.Done():
+			a.logger.Debug("Stopping pings due to context cancel")
 			return ctx.Err()
 		}
 
@@ -377,8 +397,10 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		case <-time.After(jitter):
 			// continue below
 		case <-a.stop:
+			a.logger.Debug("Stopping pings due to agent stop")
 			return nil
 		case <-ctx.Done():
+			a.logger.Debug("Stopping pings due to context cancel")
 			return ctx.Err()
 		}
 		pingWaitDurations.Observe(time.Since(startWait).Seconds())
@@ -389,11 +411,11 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		job, action, err := a.Ping(ctx)
 		if err != nil {
 			pingErrors.Inc()
-			if errors.Is(err, &errUnrecoverable{}) {
+			if isUnrecoverable(err) {
 				a.logger.Error("%v", err)
-			} else {
-				a.logger.Warn("%v", err)
+				return err
 			}
+			a.logger.Warn("%v", err)
 		}
 		pingDurations.Observe(time.Since(startPing).Seconds())
 
@@ -499,6 +521,14 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 	}
 }
 
+func (a *AgentWorker) internalStop() {
+	a.stopOnce.Do(func() {
+		// Use the closure of the stop channel as a signal to the main run
+		// loop in Start() to stop looping and terminate
+		close(a.stop)
+	})
+}
+
 // StopGracefully stops the agent from accepting new work. It allows the current
 // job to finish without interruption. Does not block.
 func (a *AgentWorker) StopGracefully() {
@@ -519,21 +549,13 @@ func (a *AgentWorker) StopGracefully() {
 		a.logger.Info("Gracefully stopping agent. Since there is no job running, the agent will disconnect immediately")
 	}
 
-	a.stopOnce.Do(func() {
-		// Use the closure of the stop channel as a signal to the main run
-		// loop in Start() to stop looping and terminate
-		close(a.stop)
-	})
+	a.internalStop()
 }
 
 // StopUngracefully stops the agent from accepting new work and cancels any
 // existing job. It blocks until the job is cancelled, if there is one.
 func (a *AgentWorker) StopUngracefully() {
-	a.stopOnce.Do(func() {
-		// Use the closure of the stop channel as a signal to the main run
-		// loop in Start() to stop looping and terminate
-		close(a.stop)
-	})
+	a.internalStop()
 
 	// If there's a job running, kill it, then disconnect.
 	if jr := a.jobRunner.Load(); jr != nil {
@@ -568,7 +590,6 @@ func (a *AgentWorker) Heartbeat(ctx context.Context) error {
 		b, resp, err := a.apiClient.Heartbeat(ctx)
 		if err != nil {
 			if resp != nil && !api.IsRetryableStatus(resp) {
-				a.StopUngracefully()
 				r.Break()
 				return nil, &errUnrecoverable{action: "Heartbeat", response: resp, err: err}
 			}
@@ -618,7 +639,6 @@ func (a *AgentWorker) Ping(ctx context.Context) (job *api.Job, action string, er
 		// The reason we do this after the disconnect check is because the backend can (and does) send disconnect actions in
 		// responses with non-retryable statuses
 		if resp != nil && !api.IsRetryableStatus(resp) {
-			a.StopUngracefully()
 			return nil, action, &errUnrecoverable{action: "Ping", response: resp, err: pingErr}
 		}
 
diff --git a/agent/agent_worker_test.go b/agent/agent_worker_test.go
index 67053b32f0..ce33d0c878 100644
--- a/agent/agent_worker_test.go
+++ b/agent/agent_worker_test.go
@@ -999,3 +999,58 @@ func TestAgentWorker_UpdateRequestHeadersDuringPing(t *testing.T) {
 		t.Errorf("agent.Pings = %d, want %d", got, want)
 	}
 }
+
+func TestAgentWorker_UnrecoverableErrorInPing(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	const agentSessionToken = "alpacas"
+
+	server := NewFakeAPIServer()
+	defer server.Close()
+
+	const headerKey = "Buildkite-Hello"
+	const headerValue = "world"
+
+	agent := server.AddAgent(agentSessionToken)
+	agent.PingHandler = func(req *http.Request) (api.Ping, error) {
+		// Invalidate the token to trigger an unrecoverable error on
+		// subsequent pings.
+		server.DeleteAgent(agentSessionToken)
+		return api.Ping{}, nil
+	}
+
+	apiClient := api.NewClient(logger.Discard, api.Config{
+		Endpoint: server.URL,
+		Token:    "llamas",
+	})
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(int) {})
+
+	worker := NewAgentWorker(
+		l,
+		&api.AgentRegisterResponse{
+			UUID:              uuid.New().String(),
+			Name:              "agent-1",
+			AccessToken:       agentSessionToken,
+			Endpoint:          server.URL,
+			PingInterval:      1,
+			JobStatusInterval: 5,
+			HeartbeatInterval: 60,
+		},
+		metrics.NewCollector(logger.Discard, metrics.CollectorConfig{}),
+		apiClient,
+		AgentWorkerConfig{},
+	)
+	worker.noWaitBetweenPingsForTesting = true
+
+	if err := worker.Start(ctx, newIdleMonitor(1)); !isUnrecoverable(err) {
+		t.Errorf("worker.Start() = %v, want an unrecoverable error", err)
+	}
+
+	if got, want := agent.Pings, 1; got != want {
+		t.Errorf("agent.Pings = %d, want %d", got, want)
+	}
+}
diff --git a/agent/fake_api_server_test.go b/agent/fake_api_server_test.go
index 52b93ed5f3..295cd070cb 100644
--- a/agent/fake_api_server_test.go
+++ b/agent/fake_api_server_test.go
@@ -58,8 +58,10 @@ type agentJob struct {
 type FakeAPIServer struct {
 	*httptest.Server
 
+	agentsMu sync.Mutex
+	agents   map[string]*FakeAgent // session token Auth header -> agent
+
 	mu            sync.Mutex
-	agents        map[string]*FakeAgent                 // session token Auth header -> agent
 	jobs          map[string]*FakeJob                   // uuid -> job
 	agentJobs     map[string]agentJob                   // job token Auth header -> (agent, job)
 	registrations map[string]*api.AgentRegisterResponse // reg token Auth header -> response
@@ -87,13 +89,25 @@ func NewFakeAPIServer() *FakeAPIServer {
 }
 
 func (fs *FakeAPIServer) AddAgent(token string) *FakeAgent {
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
+	fs.agentsMu.Lock()
+	defer fs.agentsMu.Unlock()
 	a := &FakeAgent{}
 	fs.agents["Token "+token] = a
 	return a
 }
 
+func (fs *FakeAPIServer) DeleteAgent(token string) {
+	fs.agentsMu.Lock()
+	defer fs.agentsMu.Unlock()
+	delete(fs.agents, "Token "+token)
+}
+
+func (fs *FakeAPIServer) agentForAuth(auth string) *FakeAgent {
+	fs.agentsMu.Lock()
+	defer fs.agentsMu.Unlock()
+	return fs.agents[auth]
+}
+
 func (fs *FakeAPIServer) AddJob(env map[string]string) *FakeJob {
 	fs.mu.Lock()
 	defer fs.mu.Unlock()
@@ -113,8 +127,10 @@ func (fs *FakeAPIServer) AddJob(env map[string]string) *FakeJob {
 }
 
 func (fs *FakeAPIServer) Assign(agent *FakeAgent, job *FakeJob) {
+	fs.agentsMu.Lock()
 	fs.mu.Lock()
 	defer fs.mu.Unlock()
+	defer fs.agentsMu.Unlock()
 	fs.assignNoMutex(agent, job)
 }
 
@@ -140,7 +156,7 @@ func (fs *FakeAPIServer) handleJobAcquire(rw http.ResponseWriter, req *http.Requ
 	// The agent doesn't know the job token yet, so it must use the session
 	// token.
 	auth := req.Header.Get("Authorization")
-	agent := fs.agents[auth]
+	agent := fs.agentForAuth(auth)
 	if agent == nil {
 		http.Error(rw, encodeMsgf("invalid Authorization header value %q", auth), http.StatusUnauthorized)
 		return
@@ -182,7 +198,7 @@ func (fs *FakeAPIServer) handleJobAccept(rw http.ResponseWriter, req *http.Reque
 
 	// The agent has the job info from the ping, but accepts as itself.
 	auth := req.Header.Get("Authorization")
-	agent := fs.agents[auth]
+	agent := fs.agentForAuth(auth)
 	if agent == nil {
 		http.Error(rw, encodeMsgf("invalid Authorization header value %q", auth), http.StatusUnauthorized)
 		return
@@ -317,7 +333,7 @@ func (fs *FakeAPIServer) handlePing(rw http.ResponseWriter, req *http.Request) {
 	var ping api.Ping
 
 	auth := req.Header.Get("Authorization")
-	agent := fs.agents[auth]
+	agent := fs.agentForAuth(auth)
 	if agent == nil {
 		http.Error(rw, encodeMsgf("invalid Authorization header value %q", auth), http.StatusUnauthorized)
 		return
@@ -361,9 +377,10 @@ func (fs *FakeAPIServer) handleHeartbeat(rw http.ResponseWriter, req *http.Reque
 	fs.mu.Lock()
 	defer fs.mu.Unlock()
 
-	agent := fs.agents[req.Header.Get("Authorization")]
+	auth := req.Header.Get("Authorization")
+	agent := fs.agentForAuth(auth)
 	if agent == nil {
-		http.Error(rw, encodeMsg("unauthorized"), http.StatusUnauthorized)
+		http.Error(rw, encodeMsgf("invalid Authorization header value %q", auth), http.StatusUnauthorized)
 		return
 	}
 

From 4b25eafe2439ae2157872c8eda249a849e8f0afa Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Wed, 28 Jan 2026 14:44:44 +1100
Subject: [PATCH 172/242] release/3.116.0

---
 CHANGELOG.md    | 19 +++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8db63317fc..bba00fb4fc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.116.0](https://github.com/buildkite/agent/tree/v3.116.0) (2026-01-28)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.115.4...v3.116.0)
+
+### Added
+- Support checkout skipping in agent [#3672](https://github.com/buildkite/agent/pull/3672) (@mcncl)
+- Add default BoolFlag, BoolTFlag values to descriptions [#3678](https://github.com/buildkite/agent/pull/3678) (@petetomasik)
+
+### Fixed
+- Exit with non-zero status if ping or heartbeat fail unrecoverably [#3687](https://github.com/buildkite/agent/pull/3687) (@DrJosh9000)
+- Repeated plugins run correct number of times with always-clone-fresh [#3684](https://github.com/buildkite/agent/pull/3684) (@DrJosh9000)
+- Fix nil pointer dereference in meta-data get on API timeout [#3682](https://github.com/buildkite/agent/pull/3682) (@lox)
+
+### Changed
+- In k8s mode, write BUILDKITE_ENV_FILE to /workspace [#3683](https://github.com/buildkite/agent/pull/3683) (@zhming0)
+
+### Internal
+- Refactor plugin config -> envar generation [#3655](https://github.com/buildkite/agent/pull/3655) (@moskyb)
+- Dependabot updates: [#3656](https://github.com/buildkite/agent/pull/3656), [#3654](https://github.com/buildkite/agent/pull/3654), [#3662](https://github.com/buildkite/agent/pull/3662), [#3673](https://github.com/buildkite/agent/pull/3673), [#3675](https://github.com/buildkite/agent/pull/3675), [#3680](https://github.com/buildkite/agent/pull/3680), [#3681](https://github.com/buildkite/agent/pull/3681) (@dependabot[bot])
+
 ## [v3.115.4](https://github.com/buildkite/agent/tree/v3.115.4) (2026-01-13)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.115.3...v3.115.4)
 
diff --git a/version/VERSION b/version/VERSION
index 390434eef0..f6fff00b00 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.115.4
+3.116.0

From 9447b0d377a0cf6e4a2d2fd8ca5b17ed31967d30 Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Thu, 29 Jan 2026 15:07:33 +0000
Subject: [PATCH 173/242] feat: set Agent config precedence for cloning
 submodules

Signed-off-by: Tom Watt <tom@buildkite.com>
---
 agent/job_runner.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/agent/job_runner.go b/agent/job_runner.go
index d71a7d974c..5f5e841263 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -576,7 +576,11 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_ADDITIONAL_HOOKS_PATHS", strings.Join(r.conf.AgentConfiguration.AdditionalHooksPaths, ","))
 	setEnv("BUILDKITE_PLUGINS_PATH", r.conf.AgentConfiguration.PluginsPath)
 	setEnv("BUILDKITE_SSH_KEYSCAN", fmt.Sprint(r.conf.AgentConfiguration.SSHKeyscan))
-	setEnv("BUILDKITE_GIT_SUBMODULES", fmt.Sprint(r.conf.AgentConfiguration.GitSubmodules))
+	// Disable cloning submodules if specified in Agent config as precedence
+	// else allow pipeline/step env to control it via BUILDKITE_GIT_SUBMODULES
+	if !r.conf.AgentConfiguration.GitSubmodules {
+		setEnv("BUILDKITE_GIT_SUBMODULES", "false")
+	}
 	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
 	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
 	// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config

From 99f95d505c0a5267b24773ea9c8f36fee4b593c6 Mon Sep 17 00:00:00 2001
From: Tom Watt <tom@buildkite.com>
Date: Thu, 29 Jan 2026 15:08:29 +0000
Subject: [PATCH 174/242] test: add bidirectional tests for GitSubmodules

Signed-off-by: Tom Watt <tom@buildkite.com>
---
 internal/job/config_test.go | 74 +++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/internal/job/config_test.go b/internal/job/config_test.go
index ff7d811a17..00c3448f0e 100644
--- a/internal/job/config_test.go
+++ b/internal/job/config_test.go
@@ -92,3 +92,77 @@ func TestReadFromEnvironmentIgnoresMalformedBooleans(t *testing.T) {
 		t.Errorf("config.GitSubmodules = %t, want %t", got, want)
 	}
 }
+
+func TestGitSubmodulesBidirectionalControl(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		initial     bool
+		envValue    string
+		wantValue   bool
+		wantChanged bool
+	}{
+		{
+			name:        "enable submodules",
+			initial:     false,
+			envValue:    "true",
+			wantValue:   true,
+			wantChanged: true,
+		},
+		{
+			name:        "disable submodules",
+			initial:     true,
+			envValue:    "false",
+			wantValue:   false,
+			wantChanged: true,
+		},
+		{
+			name:        "already enabled",
+			initial:     true,
+			envValue:    "true",
+			wantValue:   true,
+			wantChanged: false,
+		},
+		{
+			name:        "already disabled",
+			initial:     false,
+			envValue:    "false",
+			wantValue:   false,
+			wantChanged: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config := &ExecutorConfig{
+				GitSubmodules: tt.initial,
+			}
+
+			environ := env.FromSlice([]string{
+				"BUILDKITE_GIT_SUBMODULES=" + tt.envValue,
+			})
+
+			changes := config.ReadFromEnvironment(environ)
+
+			// Verify field value updated correctly
+			if got, want := config.GitSubmodules, tt.wantValue; got != want {
+				t.Errorf("config.GitSubmodules = %t, want %t", got, want)
+			}
+
+			// Verify changes map reflects whether value actually changed
+			if tt.wantChanged {
+				wantChanges := map[string]string{
+					"BUILDKITE_GIT_SUBMODULES": tt.envValue,
+				}
+				if diff := cmp.Diff(changes, wantChanges); diff != "" {
+					t.Errorf("changes diff (-got +want):\n%s", diff)
+				}
+			} else {
+				if len(changes) != 0 {
+					t.Errorf("changes = %v, want none (value unchanged)", changes)
+				}
+			}
+		})
+	}
+}

From ec44dca1954af7568aa8d7fea1f752d533fabfc4 Mon Sep 17 00:00:00 2001
From: Mitch Smith <mitchpsmith1998@gmail.com>
Date: Sun, 1 Feb 2026 14:39:54 +1000
Subject: [PATCH 175/242] Make bucket-url optional for cache commands

The bucket-url flag is not required when using the LocalHostedAgents
store type. The zstash library handles validation based on the store
type returned by the API - requiring bucket-url for S3 stores while
explicitly rejecting it for LocalHostedAgents.

This allows hosted compute jobs to use cache commands without needing
to specify a bucket URL.

Amp-Thread-ID: https://ampcode.com/threads/T-019c1779-ae35-7259-b2eb-860fb137fdd3
Co-authored-by: Amp <amp@ampcode.com>
---
 clicommand/cache_shared.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/cache_shared.go b/clicommand/cache_shared.go
index 94a5703779..43a9825a70 100644
--- a/clicommand/cache_shared.go
+++ b/clicommand/cache_shared.go
@@ -7,7 +7,7 @@ import "github.com/urfave/cli"
 type CacheConfig struct {
 	Ids             []string `cli:"ids"`
 	Registry        string   `cli:"registry"`
-	BucketURL       string   `cli:"bucket-url" validate:"required"`
+	BucketURL       string   `cli:"bucket-url"`
 	Branch          string   `cli:"branch" validate:"required"`
 	Pipeline        string   `cli:"pipeline" validate:"required"`
 	Organization    string   `cli:"organization" validate:"required"`

From ecf978f6f58d2f44505fc601bcffb5bebf889f2d Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 29 Jan 2026 15:54:59 +1100
Subject: [PATCH 176/242] Add fetch-diff-base flag

---
 clicommand/pipeline_upload.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index 54f04d3182..c0474122d7 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -90,6 +90,7 @@ type PipelineUploadConfig struct {
 	// Used for if_changed processing
 	ApplyIfChanged   bool   `cli:"apply-if-changed"`
 	GitDiffBase      string `cli:"git-diff-base"`
+	FetchDiffBase    bool   `cli:"fetch-diff-base"`
 	ChangedFilesPath string `cli:"changed-files-path"`
 
 	// Used for signing
@@ -146,6 +147,11 @@ var PipelineUploadCommand = cli.Command{
 			Usage:  "Provides the base from which to find the git diff when processing ′if_changed′, e.g. origin/main. If not provided, it uses the first valid value of {origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH, origin/$BUILDKITE_PIPELINE_DEFAULT_BRANCH, origin/main}.",
 			EnvVar: "BUILDKITE_GIT_DIFF_BASE",
 		},
+		cli.BoolFlag{
+			Name:   "fetch-diff-base",
+			Usage:  "When enabled, the base for computing the git diff will be git-fetched prior to computing the diff (default: false)",
+			EnvVar: "BUILDKITE_FETCH_DIFF_BASE",
+		},
 		cli.StringFlag{
 			Name:   "changed-files-path",
 			Usage:  "Path to a file containing the list of changed files (newline-separated) to use for ′if_changed′ evaluation. When provided, the agent skips running git commands to determine changed files.",

From fa4459c7b447c53182c63fb8d9121a868d1e1ddd Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 29 Jan 2026 15:54:59 +1100
Subject: [PATCH 177/242] Split out gatherChangedPaths method

---
 clicommand/pipeline_upload.go | 96 ++++++++++++++++++-----------------
 1 file changed, 50 insertions(+), 46 deletions(-)

diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index c0474122d7..424505491e 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -805,52 +805,12 @@ stepsLoop:
 
 		// If we don't know the changed paths yet, either read from file or call out to Git.
 		if !ica.gathered {
-			var cps []string
-			var err error
-
-			if ica.changedFilesPath != "" {
-				// Read changed files from the provided file path.
-				cps, err = readChangedFilesFromPath(l, ica.changedFilesPath)
-				if err != nil {
-					l.Error("Couldn't read changed files from %q, not skipping any pipeline steps: %v", ica.changedFilesPath, err)
-					ica.enabled = false
-					continue stepsLoop
-				}
-			} else {
-				// Determine changed files using git.
-				cps, err = gatherChangedFiles(l, ica.diffBase)
-				if err != nil {
-					l.Error("Couldn't determine git diff from upstream, not skipping any pipeline steps: %v", err)
-					var exitErr *exec.ExitError
-					if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
-						// stderr came from git, which is typically human readable
-						l.Error("git: %s", exitErr.Stderr)
-					}
-					switch err := err.(type) {
-					case gitRevParseError:
-						l.Error("This could be because %q might not be a commit in the repository.\n"+
-							"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
-							err.arg,
-						)
-
-					case gitMergeBaseError:
-						l.Error("This could be because %q might not be a commit in the repository.\n"+
-							"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
-							err.diffBase,
-						)
-
-					case gitDiffError:
-						l.Error("This could be because the merge-base that Git found, %q, might be invalid.\n"+
-							"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
-							err.mergeBase,
-						)
-					}
-
-					// Because changed files couldn't be determined, we switch into
-					// disabled mode.
-					ica.enabled = false
-					continue stepsLoop
-				}
+			cps, err := ica.gatherChangedPaths(l)
+			if err != nil {
+				// Because changed files couldn't be determined, we switch into
+				// disabled mode.
+				ica.enabled = false
+				continue stepsLoop
 			}
 
 			// The changed files are now known.
@@ -922,6 +882,50 @@ stepsLoop:
 	}
 }
 
+func (ica *ifChangedApplicator) gatherChangedPaths(l logger.Logger) ([]string, error) {
+	if ica.changedFilesPath != "" {
+		// Read changed files from the provided file path.
+		cps, err := readChangedFilesFromPath(l, ica.changedFilesPath)
+		if err != nil {
+			l.Error("Couldn't read changed files from %q, not skipping any pipeline steps: %v", ica.changedFilesPath, err)
+			return nil, err
+		}
+		return cps, nil
+	}
+
+	// Determine changed files using git.
+	cps, err := gatherChangedFiles(l, ica.diffBase)
+	if err != nil {
+		l.Error("Couldn't determine git diff from upstream, not skipping any pipeline steps: %v", err)
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
+			// stderr came from git, which is typically human readable
+			l.Error("git: %s", exitErr.Stderr)
+		}
+		switch err := err.(type) {
+		case gitRevParseError:
+			l.Error("This could be because %q might not be a commit in the repository.\n"+
+				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+				err.arg,
+			)
+
+		case gitMergeBaseError:
+			l.Error("This could be because %q might not be a commit in the repository.\n"+
+				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+				err.diffBase,
+			)
+
+		case gitDiffError:
+			l.Error("This could be because the merge-base that Git found, %q, might be invalid.\n"+
+				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+				err.mergeBase,
+			)
+		}
+		return nil, err
+	}
+	return cps, nil
+}
+
 // ifChangedPatterns converts a string or list within `if_changed` into a slice
 // of parsed globs.
 func ifChangedPatterns(value any) ([]*zzglob.Pattern, error) {

From d203c4335018a37c0283b2bcbdc62b0cf1bb6a40 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 29 Jan 2026 16:16:17 +1100
Subject: [PATCH 178/242] Rename gatherChangedFiles to computeGitDiff

---
 clicommand/pipeline_upload.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index 424505491e..02c95f11cd 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -645,8 +645,8 @@ func readChangedFilesFromPath(l logger.Logger, path string) ([]string, error) {
 	return changedPaths, nil
 }
 
-// gatherChangedFiles determines changed files in this build.
-func gatherChangedFiles(l logger.Logger, diffBase string) (changedPaths []string, err error) {
+// computeGitDiff determines changed files in this build.
+func computeGitDiff(l logger.Logger, diffBase string) (changedPaths []string, err error) {
 	// Corporate needs you to find the differences between diffBase and HEAD.
 	diffBaseCommit, err := exec.Command("git", "rev-parse", diffBase).Output()
 	if err != nil {
@@ -894,7 +894,7 @@ func (ica *ifChangedApplicator) gatherChangedPaths(l logger.Logger) ([]string, e
 	}
 
 	// Determine changed files using git.
-	cps, err := gatherChangedFiles(l, ica.diffBase)
+	cps, err := computeGitDiff(l, ica.diffBase)
 	if err != nil {
 		l.Error("Couldn't determine git diff from upstream, not skipping any pipeline steps: %v", err)
 		var exitErr *exec.ExitError

From 7beccc14bd940ae73690298587c6f1c2c3c34838 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 29 Jan 2026 16:16:17 +1100
Subject: [PATCH 179/242] Add --fetch-diff-base functionality

---
 clicommand/pipeline_upload.go | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go
index 02c95f11cd..b2b044f481 100644
--- a/clicommand/pipeline_upload.go
+++ b/clicommand/pipeline_upload.go
@@ -312,6 +312,7 @@ var PipelineUploadCommand = cli.Command{
 				prependOriginIfNonempty("BUILDKITE_PIPELINE_DEFAULT_BRANCH"),
 				defaultGitDiffBase,
 			),
+			fetch:            cfg.FetchDiffBase,
 			changedFilesPath: cfg.ChangedFilesPath,
 		}
 
@@ -756,6 +757,7 @@ type ifChangedApplicator struct {
 	enabled          bool // apply-if-changed is enabled
 	gathered         bool // the changed files have been computed?
 	diffBase         string
+	fetch            bool   // fetch diffBase before computing diff?
 	changedFilesPath string // path to a file containing newline-separated changed files
 	changedPaths     []string
 }
@@ -893,6 +895,25 @@ func (ica *ifChangedApplicator) gatherChangedPaths(l logger.Logger) ([]string, e
 		return cps, nil
 	}
 
+	if ica.fetch {
+		// First, fetch the remote refspec specified by diffBase.
+		remote, refspec, slash := strings.Cut(ica.diffBase, "/")
+		if !slash {
+			l.Warn("The diff-base %q was not in 'remote/refspec' form - continuing with the remote 'origin'", ica.diffBase)
+			remote = "origin"
+			refspec = ica.diffBase
+		}
+		if err := exec.Command("git", "fetch", "--", remote, refspec).Run(); err != nil {
+			l.Error("Couldn't fetch %q from origin: %v", err)
+			var exitErr *exec.ExitError
+			if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
+				// stderr came from git, which is typically human readable
+				l.Error("git: %s", exitErr.Stderr)
+			}
+			l.Info("if_changed will continue processing, but the diff may fail, or produce more paths than expected.")
+		}
+	}
+
 	// Determine changed files using git.
 	cps, err := computeGitDiff(l, ica.diffBase)
 	if err != nil {
@@ -905,19 +926,19 @@ func (ica *ifChangedApplicator) gatherChangedPaths(l logger.Logger) ([]string, e
 		switch err := err.(type) {
 		case gitRevParseError:
 			l.Error("This could be because %q might not be a commit in the repository.\n"+
-				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var, or add --fetch-diff-base.",
 				err.arg,
 			)
 
 		case gitMergeBaseError:
 			l.Error("This could be because %q might not be a commit in the repository.\n"+
-				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var, or add --fetch-diff-base.",
 				err.diffBase,
 			)
 
 		case gitDiffError:
 			l.Error("This could be because the merge-base that Git found, %q, might be invalid.\n"+
-				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var.",
+				"You may need to change the --git-diff-base flag or BUILDKITE_GIT_DIFF_BASE env var, or add --fetch-diff-base.",
 				err.mergeBase,
 			)
 		}

From 22bb602ae81947ddcaf39c89e41e7e635a734737 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Feb 2026 18:25:32 +0000
Subject: [PATCH 180/242] build(deps): bump github.com/DataDog/datadog-go/v5
 from 5.8.2 to 5.8.3

Bumps [github.com/DataDog/datadog-go/v5](https://github.com/DataDog/datadog-go) from 5.8.2 to 5.8.3.
- [Release notes](https://github.com/DataDog/datadog-go/releases)
- [Changelog](https://github.com/DataDog/datadog-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/DataDog/datadog-go/compare/v5.8.2...v5.8.3)

---
updated-dependencies:
- dependency-name: github.com/DataDog/datadog-go/v5
  dependency-version: 5.8.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9ab3265b6e..26988ac937 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	drjosh.dev/zzglob v0.4.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
-	github.com/DataDog/datadog-go/v5 v5.8.2
+	github.com/DataDog/datadog-go/v5 v5.8.3
 	github.com/Khan/genqlient v0.8.1
 	github.com/aws/aws-sdk-go-v2 v1.41.1
 	github.com/aws/aws-sdk-go-v2/config v1.32.7
diff --git a/go.sum b/go.sum
index 53a23f73a8..5493ae18c6 100644
--- a/go.sum
+++ b/go.sum
@@ -39,8 +39,8 @@ github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0 h1:aIWF85OKxXGo7rVyqJ
 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0/go.mod h1:Lfap5FuM4b/Pw9IrTuAvWBWZEmXOvZhCya3dYv4G8O0=
 github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyLGxp8E6j2f2Mh5IPbYmQ=
 github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8=
-github.com/DataDog/datadog-go/v5 v5.8.2 h1:9IEfH1Mw9AjWwhAMqCAkhbxjuJeMxm2ARX2VdgL+ols=
-github.com/DataDog/datadog-go/v5 v5.8.2/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
+github.com/DataDog/datadog-go/v5 v5.8.3 h1:s58CUJ9s8lezjhTNJO/SxkPBv2qZjS3ktpRSqGF5n0s=
+github.com/DataDog/datadog-go/v5 v5.8.3/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/DataDog/dd-trace-go/v2 v2.3.0 h1:0Y5kx+Wbod0z8moY0vUbKl6OM0oIV4zAynsVmsq+XT8=
 github.com/DataDog/dd-trace-go/v2 v2.3.0/go.mod h1:yFomJ/rqKNLDbS9ohIDibdz8q9GK0MUSSkBdVDCibGA=
 github.com/DataDog/go-libddwaf/v4 v4.3.2 h1:YGvW2Of1C4e1yU+p7iibmhN2zEOgi9XEchbhQjBxb/A=

From 644a2c2b8707569a31e6156573e97aaeed5de77b Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 2 Feb 2026 15:40:00 +1100
Subject: [PATCH 181/242] fix: Continue heartbeats while job is stopping

---
 agent/agent_worker.go | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index d6af2c7ffc..d988792c71 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -236,8 +236,13 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	// errCh receives 1 value from the heartbeat loop, and 1 from the ping loop.
 	errCh := make(chan error, 2)
 
+	// Use this context to control the heartbeat loop.
+	heartbeatCtx, stopHeartbeats := context.WithCancel(ctx)
+	defer stopHeartbeats()
+
+	// Start the heartbeat loop but don't wait for it to return.
 	go func() {
-		errCh <- a.runHeartbeatLoop(ctx)
+		errCh <- a.runHeartbeatLoop(heartbeatCtx)
 	}()
 
 	// If the agent is booted in acquisition mode, acquire that particular job
@@ -254,7 +259,6 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		if err := a.AcquireAndRunJob(ctx, a.agentConfiguration.AcquireJob); err != nil {
 			// If the job acquisition was rejected, we can exit with an error
 			// so that supervisor knows that the job was not acquired due to the job being rejected.
-			a.internalStop() // stop the heartbeat loop
 			if errors.Is(err, core.ErrJobAcquisitionRejected) {
 				return fmt.Errorf("Failed to acquire job %q: %w", a.agentConfiguration.AcquireJob, err)
 			}
@@ -272,14 +276,16 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		}
 	}
 
+	// Start the ping loop and block until it has stopped.
 	errCh <- a.runPingLoop(ctx, idleMon)
 
-	// Blocks until both heartbeat and ping loops have returned.
+	// The ping loop has ended, so stop the heartbeat loop.
+	stopHeartbeats()
+
+	// Block until both loops have returned, then join the errors.
+	// (Note that errors.Join does the right thing with nil.)
 	// Both loops are context aware, so no need to wait on ctx here.
-	for range 2 {
-		startErr = errors.Join(startErr, <-errCh)
-	}
-	return startErr
+	return errors.Join(<-errCh, <-errCh)
 }
 
 func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) error {
@@ -315,13 +321,13 @@ func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) error {
 				a.stats.Unlock()
 			}
 
-		case <-a.stop:
-			a.logger.Debug("Stopping heartbeats due to agent stop")
-			return nil
-
 		case <-ctx.Done():
 			a.logger.Debug("Stopping heartbeats due to context cancel")
-			return ctx.Err()
+			// An alternative to returning nil would be ctx.Err(), but we use
+			// the context for ordinary termination of this loop.
+			// A context cancellation from outside the agent worker would still
+			// be reflected in the value returned by the ping loop return.
+			return nil
 		}
 	}
 }
@@ -334,9 +340,6 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 	defer setStat("🛑 Ping loop stopped!")
 	setStat("🏃 Starting...")
 
-	// Everything stops once the ping loop stops
-	defer a.internalStop()
-
 	// Create the ticker
 	pingInterval := time.Second * time.Duration(a.agent.PingInterval)
 	pingTicker := time.NewTicker(pingInterval)

From d25a2b4f86fe40f5a7340efccfd955c3ca935ab7 Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Wed, 4 Feb 2026 09:29:15 +1100
Subject: [PATCH 182/242] release 3.117.0

---
 CHANGELOG.md    | 12 ++++++++++++
 version/VERSION |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bba00fb4fc..e10cd4ff08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.117.0](https://github.com/buildkite/agent/tree/v3.117.0) (2026-02-04)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.116.0...v3.117.0)
+
+### Added
+- Flag to fetch the diff-base before diffing for `if_changed` [#3689](https://github.com/buildkite/agent/pull/3689) (@DrJosh9000)
+
+### Fixed
+- Continue heartbeats while job is stopping [#3694](https://github.com/buildkite/agent/pull/3694) (@DrJosh9000)
+
+### Internal
+- Make `bucket-url` optional for cache commands [#3690](https://github.com/buildkite/agent/pull/3690) (@mitchbne)
+
 ## [v3.116.0](https://github.com/buildkite/agent/tree/v3.116.0) (2026-01-28)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.115.4...v3.116.0)
 
diff --git a/version/VERSION b/version/VERSION
index f6fff00b00..076aa917d8 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.116.0
+3.117.0

From 3e8ad39224be5560a5a7d45c683c15ca4dd81abc Mon Sep 17 00:00:00 2001
From: Sorcha Abel <sorcha@buildkite.com>
Date: Thu, 5 Feb 2026 10:21:08 +1000
Subject: [PATCH 183/242] sa-update-flag add public preview description to env
 BUILDKITE_PULL_REQUEST_USING_MERGE_REFSPEC

---
 clicommand/bootstrap.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go
index b9c6c76aaf..48217f6d6e 100644
--- a/clicommand/bootstrap.go
+++ b/clicommand/bootstrap.go
@@ -178,7 +178,7 @@ var BootstrapCommand = cli.Command{
 		},
 		cli.BoolFlag{
 			Name:   "pull-request-using-merge-refspec",
-			Usage:  "Whether the agent should attempt to checkout the pull request commit using the merge refspec (default: false)",
+			Usage:  "Whether the agent should attempt to checkout the pull request commit using the merge refspec. This feature is in private preview and requires backend enablement—contact support to enable (default: false)",
 			EnvVar: "BUILDKITE_PULL_REQUEST_USING_MERGE_REFSPEC",
 		},
 		cli.StringFlag{

From f713b272a22e9bb3537ddb0239abfcead1dae25e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Feb 2026 17:59:42 +0000
Subject: [PATCH 184/242] build(deps): bump github.com/go-chi/chi/v5 from 5.2.4
 to 5.2.5

Bumps [github.com/go-chi/chi/v5](https://github.com/go-chi/chi) from 5.2.4 to 5.2.5.
- [Release notes](https://github.com/go-chi/chi/releases)
- [Changelog](https://github.com/go-chi/chi/blob/master/CHANGELOG.md)
- [Commits](https://github.com/go-chi/chi/compare/v5.2.4...v5.2.5)

---
updated-dependencies:
- dependency-name: github.com/go-chi/chi/v5
  dependency-version: 5.2.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9ab3265b6e..14a355757f 100644
--- a/go.mod
+++ b/go.mod
@@ -32,7 +32,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2
 	github.com/gliderlabs/ssh v0.3.8
-	github.com/go-chi/chi/v5 v5.2.4
+	github.com/go-chi/chi/v5 v5.2.5
 	github.com/gofrs/flock v0.13.0
 	github.com/google/go-cmp v0.7.0
 	github.com/google/go-querystring v1.2.0
diff --git a/go.sum b/go.sum
index 53a23f73a8..3b344e325c 100644
--- a/go.sum
+++ b/go.sum
@@ -191,8 +191,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
-github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
-github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
+github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
+github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

From 0467cc4209d8758cefdfef80a1bf1221420296d1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 10 Feb 2026 23:55:25 +0000
Subject: [PATCH 185/242] build(deps): bump aws-sdk-s3 from 1.196.1 to 1.208.0

Bumps [aws-sdk-s3](https://github.com/aws/aws-sdk-ruby) from 1.196.1 to 1.208.0.
- [Release notes](https://github.com/aws/aws-sdk-ruby/releases)
- [Changelog](https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-s3/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-ruby/commits)

---
updated-dependencies:
- dependency-name: aws-sdk-s3
  dependency-version: 1.208.0
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Gemfile.lock | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index de8bb1f9d1..ef52d5ba62 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -5,8 +5,8 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     arr-pm (0.0.12)
     aws-eventstream (1.4.0)
-    aws-partitions (1.1143.0)
-    aws-sdk-core (3.229.0)
+    aws-partitions (1.1213.0)
+    aws-sdk-core (3.242.0)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.992.0)
       aws-sigv4 (~> 1.9)
@@ -14,18 +14,18 @@ GEM
       bigdecimal
       jmespath (~> 1, >= 1.6.1)
       logger
-    aws-sdk-kms (1.110.0)
-      aws-sdk-core (~> 3, >= 3.228.0)
+    aws-sdk-kms (1.121.0)
+      aws-sdk-core (~> 3, >= 3.241.4)
       aws-sigv4 (~> 1.5)
-    aws-sdk-s3 (1.196.1)
-      aws-sdk-core (~> 3, >= 3.228.0)
+    aws-sdk-s3 (1.208.0)
+      aws-sdk-core (~> 3, >= 3.234.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.5)
     aws-sigv4 (1.12.1)
       aws-eventstream (~> 1, >= 1.0.2)
     backports (3.23.0)
     base64 (0.3.0)
-    bigdecimal (3.2.2)
+    bigdecimal (4.0.1)
     cabin (0.9.0)
     clamp (1.0.1)
     deb-s3 (24.6.0)
@@ -46,7 +46,7 @@ GEM
       addressable (~> 2.8)
       rchardet (~> 1.8)
     insist (1.0.0)
-    jmespath (1.6.1)
+    jmespath (1.6.2)
     json (2.6.2)
     logger (1.7.0)
     mustache (0.99.8)

From fd286240b111895debb449689f35dde0559cacac Mon Sep 17 00:00:00 2001
From: Ben McNicholl <ben.mcnicholl@buildkite.com>
Date: Wed, 11 Feb 2026 11:51:38 +1100
Subject: [PATCH 186/242] Add retry logic to secret getting

We use retries already in parts of the tool and occasionally we'll get intermittent issues on `secret get`, too, such as TLS handshake timeouts.

These are in the list of retryable errors used in things like artifacts, but don't feature in secrets... until NOW!
---
 clicommand/secret_get.go        |  2 +-
 internal/job/executor.go        |  5 +-
 internal/secrets/secret.go      | 59 +++++++++++++++++++--
 internal/secrets/secret_test.go | 91 ++++++++++++++++++++++++++++++---
 4 files changed, 144 insertions(+), 13 deletions(-)

diff --git a/clicommand/secret_get.go b/clicommand/secret_get.go
index aab5228f10..b7e5eeb348 100644
--- a/clicommand/secret_get.go
+++ b/clicommand/secret_get.go
@@ -94,7 +94,7 @@ Examples:
 		}
 
 		agentClient := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))
-		secrets, errs := secrets.FetchSecrets(ctx, agentClient, cfg.Job, cfg.Keys, 10)
+		secrets, errs := secrets.FetchSecrets(ctx, l, agentClient, cfg.Job, cfg.Keys, 10)
 		if len(errs) > 0 {
 			sb := &strings.Builder{}
 			sb.WriteString("Failed to fetch some secrets:\n")
diff --git a/internal/job/executor.go b/internal/job/executor.go
index 4010db97bb..2f355bfb92 100644
--- a/internal/job/executor.go
+++ b/internal/job/executor.go
@@ -909,13 +909,14 @@ func (e *Executor) fetchAndSetSecrets(ctx context.Context) error {
 	}
 
 	// Create API client for fetching secrets.
-	apiClient := api.NewClient(logger.NewBuffer(), api.Config{
+	secretLogger := logger.NewBuffer()
+	apiClient := api.NewClient(secretLogger, api.Config{
 		Endpoint: e.shell.Env.GetString("BUILDKITE_AGENT_ENDPOINT", ""),
 		Token:    e.shell.Env.GetString("BUILDKITE_AGENT_ACCESS_TOKEN", ""),
 	})
 
 	// Fetch all secrets
-	fetchedSecrets, errs := secrets.FetchSecrets(ctx, apiClient, e.JobID, keys, 10)
+	fetchedSecrets, errs := secrets.FetchSecrets(ctx, secretLogger, apiClient, e.JobID, keys, 10)
 	if len(errs) > 0 {
 		var errorMsg strings.Builder
 		for _, err := range errs {
diff --git a/internal/secrets/secret.go b/internal/secrets/secret.go
index aae6ee08a8..3f14be944f 100644
--- a/internal/secrets/secret.go
+++ b/internal/secrets/secret.go
@@ -4,8 +4,11 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/buildkite/agent/v3/api"
+	"github.com/buildkite/agent/v3/logger"
+	"github.com/buildkite/roko"
 	"golang.org/x/sync/semaphore"
 )
 
@@ -34,9 +37,31 @@ func (e *SecretError) Unwrap() error {
 	return e.Err
 }
 
-// FetchSecrets retrieves all secret values from the API sequentially.
-// If any secret fails, returns error with details of all failed secrets.
-func FetchSecrets(ctx context.Context, client APIClient, jobID string, keys []string, concurrency int) ([]Secret, []error) {
+// FetchSecretsOpt is a functional option for FetchSecrets.
+type FetchSecretsOpt func(*fetchSecretsConfig)
+
+type fetchSecretsConfig struct {
+	retrySleepFunc func(time.Duration)
+}
+
+// WithRetrySleepFunc overrides the sleep function used between retries.
+// This is primarily useful for unit tests.
+func WithRetrySleepFunc(f func(time.Duration)) FetchSecretsOpt {
+	return func(c *fetchSecretsConfig) {
+		c.retrySleepFunc = f
+	}
+}
+
+// FetchSecrets retrieves all secret values from the API concurrently.
+// Each individual secret fetch is retried up to 3 times with exponential
+// backoff on retryable errors (TLS handshake failures, timeouts, 5xx, 429).
+// If any secret fails after retries, returns error with details of all failed secrets.
+func FetchSecrets(ctx context.Context, l logger.Logger, client APIClient, jobID string, keys []string, concurrency int, opts ...FetchSecretsOpt) ([]Secret, []error) {
+	var cfg fetchSecretsConfig
+	for _, opt := range opts {
+		opt(&cfg)
+	}
+
 	secrets := make([]Secret, 0, len(keys))
 	secretsMu := sync.Mutex{}
 
@@ -55,7 +80,33 @@ func FetchSecrets(ctx context.Context, client APIClient, jobID string, keys []st
 
 		go func() {
 			defer sem.Release(1)
-			apiSecret, _, err := client.GetSecret(ctx, &api.GetSecretRequest{Key: key, JobID: jobID})
+
+			r := roko.NewRetrier(
+				roko.WithMaxAttempts(3),
+				roko.WithStrategy(roko.Exponential(2*time.Second, 0)),
+				roko.WithJitterRange(-1*time.Second, 5*time.Second),
+				roko.WithSleepFunc(cfg.retrySleepFunc),
+			)
+
+			apiSecret, err := roko.DoFunc(ctx, r, func(r *roko.Retrier) (*api.Secret, error) {
+				secret, resp, err := client.GetSecret(ctx, &api.GetSecretRequest{Key: key, JobID: jobID})
+				if err != nil {
+					if resp != nil && api.IsRetryableStatus(resp) {
+						l.Warn("Retrying secret %q fetch after retryable HTTP status %d (%s)", key, resp.StatusCode, r)
+						return nil, err
+					}
+
+					if api.IsRetryableError(err) {
+						l.Warn("Retrying secret %q fetch after retryable error: %v (%s)", key, err, r)
+						return nil, err
+					}
+
+					// Non-retryable error, stop retrying
+					r.Break()
+					return nil, err
+				}
+				return secret, nil
+			})
 			if err != nil {
 				errsMu.Lock()
 				errs = append(errs, &SecretError{
diff --git a/internal/secrets/secret_test.go b/internal/secrets/secret_test.go
index ac09553643..edf95cffb9 100644
--- a/internal/secrets/secret_test.go
+++ b/internal/secrets/secret_test.go
@@ -8,8 +8,10 @@ import (
 	"net/http/httptest"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"syscall"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -18,6 +20,8 @@ import (
 	"github.com/buildkite/agent/v3/logger"
 )
 
+var noSleep = WithRetrySleepFunc(func(time.Duration) {})
+
 func TestFetchSecrets_Success(t *testing.T) {
 	t.Parallel()
 
@@ -44,7 +48,7 @@ func TestFetchSecrets_Success(t *testing.T) {
 		Token:    "llamas",
 	})
 
-	secrets, errs := FetchSecrets(t.Context(), apiClient, "test-job-id", []string{"DATABASE_URL", "API_TOKEN"}, 10)
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", []string{"DATABASE_URL", "API_TOKEN"}, 10)
 	if len(errs) > 0 {
 		t.Fatalf("expected no errors, got: %v", errs)
 	}
@@ -73,7 +77,7 @@ func TestFetchSecrets_EmptyKeys(t *testing.T) {
 	t.Cleanup(server.Close)
 
 	apiClient := api.NewClient(logger.Discard, api.Config{Endpoint: server.URL, Token: "llamas"})
-	secrets, errs := FetchSecrets(t.Context(), apiClient, "test-job-id", []string{}, 10)
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", []string{}, 10)
 
 	if len(errs) > 0 {
 		t.Fatalf("expected no errors, got: %v", errs)
@@ -95,7 +99,7 @@ func TestFetchSecrets_NilKeys(t *testing.T) {
 
 	apiClient := api.NewClient(logger.Discard, api.Config{Endpoint: server.URL, Token: "llamas"})
 
-	secrets, errs := FetchSecrets(t.Context(), apiClient, "test-job-id", nil, 10)
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", nil, 10)
 
 	if len(errs) > 0 {
 		t.Fatalf("expected no errors, got: %v", errs)
@@ -133,7 +137,7 @@ func TestFetchSecrets_SomeSecretsFail(t *testing.T) {
 	})
 
 	keys := []string{"DATABASE_URL", "MISSING"}
-	secrets, errs := FetchSecrets(t.Context(), apiClient, "test-job-id", keys, 10)
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", keys, 10)
 
 	if len(errs) != 1 {
 		t.Fatalf("expected 1 errors, got %d: %v", len(errs), errs)
@@ -177,7 +181,7 @@ func TestFetchSecrets_AllSecretsFail(t *testing.T) {
 	})
 
 	keys := []string{"API_TOKEN", "DATABASE_URL"}
-	secrets, errs := FetchSecrets(t.Context(), apiClient, "test-job-id", keys, 10)
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", keys, 10)
 
 	if len(errs) != 2 {
 		t.Fatalf("expected 2 errors, got %d: %v", len(errs), errs)
@@ -218,7 +222,7 @@ func TestFetchSecrets_APIClientError(t *testing.T) {
 	})
 
 	keys := []string{"TEST_SECRET"}
-	secrets, errs := FetchSecrets(t.Context(), apiClient, "test-job-id", keys, 10)
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", keys, 10, noSleep)
 
 	if len(errs) != 1 {
 		t.Fatalf("expected 1 error, got %d: %v", len(errs), errs)
@@ -266,3 +270,78 @@ func TestFetchSecrets_APIClientError(t *testing.T) {
 		t.Errorf("expected connection refused error, got: %v (type: %T)", netErr.Err, netErr.Err)
 	}
 }
+
+func TestFetchSecrets_RetriesOnServerError(t *testing.T) {
+	t.Parallel()
+
+	var attempts atomic.Int32
+
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		n := attempts.Add(1)
+		if n <= 2 {
+			// First two attempts return 502 Bad Gateway (retryable)
+			rw.WriteHeader(http.StatusBadGateway)
+			_, _ = fmt.Fprintf(rw, `{"message": "bad gateway"}`)
+			return
+		}
+		// Third attempt succeeds
+		rw.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprintf(rw, `{"key": "MY_SECRET", "value": "secret-value"}`)
+	}))
+	t.Cleanup(server.Close)
+
+	apiClient := api.NewClient(logger.Discard, api.Config{
+		Endpoint: server.URL,
+		Token:    "llamas",
+	})
+
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", []string{"MY_SECRET"}, 10, noSleep)
+	if len(errs) > 0 {
+		t.Fatalf("expected no errors after retries, got: %v", errs)
+	}
+
+	if len(secrets) != 1 {
+		t.Fatalf("expected 1 secret, got %d", len(secrets))
+	}
+
+	if secrets[0].Value != "secret-value" {
+		t.Errorf("expected secret value %q, got %q", "secret-value", secrets[0].Value)
+	}
+
+	if got := attempts.Load(); got != 3 {
+		t.Errorf("expected 3 attempts (2 failures + 1 success), got %d", got)
+	}
+}
+
+func TestFetchSecrets_NoRetryOnNonRetryableStatus(t *testing.T) {
+	t.Parallel()
+
+	var attempts atomic.Int32
+
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		attempts.Add(1)
+		// 404 is not retryable
+		rw.WriteHeader(http.StatusNotFound)
+		_, _ = fmt.Fprintf(rw, `{"message": "secret not found"}`)
+	}))
+	t.Cleanup(server.Close)
+
+	apiClient := api.NewClient(logger.Discard, api.Config{
+		Endpoint: server.URL,
+		Token:    "llamas",
+	})
+
+	secrets, errs := FetchSecrets(t.Context(), logger.Discard, apiClient, "test-job-id", []string{"MISSING"}, 10, noSleep)
+	if len(errs) != 1 {
+		t.Fatalf("expected 1 error, got %d: %v", len(errs), errs)
+	}
+
+	if secrets != nil {
+		t.Errorf("expected nil secrets, got: %v", secrets)
+	}
+
+	// Should have only attempted once since 404 is not retryable
+	if got := attempts.Load(); got != 1 {
+		t.Errorf("expected 1 attempt (no retries for 404), got %d", got)
+	}
+}

From d8db4a924c2356074af4908a15632c502e3dd278 Mon Sep 17 00:00:00 2001
From: Matthew Borden <matthew.b@buildkite.com>
Date: Wed, 11 Feb 2026 08:13:06 +1100
Subject: [PATCH 187/242] Add new `buildkite-agent job update timeout N`
 command

Command jobs can only be updated before they are finished.

Only one attributes can currently be updated:

 - `timeout_in_minutes` can be updated so that if a command learns more
   information about how long a job *should* take it can adjust the timeout as
   appropriate, to avoid wasting agent time (and so compute).
---
 api/jobs.go              |  23 +++++++++
 clicommand/commands.go   |   8 ++++
 clicommand/job_update.go | 101 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 132 insertions(+)
 create mode 100644 clicommand/job_update.go

diff --git a/api/jobs.go b/api/jobs.go
index c30f799571..7eb0986ae9 100644
--- a/api/jobs.go
+++ b/api/jobs.go
@@ -134,3 +134,26 @@ func (c *Client) FinishJob(ctx context.Context, job *Job, ignoreAgentInDispatche
 
 	return c.doRequest(req, nil)
 }
+
+// JobUpdateResponse is the response from updating a job
+type JobUpdateResponse struct {
+	ID string `json:"id"`
+}
+
+// UpdateJob updates mutable attributes on a job
+func (c *Client) UpdateJob(ctx context.Context, id string, attrs map[string]string) (*JobUpdateResponse, *Response, error) {
+	u := fmt.Sprintf("jobs/%s", railsPathEscape(id))
+
+	req, err := c.newRequest(ctx, "PUT", u, attrs)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	j := new(JobUpdateResponse)
+	resp, err := c.doRequest(req, j)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return j, resp, err
+}
diff --git a/clicommand/commands.go b/clicommand/commands.go
index a11b6bfb0c..80cf7f3a1d 100644
--- a/clicommand/commands.go
+++ b/clicommand/commands.go
@@ -43,6 +43,14 @@ var BuildkiteAgentCommands = []cli.Command{
 			BuildCancelCommand,
 		},
 	},
+	{
+		Name:     "job",
+		Category: categoryJobCommands,
+		Usage:    "Interact with a Buildkite job",
+		Subcommands: []cli.Command{
+			JobUpdateCommand,
+		},
+	},
 	{
 		Name:     "cache",
 		Category: categoryJobCommands,
diff --git a/clicommand/job_update.go b/clicommand/job_update.go
new file mode 100644
index 0000000000..45dd7fdd0b
--- /dev/null
+++ b/clicommand/job_update.go
@@ -0,0 +1,101 @@
+package clicommand
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"time"
+
+	"github.com/buildkite/agent/v3/api"
+	"github.com/buildkite/agent/v3/internal/redact"
+	"github.com/buildkite/roko"
+	"github.com/urfave/cli"
+)
+
+const jobUpdateHelpDescription = `Usage:
+
+    buildkite-agent job update <attribute> <value> [options...]
+
+Description:
+
+Update an attribute of a job. Only command jobs can be
+updated, and only before they are finished.
+
+Example:
+
+    $ buildkite-agent job update timeout 20
+    $ echo 20 | buildkite-agent job update timeout
+`
+
+type JobUpdateConfig struct {
+	GlobalConfig
+	APIConfig
+
+	Attribute    string   `cli:"arg:0" label:"attribute" validate:"required"`
+	Value        string   `cli:"arg:1" label:"value"`
+	Job          string   `cli:"job" validate:"required"`
+	RedactedVars []string `cli:"redacted-vars" normalize:"list"`
+}
+
+var JobUpdateCommand = cli.Command{
+	Name:        "update",
+	Usage:       "Change the value of an attribute of a job",
+	Description: jobUpdateHelpDescription,
+	Flags: slices.Concat(globalFlags(), apiFlags(), []cli.Flag{
+		cli.StringFlag{
+			Name:   "job",
+			Value:  "",
+			Usage:  "The job to update. Defaults to the current job",
+			EnvVar: "BUILDKITE_JOB_ID",
+		},
+		RedactedVars,
+	}),
+	Action: func(c *cli.Context) error {
+		ctx, cfg, l, _, done := setupLoggerAndConfig[JobUpdateConfig](context.Background(), c)
+		defer done()
+
+		if len(c.Args()) < 2 {
+			l.Info("Reading value from STDIN")
+
+			input, err := io.ReadAll(os.Stdin)
+			if err != nil {
+				return fmt.Errorf("failed to read from STDIN: %w", err)
+			}
+			cfg.Value = string(input)
+		}
+
+		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))
+
+		needles, _, err := redact.NeedlesFromEnv(cfg.RedactedVars)
+		if err != nil {
+			return err
+		}
+		if redactedValue := redact.String(cfg.Value, needles); redactedValue != cfg.Value {
+			l.Warn("New value for job %q attribute %q contained one or more secrets from environment variables that have been redacted. If this is deliberate, pass --redacted-vars='' or a list of patterns that does not match the variable containing the secret", cfg.Job, cfg.Attribute)
+			cfg.Value = redactedValue
+		}
+
+		attrs := map[string]string{cfg.Attribute: cfg.Value}
+
+		if err := roko.NewRetrier(
+			roko.WithMaxAttempts(10),
+			roko.WithStrategy(roko.ExponentialSubsecond(2*time.Second)),
+		).DoWithContext(ctx, func(r *roko.Retrier) error {
+			_, resp, err := client.UpdateJob(ctx, cfg.Job, attrs)
+			if resp != nil && (resp.StatusCode == 400 || resp.StatusCode == 401 || resp.StatusCode == 404 || resp.StatusCode == 422) {
+				r.Break()
+			}
+			if err != nil {
+				l.Warn("%s (%s)", err, r)
+				return err
+			}
+			return nil
+		}); err != nil {
+			return fmt.Errorf("failed to update job: %w", err)
+		}
+
+		return nil
+	},
+}

From f29111e9a913e3f4682e8af56eb6a49cb117940d Mon Sep 17 00:00:00 2001
From: Matthew Borden <matthew.b@buildkite.com>
Date: Wed, 11 Feb 2026 17:10:49 +1100
Subject: [PATCH 188/242] Add end to end test to verify job timeout is updated.

---
 internal/e2e/fixtures/job_update_timeout.yaml |  7 +++++++
 internal/e2e/job_update_timeout_test.go       | 19 +++++++++++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 internal/e2e/fixtures/job_update_timeout.yaml
 create mode 100644 internal/e2e/job_update_timeout_test.go

diff --git a/internal/e2e/fixtures/job_update_timeout.yaml b/internal/e2e/fixtures/job_update_timeout.yaml
new file mode 100644
index 0000000000..6b630c3a78
--- /dev/null
+++ b/internal/e2e/fixtures/job_update_timeout.yaml
@@ -0,0 +1,7 @@
+agents:
+  queue: "{{.queue}}"
+steps:
+  - command: |
+      {{.buildkite_agent_binary}} job update timeout 1
+      sleep 300
+    timeout_in_minutes: 10
diff --git a/internal/e2e/job_update_timeout_test.go b/internal/e2e/job_update_timeout_test.go
new file mode 100644
index 0000000000..51265fe4a8
--- /dev/null
+++ b/internal/e2e/job_update_timeout_test.go
@@ -0,0 +1,19 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"testing"
+)
+
+func TestJobUpdateTimeout(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, "job_update_timeout.yaml")
+
+	tc.startAgent()
+	build := tc.triggerBuild()
+	state := tc.waitForBuild(ctx, build)
+	if got, want := state, "failed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+}

From 960ce13f3e13c6e6403e0fef7b705d7b4a5a6c44 Mon Sep 17 00:00:00 2001
From: Matthew Borden <matthew.b@buildkite.com>
Date: Fri, 13 Feb 2026 16:09:46 +1100
Subject: [PATCH 189/242] Add comment about expecting the build to fail

---
 internal/e2e/job_update_timeout_test.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/internal/e2e/job_update_timeout_test.go b/internal/e2e/job_update_timeout_test.go
index 51265fe4a8..f2743ab927 100644
--- a/internal/e2e/job_update_timeout_test.go
+++ b/internal/e2e/job_update_timeout_test.go
@@ -6,6 +6,9 @@ import (
 	"testing"
 )
 
+// TestJobUpdateTimeout verifies that a job which reduces its timeout from 10
+// minutes to 1 minute via `job update timeout`, then sleeps for 5 minutes,
+// exceeds the new timeout and fails.
 func TestJobUpdateTimeout(t *testing.T) {
 	ctx := t.Context()
 	tc := newTestCase(t, "job_update_timeout.yaml")

From d3d8e2bbc93238e10a97a12d22661bbbbadbe99b Mon Sep 17 00:00:00 2001
From: Matthew Borden <matthew.b@buildkite.com>
Date: Mon, 16 Feb 2026 16:43:53 +1100
Subject: [PATCH 190/242] Bump version and CHANGELOG for v3.118.0

---
 CHANGELOG.md    | 12 ++++++++++++
 version/VERSION |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e10cd4ff08..c86beab364 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.118.0](https://github.com/buildkite/agent/compare/v3.117.0...v3.118.0) (2026-02-16)
+
+### Added
+* Add new `buildkite-agent job update` command to update job timeouts [#3707](https://github.com/buildkite/agent/pull/3707) ([matthewborden](https://github.com/matthewborden))
+* Enable setting BUILDKITE_GIT_SUBMODULE with Environment Variables [#3677](https://github.com/buildkite/agent/pull/3677) ([tomowatt](https://github.com/tomowatt))
+
+### Fixed
+* chore: Modified mktemp command for tarball extraction on macOS VMs [#3698](https://github.com/buildkite/agent/pull/3698) ([chrisnavar](https://github.com/chrisnavar))
+
+### Internal
+* Add public preview description to env BUILDKITE_PULL_REQUEST_USING_MERGE_REFSPEC [#3699](https://github.com/buildkite/agent/pull/3699) ([SorchaAbel](https://github.com/SorchaAbel))
+
 ## [v3.117.0](https://github.com/buildkite/agent/tree/v3.117.0) (2026-02-04)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.116.0...v3.117.0)
 
diff --git a/version/VERSION b/version/VERSION
index 076aa917d8..cb00d0cca5 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.117.0
+3.118.0

From 101b07a841164c174573096ca9461ea4e3fbb591 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Feb 2026 16:51:41 +0000
Subject: [PATCH 191/242] build(deps): bump the golang-x group across 1
 directory with 5 updates

Bumps the golang-x group with 3 updates in the / directory: [golang.org/x/crypto](https://github.com/golang/crypto), [golang.org/x/net](https://github.com/golang/net) and [golang.org/x/oauth2](https://github.com/golang/oauth2).


Updates `golang.org/x/crypto` from 0.47.0 to 0.48.0
- [Commits](https://github.com/golang/crypto/compare/v0.47.0...v0.48.0)

Updates `golang.org/x/net` from 0.49.0 to 0.50.0
- [Commits](https://github.com/golang/net/compare/v0.49.0...v0.50.0)

Updates `golang.org/x/oauth2` from 0.34.0 to 0.35.0
- [Commits](https://github.com/golang/oauth2/compare/v0.34.0...v0.35.0)

Updates `golang.org/x/sys` from 0.40.0 to 0.41.0
- [Commits](https://github.com/golang/sys/compare/v0.40.0...v0.41.0)

Updates `golang.org/x/term` from 0.39.0 to 0.40.0
- [Commits](https://github.com/golang/term/compare/v0.39.0...v0.40.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.48.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/net
  dependency-version: 0.50.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/oauth2
  dependency-version: 0.35.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/sys
  dependency-version: 0.41.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
- dependency-name: golang.org/x/term
  dependency-version: 0.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: golang-x
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 16 ++++++++--------
 go.sum | 32 ++++++++++++++++----------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/go.mod b/go.mod
index 9ab3265b6e..99029237f1 100644
--- a/go.mod
+++ b/go.mod
@@ -56,12 +56,12 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0
 	go.opentelemetry.io/otel/sdk v1.39.0
 	go.opentelemetry.io/otel/trace v1.39.0
-	golang.org/x/crypto v0.47.0
-	golang.org/x/net v0.49.0
-	golang.org/x/oauth2 v0.34.0
+	golang.org/x/crypto v0.48.0
+	golang.org/x/net v0.50.0
+	golang.org/x/oauth2 v0.35.0
 	golang.org/x/sync v0.19.0
-	golang.org/x/sys v0.40.0
-	golang.org/x/term v0.39.0
+	golang.org/x/sys v0.41.0
+	golang.org/x/term v0.40.0
 	google.golang.org/api v0.260.0
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
@@ -198,10 +198,10 @@ require (
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
-	golang.org/x/mod v0.31.0 // indirect
-	golang.org/x/text v0.33.0 // indirect
+	golang.org/x/mod v0.32.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.40.0 // indirect
+	golang.org/x/tools v0.41.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
diff --git a/go.sum b/go.sum
index 53a23f73a8..c4354e8ae9 100644
--- a/go.sum
+++ b/go.sum
@@ -491,24 +491,24 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
-golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
 golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
-golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
+golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
+golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
-golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
-golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
+golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -528,15 +528,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
-golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
+golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
+golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -544,8 +544,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
-golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
+golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
+golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From 8eb48e926501089045c63643e04cc1e758c80523 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Feb 2026 17:00:42 +0000
Subject: [PATCH 192/242] build(deps): bump the container-images group across 3
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `174a2a9` to `6ac8150`

Updates `buildkite/agent-base` from `6cb7bd9` to `036845c`

Updates `buildkite/agent-base` from `14db4ab` to `7bfce10`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index 9cac337183..aea2de9fb8 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:2dca5c55276726181d090ab89ac9a130e2818df9159de3dc4557db1931dff84b
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:e4ae53ac87c5b870537c3509e8dfe1ea95cba33ead9b2958fd2a8c7ae301ec1f
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index 5c2365a683..fa134042a3 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:3851483427a45549035f219464e1d1d45f628bb1898318de218913e8adc3363d
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:d1a0b17c3fef91918940272dcb1f67222d8d1afa7a1dfcd7a3834bd96ddd23ff
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index b3a0608d61..d89dd512c2 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:fdf8b6dc594555da27bbea0fe37e1c9249caf0b5f4d7ecc41d844f81bdd00ec2
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:861ff33773b82f0d3677d735e95876c867a97593705ff903eecbad30d07f94e4
 ARG TARGETOS
 ARG TARGETARCH
 

From b107466d69a86a9a23128b1a8dadefc46d963b52 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Feb 2026 17:00:57 +0000
Subject: [PATCH 193/242] build(deps): bump golangci/golangci-lint

Bumps the container-images group with 1 update in the /.buildkite directory: golangci/golangci-lint.


Updates `golangci/golangci-lint` from v2.8-alpine to v2.9-alpine

---
updated-dependencies:
- dependency-name: golangci/golangci-lint
  dependency-version: v2.9-alpine
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .buildkite/Dockerfile-lint | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/Dockerfile-lint b/.buildkite/Dockerfile-lint
index cf846803b6..cfbffa1421 100644
--- a/.buildkite/Dockerfile-lint
+++ b/.buildkite/Dockerfile-lint
@@ -1 +1 @@
-FROM golangci/golangci-lint:v2.8-alpine@sha256:1194f3bfcbaeeb92d8d159fdfbe2a79d18ec0a222d9d984b1438906bca416b51
\ No newline at end of file
+FROM golangci/golangci-lint:v2.9-alpine@sha256:efea7fae4d772680c2c2dc3a067bde22c8c0344dde7e800d110589aaee6ce977
\ No newline at end of file

From a44fffe3f88f02af1f4088a2e19b6b6673129817 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Feb 2026 21:44:29 +0000
Subject: [PATCH 194/242] build(deps): bump the otel group across 1 directory
 with 9 updates

Bumps the otel group with 6 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [go.opentelemetry.io/contrib/propagators/aws](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.39.0` | `1.40.0` |
| [go.opentelemetry.io/contrib/propagators/b3](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.39.0` | `1.40.0` |
| [go.opentelemetry.io/contrib/propagators/jaeger](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.39.0` | `1.40.0` |
| [go.opentelemetry.io/contrib/propagators/ot](https://github.com/open-telemetry/opentelemetry-go-contrib) | `1.39.0` | `1.40.0` |
| [go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc](https://github.com/open-telemetry/opentelemetry-go) | `1.39.0` | `1.40.0` |
| [go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp](https://github.com/open-telemetry/opentelemetry-go) | `1.39.0` | `1.40.0` |



Updates `go.opentelemetry.io/contrib/propagators/aws` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/contrib/propagators/b3` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/contrib/propagators/jaeger` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/contrib/propagators/ot` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/otel` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/otel/sdk` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.39.0...v1.40.0)

Updates `go.opentelemetry.io/otel/trace` from 1.39.0 to 1.40.0
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.39.0...v1.40.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/contrib/propagators/aws
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/contrib/propagators/b3
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/contrib/propagators/jaeger
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/contrib/propagators/ot
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/otel
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/otel/sdk
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
- dependency-name: go.opentelemetry.io/otel/trace
  dependency-version: 1.40.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: otel
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 28 +++++++++++++--------------
 go.sum | 60 +++++++++++++++++++++++++++++-----------------------------
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/go.mod b/go.mod
index 99029237f1..42d8f12c26 100644
--- a/go.mod
+++ b/go.mod
@@ -47,15 +47,15 @@ require (
 	github.com/qri-io/jsonschema v0.2.1
 	github.com/stretchr/testify v1.11.1
 	github.com/urfave/cli v1.22.17
-	go.opentelemetry.io/contrib/propagators/aws v1.39.0
-	go.opentelemetry.io/contrib/propagators/b3 v1.39.0
-	go.opentelemetry.io/contrib/propagators/jaeger v1.39.0
-	go.opentelemetry.io/contrib/propagators/ot v1.39.0
-	go.opentelemetry.io/otel v1.39.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0
-	go.opentelemetry.io/otel/sdk v1.39.0
-	go.opentelemetry.io/otel/trace v1.39.0
+	go.opentelemetry.io/contrib/propagators/aws v1.40.0
+	go.opentelemetry.io/contrib/propagators/b3 v1.40.0
+	go.opentelemetry.io/contrib/propagators/jaeger v1.40.0
+	go.opentelemetry.io/contrib/propagators/ot v1.40.0
+	go.opentelemetry.io/otel v1.40.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0
+	go.opentelemetry.io/otel/sdk v1.40.0
+	go.opentelemetry.io/otel/trace v1.40.0
 	golang.org/x/crypto v0.48.0
 	golang.org/x/net v0.50.0
 	golang.org/x/oauth2 v0.35.0
@@ -137,7 +137,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.9 // indirect
 	github.com/googleapis/gax-go/v2 v2.16.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
@@ -189,9 +189,9 @@ require (
 	go.opentelemetry.io/collector/semconv v0.125.0 // indirect
 	go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
 	go.opentelemetry.io/otel/log v0.11.0 // indirect
-	go.opentelemetry.io/otel/metric v1.39.0 // indirect
+	go.opentelemetry.io/otel/metric v1.40.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
@@ -203,8 +203,8 @@ require (
 	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.41.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
 	google.golang.org/grpc v1.78.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/go.sum b/go.sum
index c4354e8ae9..539d4da857 100644
--- a/go.sum
+++ b/go.sum
@@ -243,8 +243,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU=
 github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
@@ -449,32 +449,32 @@ go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpo
 go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY=
-go.opentelemetry.io/contrib/propagators/aws v1.39.0 h1:IvNR8pAVGpkK1CHMjU/YE6B6TlnAPGFvogkMWRWU6wo=
-go.opentelemetry.io/contrib/propagators/aws v1.39.0/go.mod h1:TUsFCERuGM4IGhJG9w+9l0nzmHUKHuaDYYNF6mtNgjY=
-go.opentelemetry.io/contrib/propagators/b3 v1.39.0 h1:PI7pt9pkSnimWcp5sQhUA9OzLbc3Ba4sL+VEUTNsxrk=
-go.opentelemetry.io/contrib/propagators/b3 v1.39.0/go.mod h1:5gV/EzPnfYIwjzj+6y8tbGW2PKWhcsz5e/7twptRVQY=
-go.opentelemetry.io/contrib/propagators/jaeger v1.39.0 h1:Gz3yKzfMSEFzF0Vy5eIpu9ndpo4DhXMCxsLMF0OOApo=
-go.opentelemetry.io/contrib/propagators/jaeger v1.39.0/go.mod h1:2D/cxxCqTlrday0rZrPujjg5aoAdqk1NaNyoXn8FJn8=
-go.opentelemetry.io/contrib/propagators/ot v1.39.0 h1:vKTve1W/WKPVp1fzJamhCDDECt+5upJJ65bPyWoddGg=
-go.opentelemetry.io/contrib/propagators/ot v1.39.0/go.mod h1:FH5VB2N19duNzh1Q8ks6CsZFyu3LFhNLiA9lPxyEkvU=
-go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
-go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 h1:in9O8ESIOlwJAEGTkkf34DesGRAc/Pn8qJ7k3r/42LM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0/go.mod h1:Rp0EXBm5tfnv0WL+ARyO/PHBEaEAT8UUHQ6AGJcSq6c=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU=
+go.opentelemetry.io/contrib/propagators/aws v1.40.0 h1:4VIrh75jW4RTimUNx1DSk+6H9/nDr1FvmKoOVDh3K04=
+go.opentelemetry.io/contrib/propagators/aws v1.40.0/go.mod h1:B0dCov9KNQGlut3T8wZZjDnLXEXdBroM7bFsHh/gRos=
+go.opentelemetry.io/contrib/propagators/b3 v1.40.0 h1:xariChe8OOVF3rNlfzGFgQc61npQmXhzZj/i82mxMfg=
+go.opentelemetry.io/contrib/propagators/b3 v1.40.0/go.mod h1:72WvbdxbOfXaELEQfonFfOL6osvcVjI7uJEE8C2nkrs=
+go.opentelemetry.io/contrib/propagators/jaeger v1.40.0 h1:aXl9uobjJs5vquMLt9ZkI/3zIuz8XQ3TqOKSWx0/xdU=
+go.opentelemetry.io/contrib/propagators/jaeger v1.40.0/go.mod h1:ioMePqe6k6c/ovXSkmkMr1mbN5qRBGJxNTVop7/2XO0=
+go.opentelemetry.io/contrib/propagators/ot v1.40.0 h1:Lon8J5SPmWaL1Ko2TIlCNHJ42/J1b5XbJlgJaE/9m7I=
+go.opentelemetry.io/contrib/propagators/ot v1.40.0/go.mod h1:dKWtJTlp1Yj+8Cneye5idO46eRPIbi23qVuJYKjNnvY=
+go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
+go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40=
 go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y=
 go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc=
-go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
-go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
-go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
-go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
-go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
-go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
-go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
-go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
+go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
+go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
+go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
+go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
+go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
+go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
+go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
+go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
 go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -560,10 +560,10 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934=
 google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
+google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
 google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
 google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

From 41bb1f7150a5118ef8476890d3fc0c06d72d1760 Mon Sep 17 00:00:00 2001
From: Rajat Vig <rvig@etsy.com>
Date: Tue, 24 Feb 2026 23:43:37 +0000
Subject: [PATCH 195/242] Fix false URL mismatch detection with insteadOf

When url.*.insteadOf is configured in gitconfig (e.g., to rewrite SSH
to HTTPS for GitHub Apps auth), `git remote get-url` returns the
transformed URL, not the stored URL. This causes false URL mismatch
detection and unnecessary fsck/gc runs on every checkout.

Use `git config --get remote.origin.url` instead, which returns the
raw stored URL without insteadOf transformation.

Signed-off-by: Rajat Vig <rvig@etsy.com>
---
 internal/job/checkout.go                              | 2 +-
 internal/job/integration/checkout_integration_test.go | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index 50999c5f97..460bc10fd5 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -538,7 +538,7 @@ func (e *Executor) updateRemoteURL(ctx context.Context, gitDir, repository strin
 
 	// First check what the existing remote is, for both logging and debugging
 	// purposes.
-	args := []string{"remote", "get-url", "origin"}
+	args := []string{"config", "--get", "remote.origin.url"}
 	if gitDir != "" {
 		args = append([]string{"--git-dir", gitDir}, args...)
 	}
diff --git a/internal/job/integration/checkout_integration_test.go b/internal/job/integration/checkout_integration_test.go
index ebf42051cc..baf4492565 100644
--- a/internal/job/integration/checkout_integration_test.go
+++ b/internal/job/integration/checkout_integration_test.go
@@ -297,7 +297,7 @@ func TestCheckingOutLocalGitProjectWithShortCommitHash(t *testing.T) {
 	// Git should attempt to fetch the shortHash, but fail. Then fallback to fetching
 	// all the heads and tags and checking out the short commit hash.
 	git.ExpectAll([][]any{
-		{"remote", "get-url", "origin"},
+		{"config", "--get", "remote.origin.url"},
 		{"clean", "-ffxdq"},
 		{"fetch", "--", "origin", shortCommitHash},
 		{"config", "remote.origin.fetch"},
@@ -615,7 +615,7 @@ func TestCheckoutErrorIsRetried(t *testing.T) {
 
 	// But assert which ones are called
 	git.ExpectAll([][]any{
-		{"remote", "get-url", "origin"},
+		{"config", "--get", "remote.origin.url"},
 		{"clean", "-fdq"},
 		{"fetch", "-v", "--", "origin", "main"},
 		{"checkout", "-f", "FETCH_HEAD"},
@@ -678,7 +678,7 @@ func TestFetchErrorIsRetried(t *testing.T) {
 
 	// But assert which ones are called
 	git.ExpectAll([][]any{
-		{"remote", "get-url", "origin"},
+		{"config", "--get", "remote.origin.url"},
 		{"clean", "-ffxdq"},
 		{"fetch", "-v", "--prune", "--depth=1", "--", "origin", "main"},
 		{"clone", "-v", "--depth=1", "--", tester.Repo.Path, "."},

From acb59be66f975a31fa27543b1c2a6d562c20de4c Mon Sep 17 00:00:00 2001
From: Rajat Vig <rvig@etsy.com>
Date: Wed, 25 Feb 2026 00:45:15 +0000
Subject: [PATCH 196/242] fix: fallback to earlier method when origin has
 multiple URIs

Signed-off-by: Rajat Vig <rvig@etsy.com>
---
 internal/job/checkout.go                      | 24 ++++++-
 .../integration/checkout_integration_test.go  | 63 ++++++++++++++++++-
 2 files changed, 82 insertions(+), 5 deletions(-)

diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index 460bc10fd5..338f71aa23 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -538,15 +538,35 @@ func (e *Executor) updateRemoteURL(ctx context.Context, gitDir, repository strin
 
 	// First check what the existing remote is, for both logging and debugging
 	// purposes.
-	args := []string{"config", "--get", "remote.origin.url"}
+
+	// Check if there are multiple URLs configured (e.g., via git remote set-url --add).
+	args := []string{"config", "--get-all", "remote.origin.url"}
 	if gitDir != "" {
 		args = append([]string{"--git-dir", gitDir}, args...)
 	}
-	gotURL, err := e.shell.Command("git", args...).RunAndCaptureStdout(ctx)
+	allURLs, err := e.shell.Command("git", args...).RunAndCaptureStdout(ctx)
 	if err != nil {
 		return false, err
 	}
 
+	var gotURL string
+	urls := strings.Split(strings.TrimSpace(allURLs), "\n")
+	if len(urls) > 1 {
+		// Multiple URLs configured - fall back to git remote get-url which
+		// handles this correctly (returns primary fetch URL).
+		args = []string{"remote", "get-url", "origin"}
+		if gitDir != "" {
+			args = append([]string{"--git-dir", gitDir}, args...)
+		}
+		gotURL, err = e.shell.Command("git", args...).RunAndCaptureStdout(ctx)
+		if err != nil {
+			return false, err
+		}
+	} else {
+		// Single URL - use config output directly to avoid insteadOf transformation.
+		gotURL = urls[0]
+	}
+
 	if gotURL == repository {
 		// No need to update anything
 		return false, nil
diff --git a/internal/job/integration/checkout_integration_test.go b/internal/job/integration/checkout_integration_test.go
index baf4492565..fe193617d0 100644
--- a/internal/job/integration/checkout_integration_test.go
+++ b/internal/job/integration/checkout_integration_test.go
@@ -297,7 +297,7 @@ func TestCheckingOutLocalGitProjectWithShortCommitHash(t *testing.T) {
 	// Git should attempt to fetch the shortHash, but fail. Then fallback to fetching
 	// all the heads and tags and checking out the short commit hash.
 	git.ExpectAll([][]any{
-		{"config", "--get", "remote.origin.url"},
+		{"config", "--get-all", "remote.origin.url"},
 		{"clean", "-ffxdq"},
 		{"fetch", "--", "origin", shortCommitHash},
 		{"config", "remote.origin.fetch"},
@@ -615,7 +615,7 @@ func TestCheckoutErrorIsRetried(t *testing.T) {
 
 	// But assert which ones are called
 	git.ExpectAll([][]any{
-		{"config", "--get", "remote.origin.url"},
+		{"config", "--get-all", "remote.origin.url"},
 		{"clean", "-fdq"},
 		{"fetch", "-v", "--", "origin", "main"},
 		{"checkout", "-f", "FETCH_HEAD"},
@@ -678,7 +678,7 @@ func TestFetchErrorIsRetried(t *testing.T) {
 
 	// But assert which ones are called
 	git.ExpectAll([][]any{
-		{"config", "--get", "remote.origin.url"},
+		{"config", "--get-all", "remote.origin.url"},
 		{"clean", "-ffxdq"},
 		{"fetch", "-v", "--prune", "--depth=1", "--", "origin", "main"},
 		{"clone", "-v", "--depth=1", "--", tester.Repo.Path, "."},
@@ -1096,6 +1096,63 @@ func TestGitCheckoutWithoutCommitResolvedAndNoMetaData(t *testing.T) {
 	tester.RunAndCheck(t, env...)
 }
 
+func TestMultipleRemoteURLsFallsBackToGetURL(t *testing.T) {
+	t.Parallel()
+
+	tester, err := NewExecutorTester(mainCtx)
+	if err != nil {
+		t.Fatalf("NewExecutorTester() error = %v", err)
+	}
+	defer tester.Close()
+
+	env := []string{
+		"BUILDKITE_GIT_CLONE_FLAGS=-v",
+		"BUILDKITE_GIT_CLEAN_FLAGS=-fdq",
+		"BUILDKITE_GIT_FETCH_FLAGS=-v",
+	}
+
+	// Simulate state from a previous checkout
+	if err := os.MkdirAll(tester.CheckoutDir(), 0o755); err != nil {
+		t.Fatalf("error creating dir to clone from: %s", err)
+	}
+	cmd := exec.Command("git", "clone", "-v", "--", tester.Repo.Path, ".")
+	cmd.Dir = tester.CheckoutDir()
+	if _, err = cmd.Output(); err != nil {
+		t.Fatalf("error cloning test repo: %s", err)
+	}
+
+	// Add a second remote URL to simulate multi-URL configuration
+	cmd = exec.Command("git", "remote", "set-url", "--add", "origin", "https://example.com/extra.git")
+	cmd.Dir = tester.CheckoutDir()
+	if _, err = cmd.Output(); err != nil {
+		t.Fatalf("error adding second remote URL: %s", err)
+	}
+
+	// Actually execute git commands, but with expectations
+	git := tester.
+		MustMock(t, "git").
+		PassthroughToLocalCommand()
+
+	// Assert the expected git commands - should call config --get-all first,
+	// then fall back to remote get-url when multiple URLs are detected
+	git.ExpectAll([][]any{
+		{"config", "--get-all", "remote.origin.url"},
+		{"remote", "get-url", "origin"},
+		{"clean", "-fdq"},
+		{"fetch", "-v", "--", "origin", "main"},
+		{"checkout", "-f", "FETCH_HEAD"},
+		{"clean", "-fdq"},
+		{"--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg},
+	})
+
+	// Mock out the meta-data calls to the agent after checkout
+	agent := tester.MockAgent(t)
+	agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1)
+	agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern)
+
+	tester.RunAndCheck(t, env...)
+}
+
 type subDirMatcher struct {
 	dir string
 }

From 86962eb92ecdc31f6db46af3141f840f62875d77 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Wed, 25 Feb 2026 14:56:47 +1100
Subject: [PATCH 197/242] Add test to ensure new experiments are documented in
 EXPERIMENTS.md

---
 internal/experiments/experiments_test.go | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 internal/experiments/experiments_test.go

diff --git a/internal/experiments/experiments_test.go b/internal/experiments/experiments_test.go
new file mode 100644
index 0000000000..a41f51d7b6
--- /dev/null
+++ b/internal/experiments/experiments_test.go
@@ -0,0 +1,23 @@
+package experiments
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestAvailableExperimentsDocumented(t *testing.T) {
+	data, err := os.ReadFile("../../EXPERIMENTS.md")
+	if err != nil {
+		t.Fatalf("reading EXPERIMENTS.md: %v", err)
+	}
+	contents := string(data)
+
+	for name := range Available {
+		heading := fmt.Sprintf("### `%s`", name)
+		if !strings.Contains(contents, heading) {
+			t.Errorf("available experiment %q is missing a %q section in EXPERIMENTS.md", name, heading)
+		}
+	}
+}

From d7bfcfcf4cf9bea53f6ca088d3c6c2aafeb64723 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Wed, 25 Feb 2026 14:59:11 +1100
Subject: [PATCH 198/242] Remove promoted experiments from EXPERIMENTS.md, add
 new ones

---
 EXPERIMENTS.md | 39 +++++++++++++--------------------------
 1 file changed, 13 insertions(+), 26 deletions(-)

diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md
index d66b90e80e..36e566b8af 100644
--- a/EXPERIMENTS.md
+++ b/EXPERIMENTS.md
@@ -37,18 +37,6 @@ After repository checkout, resolve `BUILDKITE_COMMIT` to a commit hash. This mak
 
 **Status**: broadly useful, we'd like this to be the standard behaviour in 4.0. 👍👍
 
-### `polyglot-hooks`
-
-Allows the agent to run hooks written in languages other than bash. This enables the agent to run hooks written in any language, as long as the language has a runtime available on the agent. Polyglot hooks can be in interpreted languages, so long as they have a valid shebang, and the interpreter specified in the shebang is installed on the agent.
-
-This experiment also allows the agent to run compiled binaries (such as those produced by Go, Rust, Zig, C et al.) as hooks, so long as they are executable.
-
-Hooks are run in a subshell, so they can't modify the environment of the agent process. However, they can use the [job-api](#job-api) to modify the environment of the job.
-
-Binary hooks are available on all platforms, but interpreted hooks are unfortunately unavailable on Windows, as Windows does not support shebangs.
-
-**Status:** Experimental while we try to cover the various corner cases. We'll probably promote this to non-experiment soon™️.
-
 ### `agent-api`
 
 This exposes a local API for interacting with the agent process.
@@ -58,19 +46,6 @@ The API is exposed via a Unix Domain Socket. The path to the socket is not avail
 
 **Status:** Experimental while we iron out the API and test it out in the wild. We'll probably promote this to non-experiment soon™.
 
-### `use-zzglob`
-
-Uses a different library for resolving glob expressions used for `artifact upload`.
-The new glob library should resolve a few issues experienced with the old library:
-
-- Because `**` is used to mean "zero or more path segments", `/**/` should match `/`.
-- Directories that cannot match the glob pattern shouldn't be walked while resolving the pattern. Failure to do this makes `artifact upload` difficult to use when run in a directory containing a mix of subdirectories with different permissions.
-- Failures to walk potential file paths should be reported individually.
-
-The new library should handle all syntax supported by the old library, but because of the chance of incompatibilities and bugs, we're providing it via experiment only for now.
-
-**Status:** Since using the old library causes problems, we hope to promote this to be the default soon™️.
-
 ### `pty-raw`
 
 Set PTY to raw mode, to avoid mapping LF (\n) to CR,LF (\r\n) in job command output.
@@ -96,7 +71,7 @@ a cancelled job should appear as a failure, regardless of the OS the agent is ru
 
 ### `interpolation-prefers-runtime-env`
 
-When interpolating the pipeline level environment block, a pipeline level environment variable could take precedence over environment variables depending on the ordering. This may contravene Buildkite's [documentation](https://buildkite.com/docs/pipelines/environment-variables#environment-variable-precedence) that suggests the Job runtime environment takes precedence over that defined by combining environment variables defined in a pipeline. 
+When interpolating the pipeline level environment block, a pipeline level environment variable could take precedence over environment variables depending on the ordering. This may contravene Buildkite's [documentation](https://buildkite.com/docs/pipelines/environment-variables#environment-variable-precedence) that suggests the Job runtime environment takes precedence over that defined by combining environment variables defined in a pipeline.
 
 We previously made this the default behaviour of the agent (as of v3.63.0) but have since reverted it.
 
@@ -122,3 +97,15 @@ has a different effect depending on this experiment:
 - With `allow-artifact-path-traversal` enabled, `foo.txt` is downloaded to `../../foo.txt`.
 
 **Status:** This experiment is an escape hatch for a security fix. While the new behaviour is more secure, it may break downloading of legitimately-uploaded artifacts.
+
+### `descending-spawn-priority`
+
+When using `--spawn` with `--spawn-with-priority`, the agent assigns ascending priorities to each spawned agent (1, 2, 3, ...). This experiment changes the priorities to be descending (-1, -2, -3, ...) instead. This helps jobs be assigned across all hosts in cases where the value of `--spawn` varies between hosts.
+
+**Status:** Experimental as an escape hatch to default behaviour. Will soon be promoted to a regular flag.
+
+### `propagate-agent-config-vars`
+
+Prepends agent configuration variables (such as `BUILDKITE_GIT_*`, `BUILDKITE_SHELL`, `BUILDKITE_CANCEL_GRACE_PERIOD`, etc.) to the environment file used by the job runner. This is useful in environments like Docker where the agent configuration is not otherwise available to the job process.
+
+**Status:** Experimental while we test the impact on job environments

From 515f69a8b02f09b2c258bd0ea93307956d03a1b5 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 26 Feb 2026 10:09:47 +1100
Subject: [PATCH 199/242] Bump version + changelog for v3.118.1

---
 CHANGELOG.md    | 9 +++++++++
 version/VERSION | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c86beab364..e9d86a43d7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.118.1](https://github.com/buildkite/agent/tree/v3.118.1) (2026-02-25)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.118.0...v3.118.1)
+
+### Changed
+- Add retry logic to secret getting [#3706](https://github.com/buildkite/agent/pull/3706) (@mcncl)
+
+#### Internal
+- Test experiment documentation [#3719](https://github.com/buildkite/agent/pull/3719) (@moskyb)
+
 ## [v3.118.0](https://github.com/buildkite/agent/compare/v3.117.0...v3.118.0) (2026-02-16)
 
 ### Added
diff --git a/version/VERSION b/version/VERSION
index cb00d0cca5..5826da02b3 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.118.0
+3.118.1

From ee0b8929ba63cb2b07430dd4295aef644d66e28b Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Mon, 2 Mar 2026 11:57:27 +1100
Subject: [PATCH 200/242] PS-1663: log s3 credential source for visibility

---
 internal/artifact/s3.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/internal/artifact/s3.go b/internal/artifact/s3.go
index 978d5045a9..e7e4cc0fe7 100644
--- a/internal/artifact/s3.go
+++ b/internal/artifact/s3.go
@@ -126,6 +126,16 @@ func NewS3Client(ctx context.Context, l logger.Logger, bucket string) (*s3.Clien
 		o.UsePathStyle = usePathStyle
 	})
 
+	creds, credErr := cfg.Credentials.Retrieve(ctx)
+	if credErr == nil {
+		profile := cmp.Or(os.Getenv("BUILDKITE_S3_PROFILE"), os.Getenv("AWS_PROFILE"))
+		if creds.Source == "buildkiteEnvProvider" {
+			l.Info("S3 credentials found in Buildkite environment (BUILDKITE_S3_ACCESS_KEY_ID, BUILDKITE_S3_SECRET_ACCESS_KEY)")
+		} else {
+			l.Info("S3 credentials loaded from the default AWS credential chain or BUILDKITE_S3_PROFILE, profile: %q", profile)
+		}
+	}
+
 	l.Debug("Testing AWS S3 credentials for bucket %q in region %q...", bucket, cfg.Region)
 
 	// Test the authentication by trying to list the first 0 objects in the bucket.

From 542b4396f3a783f854e9e44cef4a68cddace31b7 Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Mon, 2 Mar 2026 15:46:14 +1100
Subject: [PATCH 201/242] update codeowner

---
 CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index 4aa63af558..437ac1c9df 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -1 +1 @@
-@buildkite/agent-stewards
+* @buildkite/agent-stewards @buildkite/agents

From 2f17a0086b3abfece882239f2308fefe4de65aec Mon Sep 17 00:00:00 2001
From: Ming <ming@buildkite.com>
Date: Mon, 2 Mar 2026 16:24:03 +1100
Subject: [PATCH 202/242] A-970: Skip git fetch for already-present commits
 during checkout

---
 agent/agent_configuration.go |   1 +
 agent/job_runner.go          |   3 +
 clicommand/agent_start.go    |  27 ++++---
 clicommand/bootstrap.go      |   7 ++
 internal/job/checkout.go     | 145 ++++++++++++++++++++---------------
 internal/job/config.go       |   3 +
 6 files changed, 113 insertions(+), 73 deletions(-)

diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go
index a9b53a3ddd..3929b743b9 100644
--- a/agent/agent_configuration.go
+++ b/agent/agent_configuration.go
@@ -25,6 +25,7 @@ type AgentConfiguration struct {
 	GitFetchFlags               string
 	GitSubmodules               bool
 	SkipCheckout                bool
+	GitSkipFetchExistingCommits bool
 	AllowedRepositories         []*regexp.Regexp
 	AllowedPlugins              []*regexp.Regexp
 	AllowedEnvironmentVariables []*regexp.Regexp
diff --git a/agent/job_runner.go b/agent/job_runner.go
index a15e9ff1b4..b3282987b1 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -568,6 +568,9 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	if r.conf.AgentConfiguration.SkipCheckout {
 		setEnv("BUILDKITE_SKIP_CHECKOUT", "true")
 	}
+	if r.conf.AgentConfiguration.GitSkipFetchExistingCommits {
+		setEnv("BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS", "true")
+	}
 	setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
 	setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
 	// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index a9f811885c..8d919ed520 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -141,16 +141,17 @@ type AgentStartConfig struct {
 	WaitForECSMetaDataTimeout string   `cli:"wait-for-ecs-meta-data-timeout"`
 	WaitForGCPLabelsTimeout   string   `cli:"wait-for-gcp-labels-timeout"`
 
-	GitCheckoutFlags      string `cli:"git-checkout-flags"`
-	GitCloneFlags         string `cli:"git-clone-flags"`
-	GitCloneMirrorFlags   string `cli:"git-clone-mirror-flags"`
-	GitCleanFlags         string `cli:"git-clean-flags"`
-	GitFetchFlags         string `cli:"git-fetch-flags"`
-	GitMirrorsPath        string `cli:"git-mirrors-path" normalize:"filepath"`
-	GitMirrorsLockTimeout int    `cli:"git-mirrors-lock-timeout"`
-	GitMirrorsSkipUpdate  bool   `cli:"git-mirrors-skip-update"`
-	NoGitSubmodules       bool   `cli:"no-git-submodules"`
-	SkipCheckout          bool   `cli:"skip-checkout"`
+	GitCheckoutFlags            string `cli:"git-checkout-flags"`
+	GitCloneFlags               string `cli:"git-clone-flags"`
+	GitCloneMirrorFlags         string `cli:"git-clone-mirror-flags"`
+	GitCleanFlags               string `cli:"git-clean-flags"`
+	GitFetchFlags               string `cli:"git-fetch-flags"`
+	GitMirrorsPath              string `cli:"git-mirrors-path" normalize:"filepath"`
+	GitMirrorsLockTimeout       int    `cli:"git-mirrors-lock-timeout"`
+	GitMirrorsSkipUpdate        bool   `cli:"git-mirrors-skip-update"`
+	NoGitSubmodules             bool   `cli:"no-git-submodules"`
+	SkipCheckout                bool   `cli:"skip-checkout"`
+	GitSkipFetchExistingCommits bool   `cli:"git-skip-fetch-existing-commits"`
 
 	NoSSHKeyscan            bool     `cli:"no-ssh-keyscan"`
 	NoCommandEval           bool     `cli:"no-command-eval"`
@@ -630,6 +631,11 @@ var AgentStartCommand = cli.Command{
 			Usage:  "Skip the git checkout phase entirely",
 			EnvVar: "BUILDKITE_SKIP_CHECKOUT",
 		},
+		cli.BoolFlag{
+			Name:   "git-skip-fetch-existing-commits",
+			Usage:  "Skip git fetch if the commit already exists in the local git directory (default: false)",
+			EnvVar: "BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS",
+		},
 		cli.BoolFlag{
 			Name:   "no-feature-reporting",
 			Usage:  "Disables sending a list of enabled features back to the Buildkite mothership. We use this information to measure feature usage, but if you're not comfortable sharing that information then that's totally okay :) (default: false)",
@@ -1047,6 +1053,7 @@ var AgentStartCommand = cli.Command{
 			GitFetchFlags:                cfg.GitFetchFlags,
 			GitSubmodules:                !cfg.NoGitSubmodules,
 			SkipCheckout:                 cfg.SkipCheckout,
+			GitSkipFetchExistingCommits:  cfg.GitSkipFetchExistingCommits,
 			SSHKeyscan:                   !cfg.NoSSHKeyscan,
 			CommandEval:                  !cfg.NoCommandEval,
 			PluginsEnabled:               !cfg.NoPlugins,
diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go
index 48217f6d6e..eec06e2050 100644
--- a/clicommand/bootstrap.go
+++ b/clicommand/bootstrap.go
@@ -69,6 +69,7 @@ type BootstrapConfig struct {
 	ArtifactUploadDestination    string   `cli:"artifact-upload-destination"`
 	CleanCheckout                bool     `cli:"clean-checkout"`
 	SkipCheckout                 bool     `cli:"skip-checkout"`
+	GitSkipFetchExistingCommits  bool     `cli:"git-skip-fetch-existing-commits"`
 	GitCheckoutFlags             string   `cli:"git-checkout-flags"`
 	GitCloneFlags                string   `cli:"git-clone-flags"`
 	GitFetchFlags                string   `cli:"git-fetch-flags"`
@@ -233,6 +234,11 @@ var BootstrapCommand = cli.Command{
 			Usage:  "Skip the git checkout phase entirely",
 			EnvVar: "BUILDKITE_SKIP_CHECKOUT",
 		},
+		cli.BoolFlag{
+			Name:   "git-skip-fetch-existing-commits",
+			Usage:  "Skip git fetch if the commit already exists in the local git directory",
+			EnvVar: "BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS",
+		},
 		cli.StringFlag{
 			Name:   "git-checkout-flags",
 			Value:  "-f",
@@ -475,6 +481,7 @@ var BootstrapCommand = cli.Command{
 			SignalGracePeriod:            signalGracePeriod,
 			CleanCheckout:                cfg.CleanCheckout,
 			SkipCheckout:                 cfg.SkipCheckout,
+			GitSkipFetchExistingCommits:  cfg.GitSkipFetchExistingCommits,
 			Command:                      cfg.Command,
 			CommandEval:                  cfg.CommandEval,
 			Commit:                       cfg.Commit,
diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index 50999c5f97..094196ba2d 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -587,69 +587,16 @@ func (e *Executor) getOrUpdateMirrorDir(ctx context.Context, repository string)
 	return e.updateGitMirror(ctx, repository)
 }
 
-// defaultCheckoutPhase is called by the CheckoutPhase if no global or plugin checkout
-// hook exists. It performs the default checkout on the Repository provided in the config
-func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
-	span, _ := tracetools.StartSpanFromContext(ctx, "repo-checkout", e.TracingBackend)
-	span.AddAttributes(map[string]string{
-		"checkout.repo_name": e.Repository,
-		"checkout.refspec":   e.RefSpec,
-		"checkout.commit":    e.Commit,
-	})
-	var err error
-	defer func() { span.FinishWithError(err) }()
-
-	if e.SSHKeyscan {
-		addRepositoryHostToSSHKnownHosts(ctx, e.shell, e.Repository)
-	}
-
-	var mirrorDir string
-
-	// If we can, get a mirror of the git repository to use for reference later
-	if e.GitMirrorsPath != "" && e.Repository != "" {
-		span.AddAttributes(map[string]string{"checkout.is_using_git_mirrors": "true"})
-		mirrorDir, err = e.getOrUpdateMirrorDir(ctx, e.Repository)
-		if err != nil {
-			return fmt.Errorf("getting/updating git mirror: %w", err)
-		}
-
-		e.shell.Env.Set("BUILDKITE_REPO_MIRROR", mirrorDir)
-	}
-
-	// Make sure the build directory exists and that we change directory into it
-	if err := e.createCheckoutDir(); err != nil {
-		return fmt.Errorf("creating checkout dir: %w", err)
-	}
-
-	gitCloneFlags := e.GitCloneFlags
-	if mirrorDir != "" {
-		gitCloneFlags += fmt.Sprintf(" --reference %q", mirrorDir)
-	}
-
-	// Does the git directory exist?
-	existingGitDir := filepath.Join(e.shell.Getwd(), ".git")
-	if osutil.FileExists(existingGitDir) {
-		// Update the origin of the repository so we can gracefully handle
-		// repository renames
-		if _, err := e.updateRemoteURL(ctx, "", e.Repository); err != nil {
-			return fmt.Errorf("setting origin: %w", err)
-		}
-	} else {
-		if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
-			return fmt.Errorf("cloning git repository: %w", err)
-		}
-	}
-
-	// Git clean prior to checkout, we do this even if submodules have been
-	// disabled to ensure previous submodules are cleaned up
-	if hasGitSubmodules(e.shell) {
-		if err := gitCleanSubmodules(ctx, e.shell, e.GitCleanFlags); err != nil {
-			return fmt.Errorf("cleaning git submodules: %w", err)
-		}
-	}
-
-	if err := gitClean(ctx, e.shell, e.GitCleanFlags); err != nil {
-		return fmt.Errorf("cleaning git repository: %w", err)
+// fetchSource fetches the git source for the job. If GitSkipFetchExistingCommits is
+// enabled and the commit already exists locally, the fetch is skipped entirely.
+func (e *Executor) fetchSource(ctx context.Context) error {
+	// If configured, skip the fetch when the commit already exists locally.
+	// This is useful when a pre-populated git mirror is used with --reference,
+	// as the commit objects are already reachable and fetching is redundant.
+	if e.GitSkipFetchExistingCommits && e.Commit != "HEAD" &&
+		hasGitCommit(ctx, e.shell, ".git", e.Commit) {
+		e.shell.Commentf("Commit %q already exists locally, skipping fetch", e.Commit)
+		return nil
 	}
 
 	gitFetchFlags := e.GitFetchFlags
@@ -738,6 +685,78 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 		}
 	}
 
+	return nil
+}
+
+// defaultCheckoutPhase is called by the CheckoutPhase if no global or plugin checkout
+// hook exists. It performs the default checkout on the Repository provided in the config
+func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
+	span, _ := tracetools.StartSpanFromContext(ctx, "repo-checkout", e.TracingBackend)
+	span.AddAttributes(map[string]string{
+		"checkout.repo_name": e.Repository,
+		"checkout.refspec":   e.RefSpec,
+		"checkout.commit":    e.Commit,
+	})
+	var err error
+	defer func() { span.FinishWithError(err) }()
+
+	if e.SSHKeyscan {
+		addRepositoryHostToSSHKnownHosts(ctx, e.shell, e.Repository)
+	}
+
+	var mirrorDir string
+
+	// If we can, get a mirror of the git repository to use for reference later
+	if e.GitMirrorsPath != "" && e.Repository != "" {
+		span.AddAttributes(map[string]string{"checkout.is_using_git_mirrors": "true"})
+		mirrorDir, err = e.getOrUpdateMirrorDir(ctx, e.Repository)
+		if err != nil {
+			return fmt.Errorf("getting/updating git mirror: %w", err)
+		}
+
+		e.shell.Env.Set("BUILDKITE_REPO_MIRROR", mirrorDir)
+	}
+
+	// Make sure the build directory exists and that we change directory into it
+	if err := e.createCheckoutDir(); err != nil {
+		return fmt.Errorf("creating checkout dir: %w", err)
+	}
+
+	gitCloneFlags := e.GitCloneFlags
+	if mirrorDir != "" {
+		gitCloneFlags += fmt.Sprintf(" --reference %q", mirrorDir)
+	}
+
+	// Does the git directory exist?
+	existingGitDir := filepath.Join(e.shell.Getwd(), ".git")
+	if osutil.FileExists(existingGitDir) {
+		// Update the origin of the repository so we can gracefully handle
+		// repository renames
+		if _, err := e.updateRemoteURL(ctx, "", e.Repository); err != nil {
+			return fmt.Errorf("setting origin: %w", err)
+		}
+	} else {
+		if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
+			return fmt.Errorf("cloning git repository: %w", err)
+		}
+	}
+
+	// Git clean prior to checkout, we do this even if submodules have been
+	// disabled to ensure previous submodules are cleaned up
+	if hasGitSubmodules(e.shell) {
+		if err := gitCleanSubmodules(ctx, e.shell, e.GitCleanFlags); err != nil {
+			return fmt.Errorf("cleaning git submodules: %w", err)
+		}
+	}
+
+	if err := gitClean(ctx, e.shell, e.GitCleanFlags); err != nil {
+		return fmt.Errorf("cleaning git repository: %w", err)
+	}
+
+	if err := e.fetchSource(ctx); err != nil {
+		return err
+	}
+
 	gitCheckoutFlags := e.GitCheckoutFlags
 
 	if e.Commit == "HEAD" {
diff --git a/internal/job/config.go b/internal/job/config.go
index 6921f5a763..55cee9889a 100644
--- a/internal/job/config.go
+++ b/internal/job/config.go
@@ -78,6 +78,9 @@ type ExecutorConfig struct {
 	// Skip the checkout phase entirely
 	SkipCheckout bool `env:"BUILDKITE_SKIP_CHECKOUT"`
 
+	// Skip git fetch if the commit already exists locally
+	GitSkipFetchExistingCommits bool `env:"BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS"`
+
 	// Flags to pass to "git checkout" command
 	GitCheckoutFlags string `env:"BUILDKITE_GIT_CHECKOUT_FLAGS"`
 

From 056a17d70dd40b13518b96318d61b38f3ee29bb0 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 2 Feb 2026 15:40:00 +1100
Subject: [PATCH 203/242] Vendor new protobuf, generate the stubs

---
 .buildkite/pipeline.yml                       |  24 +-
 .buildkite/steps/check-protobuf-generation.sh |  19 +
 api/proto/agentedge.proto                     |  41 ++
 api/proto/buf.gen.yaml                        |  18 +
 api/proto/buf.lock                            |   6 +
 api/proto/buf.yaml                            |  10 +
 api/proto/gen/agentedge.pb.go                 | 488 ++++++++++++++++++
 .../agentedgev1connect/agentedge.connect.go   | 109 ++++
 go.mod                                        |   4 +-
 go.sum                                        |   4 +
 10 files changed, 720 insertions(+), 3 deletions(-)
 create mode 100755 .buildkite/steps/check-protobuf-generation.sh
 create mode 100644 api/proto/agentedge.proto
 create mode 100644 api/proto/buf.gen.yaml
 create mode 100644 api/proto/buf.lock
 create mode 100644 api/proto/buf.yaml
 create mode 100644 api/proto/gen/agentedge.pb.go
 create mode 100644 api/proto/gen/agentedgev1connect/agentedge.connect.go

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index cd37e5a7c3..f2c9d90818 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -13,7 +13,10 @@ steps:
   - name: ":go::robot_face: Lint"
     key: check-code-committed
     command: .buildkite/steps/check-code-committed.sh
-    if_changed: "{go.mod,go.sum,**.go,.buildkite/steps/check-code-committed.sh}"
+    if_changed:
+      - go.{mod,sum}
+      - "**.go"
+      - .buildkite/steps/check-code-committed.sh
     plugins:
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
@@ -21,8 +24,25 @@ steps:
           mount-buildkite-agent: true
           run: lint
 
+  - name: ":go::robot_face: Check protobuf generation"
+    key: check-protobuf-genreation
+    command: .buildkite/steps/check-protobuf-generation.sh
+    if_changed:
+      - api/proto/**
+      - .buildkite/steps/check-protobuf-generation.sh
+    plugins:
+    - docker-compose#v4.14.0:
+        config: .buildkite/docker-compose.yml
+        cli-version: 2
+        mount-buildkite-agent: true
+        run: lint
+
   - group: ":go::scientist: Tests and Coverage"
-    if_changed: "{go.mod,go.sum,**.go,**/fixtures/**,.buildkite/steps/{tests,test-coverage-report}.sh}"
+    if_changed:
+      - go.{mod,sum}
+      - "**.go"
+      - "**/fixtures/**"
+      - .buildkite/steps/{tests,test-coverage-report}.sh
     steps:
     - name: ":linux: Linux AMD64 Tests"
       key: test-linux-amd64
diff --git a/.buildkite/steps/check-protobuf-generation.sh b/.buildkite/steps/check-protobuf-generation.sh
new file mode 100755
index 0000000000..cbbad46245
--- /dev/null
+++ b/.buildkite/steps/check-protobuf-generation.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+
+set -euf
+
+cd api/proto
+
+echo --- :buf: Installing buf...
+go install github.com/bufbuild/buf/cmd/buf@v1.61.0
+go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.10
+go install connectrpc.com/connect/cmd/protoc-gen-connect-go@v1.19.1
+
+echo --- :connectrpc: Checking protobuf file generation...
+buf generate
+if ! git diff --no-ext-diff --exit-code; then
+  echo ^^^ +++
+  echo "Generated protobuf files are out of sync with the source code"
+  echo "Please run \`buf generate\` in the internal/proto directory locally, and commit the result."
+  exit 1
+fi
diff --git a/api/proto/agentedge.proto b/api/proto/agentedge.proto
new file mode 100644
index 0000000000..4e2645ab11
--- /dev/null
+++ b/api/proto/agentedge.proto
@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package agentedge.v1;
+
+import "buf/validate/validate.proto";
+
+message StreamPingsRequest {
+  string agent_id = 1;
+}
+
+message StreamPingsResponse {
+
+  oneof action {
+    ResumeAction resume = 2;
+    PauseAction pause = 3;
+    DisconnectAction disconnect = 4;
+    JobAssignedAction job_assigned = 5;
+  }
+}
+
+message ResumeAction {}
+
+message PauseAction {
+  string reason = 1;
+}
+
+message DisconnectAction {
+  string reason = 1;
+}
+
+message JobAssignedAction {
+  Job job = 1;
+}
+
+message Job {
+  string id = 1;
+}
+
+service AgentEdgeService {
+  rpc StreamPings(StreamPingsRequest) returns (stream StreamPingsResponse) {}
+}
diff --git a/api/proto/buf.gen.yaml b/api/proto/buf.gen.yaml
new file mode 100644
index 0000000000..9cb9c34231
--- /dev/null
+++ b/api/proto/buf.gen.yaml
@@ -0,0 +1,18 @@
+version: v2
+plugins:
+  - local: protoc-gen-go
+    out: gen
+    opt:
+      - paths=source_relative
+  - local: protoc-gen-connect-go
+    out: gen
+    opt:
+      - paths=source_relative
+managed:
+  enabled: true
+  override:
+    - file_option: go_package_prefix
+      value: github.com/buildkite/agent/v3/api/proto/gen
+  disable:
+    - file_option: go_package
+      module: buf.build/bufbuild/protovalidate
diff --git a/api/proto/buf.lock b/api/proto/buf.lock
new file mode 100644
index 0000000000..bb66131a69
--- /dev/null
+++ b/api/proto/buf.lock
@@ -0,0 +1,6 @@
+# Generated by buf. DO NOT EDIT.
+version: v2
+deps:
+  - name: buf.build/bufbuild/protovalidate
+    commit: 52f32327d4b045a79293a6ad4e7e1236
+    digest: b5:cbabc98d4b7b7b0447c9b15f68eeb8a7a44ef8516cb386ac5f66e7fd4062cd6723ed3f452ad8c384b851f79e33d26e7f8a94e2b807282b3def1cd966c7eace97
diff --git a/api/proto/buf.yaml b/api/proto/buf.yaml
new file mode 100644
index 0000000000..5685844978
--- /dev/null
+++ b/api/proto/buf.yaml
@@ -0,0 +1,10 @@
+# For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml
+version: v2
+deps:
+  - buf.build/bufbuild/protovalidate
+lint:
+  use:
+    - STANDARD
+breaking:
+  use:
+    - FILE
diff --git a/api/proto/gen/agentedge.pb.go b/api/proto/gen/agentedge.pb.go
new file mode 100644
index 0000000000..08c33aebac
--- /dev/null
+++ b/api/proto/gen/agentedge.pb.go
@@ -0,0 +1,488 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.10
+// 	protoc        (unknown)
+// source: agentedge.proto
+
+package agentedgev1
+
+import (
+	_ "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type StreamPingsRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	AgentId       string                 `protobuf:"bytes,1,opt,name=agent_id,json=agentId,proto3" json:"agent_id,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *StreamPingsRequest) Reset() {
+	*x = StreamPingsRequest{}
+	mi := &file_agentedge_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *StreamPingsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StreamPingsRequest) ProtoMessage() {}
+
+func (x *StreamPingsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_agentedge_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StreamPingsRequest.ProtoReflect.Descriptor instead.
+func (*StreamPingsRequest) Descriptor() ([]byte, []int) {
+	return file_agentedge_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *StreamPingsRequest) GetAgentId() string {
+	if x != nil {
+		return x.AgentId
+	}
+	return ""
+}
+
+type StreamPingsResponse struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Types that are valid to be assigned to Action:
+	//
+	//	*StreamPingsResponse_Resume
+	//	*StreamPingsResponse_Pause
+	//	*StreamPingsResponse_Disconnect
+	//	*StreamPingsResponse_JobAssigned
+	Action        isStreamPingsResponse_Action `protobuf_oneof:"action"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *StreamPingsResponse) Reset() {
+	*x = StreamPingsResponse{}
+	mi := &file_agentedge_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *StreamPingsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StreamPingsResponse) ProtoMessage() {}
+
+func (x *StreamPingsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_agentedge_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StreamPingsResponse.ProtoReflect.Descriptor instead.
+func (*StreamPingsResponse) Descriptor() ([]byte, []int) {
+	return file_agentedge_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *StreamPingsResponse) GetAction() isStreamPingsResponse_Action {
+	if x != nil {
+		return x.Action
+	}
+	return nil
+}
+
+func (x *StreamPingsResponse) GetResume() *ResumeAction {
+	if x != nil {
+		if x, ok := x.Action.(*StreamPingsResponse_Resume); ok {
+			return x.Resume
+		}
+	}
+	return nil
+}
+
+func (x *StreamPingsResponse) GetPause() *PauseAction {
+	if x != nil {
+		if x, ok := x.Action.(*StreamPingsResponse_Pause); ok {
+			return x.Pause
+		}
+	}
+	return nil
+}
+
+func (x *StreamPingsResponse) GetDisconnect() *DisconnectAction {
+	if x != nil {
+		if x, ok := x.Action.(*StreamPingsResponse_Disconnect); ok {
+			return x.Disconnect
+		}
+	}
+	return nil
+}
+
+func (x *StreamPingsResponse) GetJobAssigned() *JobAssignedAction {
+	if x != nil {
+		if x, ok := x.Action.(*StreamPingsResponse_JobAssigned); ok {
+			return x.JobAssigned
+		}
+	}
+	return nil
+}
+
+type isStreamPingsResponse_Action interface {
+	isStreamPingsResponse_Action()
+}
+
+type StreamPingsResponse_Resume struct {
+	Resume *ResumeAction `protobuf:"bytes,2,opt,name=resume,proto3,oneof"`
+}
+
+type StreamPingsResponse_Pause struct {
+	Pause *PauseAction `protobuf:"bytes,3,opt,name=pause,proto3,oneof"`
+}
+
+type StreamPingsResponse_Disconnect struct {
+	Disconnect *DisconnectAction `protobuf:"bytes,4,opt,name=disconnect,proto3,oneof"`
+}
+
+type StreamPingsResponse_JobAssigned struct {
+	JobAssigned *JobAssignedAction `protobuf:"bytes,5,opt,name=job_assigned,json=jobAssigned,proto3,oneof"`
+}
+
+func (*StreamPingsResponse_Resume) isStreamPingsResponse_Action() {}
+
+func (*StreamPingsResponse_Pause) isStreamPingsResponse_Action() {}
+
+func (*StreamPingsResponse_Disconnect) isStreamPingsResponse_Action() {}
+
+func (*StreamPingsResponse_JobAssigned) isStreamPingsResponse_Action() {}
+
+type ResumeAction struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ResumeAction) Reset() {
+	*x = ResumeAction{}
+	mi := &file_agentedge_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ResumeAction) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ResumeAction) ProtoMessage() {}
+
+func (x *ResumeAction) ProtoReflect() protoreflect.Message {
+	mi := &file_agentedge_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ResumeAction.ProtoReflect.Descriptor instead.
+func (*ResumeAction) Descriptor() ([]byte, []int) {
+	return file_agentedge_proto_rawDescGZIP(), []int{2}
+}
+
+type PauseAction struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Reason        string                 `protobuf:"bytes,1,opt,name=reason,proto3" json:"reason,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *PauseAction) Reset() {
+	*x = PauseAction{}
+	mi := &file_agentedge_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PauseAction) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PauseAction) ProtoMessage() {}
+
+func (x *PauseAction) ProtoReflect() protoreflect.Message {
+	mi := &file_agentedge_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PauseAction.ProtoReflect.Descriptor instead.
+func (*PauseAction) Descriptor() ([]byte, []int) {
+	return file_agentedge_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *PauseAction) GetReason() string {
+	if x != nil {
+		return x.Reason
+	}
+	return ""
+}
+
+type DisconnectAction struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Reason        string                 `protobuf:"bytes,1,opt,name=reason,proto3" json:"reason,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *DisconnectAction) Reset() {
+	*x = DisconnectAction{}
+	mi := &file_agentedge_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *DisconnectAction) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DisconnectAction) ProtoMessage() {}
+
+func (x *DisconnectAction) ProtoReflect() protoreflect.Message {
+	mi := &file_agentedge_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DisconnectAction.ProtoReflect.Descriptor instead.
+func (*DisconnectAction) Descriptor() ([]byte, []int) {
+	return file_agentedge_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *DisconnectAction) GetReason() string {
+	if x != nil {
+		return x.Reason
+	}
+	return ""
+}
+
+type JobAssignedAction struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Job           *Job                   `protobuf:"bytes,1,opt,name=job,proto3" json:"job,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *JobAssignedAction) Reset() {
+	*x = JobAssignedAction{}
+	mi := &file_agentedge_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobAssignedAction) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobAssignedAction) ProtoMessage() {}
+
+func (x *JobAssignedAction) ProtoReflect() protoreflect.Message {
+	mi := &file_agentedge_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobAssignedAction.ProtoReflect.Descriptor instead.
+func (*JobAssignedAction) Descriptor() ([]byte, []int) {
+	return file_agentedge_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *JobAssignedAction) GetJob() *Job {
+	if x != nil {
+		return x.Job
+	}
+	return nil
+}
+
+type Job struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Id            string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Job) Reset() {
+	*x = Job{}
+	mi := &file_agentedge_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Job) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Job) ProtoMessage() {}
+
+func (x *Job) ProtoReflect() protoreflect.Message {
+	mi := &file_agentedge_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Job.ProtoReflect.Descriptor instead.
+func (*Job) Descriptor() ([]byte, []int) {
+	return file_agentedge_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *Job) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+var File_agentedge_proto protoreflect.FileDescriptor
+
+const file_agentedge_proto_rawDesc = "" +
+	"\n" +
+	"\x0fagentedge.proto\x12\fagentedge.v1\x1a\x1bbuf/validate/validate.proto\"/\n" +
+	"\x12StreamPingsRequest\x12\x19\n" +
+	"\bagent_id\x18\x01 \x01(\tR\aagentId\"\x90\x02\n" +
+	"\x13StreamPingsResponse\x124\n" +
+	"\x06resume\x18\x02 \x01(\v2\x1a.agentedge.v1.ResumeActionH\x00R\x06resume\x121\n" +
+	"\x05pause\x18\x03 \x01(\v2\x19.agentedge.v1.PauseActionH\x00R\x05pause\x12@\n" +
+	"\n" +
+	"disconnect\x18\x04 \x01(\v2\x1e.agentedge.v1.DisconnectActionH\x00R\n" +
+	"disconnect\x12D\n" +
+	"\fjob_assigned\x18\x05 \x01(\v2\x1f.agentedge.v1.JobAssignedActionH\x00R\vjobAssignedB\b\n" +
+	"\x06action\"\x0e\n" +
+	"\fResumeAction\"%\n" +
+	"\vPauseAction\x12\x16\n" +
+	"\x06reason\x18\x01 \x01(\tR\x06reason\"*\n" +
+	"\x10DisconnectAction\x12\x16\n" +
+	"\x06reason\x18\x01 \x01(\tR\x06reason\"8\n" +
+	"\x11JobAssignedAction\x12#\n" +
+	"\x03job\x18\x01 \x01(\v2\x11.agentedge.v1.JobR\x03job\"\x15\n" +
+	"\x03Job\x12\x0e\n" +
+	"\x02id\x18\x01 \x01(\tR\x02id2j\n" +
+	"\x10AgentEdgeService\x12V\n" +
+	"\vStreamPings\x12 .agentedge.v1.StreamPingsRequest\x1a!.agentedge.v1.StreamPingsResponse\"\x000\x01B\xac\x01\n" +
+	"\x10com.agentedge.v1B\x0eAgentedgeProtoP\x01Z7github.com/buildkite/agent/v3/api/proto/gen;agentedgev1\xa2\x02\x03AXX\xaa\x02\fAgentedge.V1\xca\x02\fAgentedge\\V1\xe2\x02\x18Agentedge\\V1\\GPBMetadata\xea\x02\rAgentedge::V1b\x06proto3"
+
+var (
+	file_agentedge_proto_rawDescOnce sync.Once
+	file_agentedge_proto_rawDescData []byte
+)
+
+func file_agentedge_proto_rawDescGZIP() []byte {
+	file_agentedge_proto_rawDescOnce.Do(func() {
+		file_agentedge_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agentedge_proto_rawDesc), len(file_agentedge_proto_rawDesc)))
+	})
+	return file_agentedge_proto_rawDescData
+}
+
+var file_agentedge_proto_msgTypes = make([]protoimpl.MessageInfo, 7)
+var file_agentedge_proto_goTypes = []any{
+	(*StreamPingsRequest)(nil),  // 0: agentedge.v1.StreamPingsRequest
+	(*StreamPingsResponse)(nil), // 1: agentedge.v1.StreamPingsResponse
+	(*ResumeAction)(nil),        // 2: agentedge.v1.ResumeAction
+	(*PauseAction)(nil),         // 3: agentedge.v1.PauseAction
+	(*DisconnectAction)(nil),    // 4: agentedge.v1.DisconnectAction
+	(*JobAssignedAction)(nil),   // 5: agentedge.v1.JobAssignedAction
+	(*Job)(nil),                 // 6: agentedge.v1.Job
+}
+var file_agentedge_proto_depIdxs = []int32{
+	2, // 0: agentedge.v1.StreamPingsResponse.resume:type_name -> agentedge.v1.ResumeAction
+	3, // 1: agentedge.v1.StreamPingsResponse.pause:type_name -> agentedge.v1.PauseAction
+	4, // 2: agentedge.v1.StreamPingsResponse.disconnect:type_name -> agentedge.v1.DisconnectAction
+	5, // 3: agentedge.v1.StreamPingsResponse.job_assigned:type_name -> agentedge.v1.JobAssignedAction
+	6, // 4: agentedge.v1.JobAssignedAction.job:type_name -> agentedge.v1.Job
+	0, // 5: agentedge.v1.AgentEdgeService.StreamPings:input_type -> agentedge.v1.StreamPingsRequest
+	1, // 6: agentedge.v1.AgentEdgeService.StreamPings:output_type -> agentedge.v1.StreamPingsResponse
+	6, // [6:7] is the sub-list for method output_type
+	5, // [5:6] is the sub-list for method input_type
+	5, // [5:5] is the sub-list for extension type_name
+	5, // [5:5] is the sub-list for extension extendee
+	0, // [0:5] is the sub-list for field type_name
+}
+
+func init() { file_agentedge_proto_init() }
+func file_agentedge_proto_init() {
+	if File_agentedge_proto != nil {
+		return
+	}
+	file_agentedge_proto_msgTypes[1].OneofWrappers = []any{
+		(*StreamPingsResponse_Resume)(nil),
+		(*StreamPingsResponse_Pause)(nil),
+		(*StreamPingsResponse_Disconnect)(nil),
+		(*StreamPingsResponse_JobAssigned)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentedge_proto_rawDesc), len(file_agentedge_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   7,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_agentedge_proto_goTypes,
+		DependencyIndexes: file_agentedge_proto_depIdxs,
+		MessageInfos:      file_agentedge_proto_msgTypes,
+	}.Build()
+	File_agentedge_proto = out.File
+	file_agentedge_proto_goTypes = nil
+	file_agentedge_proto_depIdxs = nil
+}
diff --git a/api/proto/gen/agentedgev1connect/agentedge.connect.go b/api/proto/gen/agentedgev1connect/agentedge.connect.go
new file mode 100644
index 0000000000..f2738d669f
--- /dev/null
+++ b/api/proto/gen/agentedgev1connect/agentedge.connect.go
@@ -0,0 +1,109 @@
+// Code generated by protoc-gen-connect-go. DO NOT EDIT.
+//
+// Source: agentedge.proto
+
+package agentedgev1connect
+
+import (
+	connect "connectrpc.com/connect"
+	context "context"
+	errors "errors"
+	gen "github.com/buildkite/agent/v3/api/proto/gen"
+	http "net/http"
+	strings "strings"
+)
+
+// This is a compile-time assertion to ensure that this generated file and the connect package are
+// compatible. If you get a compiler error that this constant is not defined, this code was
+// generated with a version of connect newer than the one compiled into your binary. You can fix the
+// problem by either regenerating this code with an older version of connect or updating the connect
+// version compiled into your binary.
+const _ = connect.IsAtLeastVersion1_13_0
+
+const (
+	// AgentEdgeServiceName is the fully-qualified name of the AgentEdgeService service.
+	AgentEdgeServiceName = "agentedge.v1.AgentEdgeService"
+)
+
+// These constants are the fully-qualified names of the RPCs defined in this package. They're
+// exposed at runtime as Spec.Procedure and as the final two segments of the HTTP route.
+//
+// Note that these are different from the fully-qualified method names used by
+// google.golang.org/protobuf/reflect/protoreflect. To convert from these constants to
+// reflection-formatted method names, remove the leading slash and convert the remaining slash to a
+// period.
+const (
+	// AgentEdgeServiceStreamPingsProcedure is the fully-qualified name of the AgentEdgeService's
+	// StreamPings RPC.
+	AgentEdgeServiceStreamPingsProcedure = "/agentedge.v1.AgentEdgeService/StreamPings"
+)
+
+// AgentEdgeServiceClient is a client for the agentedge.v1.AgentEdgeService service.
+type AgentEdgeServiceClient interface {
+	StreamPings(context.Context, *connect.Request[gen.StreamPingsRequest]) (*connect.ServerStreamForClient[gen.StreamPingsResponse], error)
+}
+
+// NewAgentEdgeServiceClient constructs a client for the agentedge.v1.AgentEdgeService service. By
+// default, it uses the Connect protocol with the binary Protobuf Codec, asks for gzipped responses,
+// and sends uncompressed requests. To use the gRPC or gRPC-Web protocols, supply the
+// connect.WithGRPC() or connect.WithGRPCWeb() options.
+//
+// The URL supplied here should be the base URL for the Connect or gRPC server (for example,
+// http://api.acme.com or https://acme.com/grpc).
+func NewAgentEdgeServiceClient(httpClient connect.HTTPClient, baseURL string, opts ...connect.ClientOption) AgentEdgeServiceClient {
+	baseURL = strings.TrimRight(baseURL, "/")
+	agentEdgeServiceMethods := gen.File_agentedge_proto.Services().ByName("AgentEdgeService").Methods()
+	return &agentEdgeServiceClient{
+		streamPings: connect.NewClient[gen.StreamPingsRequest, gen.StreamPingsResponse](
+			httpClient,
+			baseURL+AgentEdgeServiceStreamPingsProcedure,
+			connect.WithSchema(agentEdgeServiceMethods.ByName("StreamPings")),
+			connect.WithClientOptions(opts...),
+		),
+	}
+}
+
+// agentEdgeServiceClient implements AgentEdgeServiceClient.
+type agentEdgeServiceClient struct {
+	streamPings *connect.Client[gen.StreamPingsRequest, gen.StreamPingsResponse]
+}
+
+// StreamPings calls agentedge.v1.AgentEdgeService.StreamPings.
+func (c *agentEdgeServiceClient) StreamPings(ctx context.Context, req *connect.Request[gen.StreamPingsRequest]) (*connect.ServerStreamForClient[gen.StreamPingsResponse], error) {
+	return c.streamPings.CallServerStream(ctx, req)
+}
+
+// AgentEdgeServiceHandler is an implementation of the agentedge.v1.AgentEdgeService service.
+type AgentEdgeServiceHandler interface {
+	StreamPings(context.Context, *connect.Request[gen.StreamPingsRequest], *connect.ServerStream[gen.StreamPingsResponse]) error
+}
+
+// NewAgentEdgeServiceHandler builds an HTTP handler from the service implementation. It returns the
+// path on which to mount the handler and the handler itself.
+//
+// By default, handlers support the Connect, gRPC, and gRPC-Web protocols with the binary Protobuf
+// and JSON codecs. They also support gzip compression.
+func NewAgentEdgeServiceHandler(svc AgentEdgeServiceHandler, opts ...connect.HandlerOption) (string, http.Handler) {
+	agentEdgeServiceMethods := gen.File_agentedge_proto.Services().ByName("AgentEdgeService").Methods()
+	agentEdgeServiceStreamPingsHandler := connect.NewServerStreamHandler(
+		AgentEdgeServiceStreamPingsProcedure,
+		svc.StreamPings,
+		connect.WithSchema(agentEdgeServiceMethods.ByName("StreamPings")),
+		connect.WithHandlerOptions(opts...),
+	)
+	return "/agentedge.v1.AgentEdgeService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case AgentEdgeServiceStreamPingsProcedure:
+			agentEdgeServiceStreamPingsHandler.ServeHTTP(w, r)
+		default:
+			http.NotFound(w, r)
+		}
+	})
+}
+
+// UnimplementedAgentEdgeServiceHandler returns CodeUnimplemented from all methods.
+type UnimplementedAgentEdgeServiceHandler struct{}
+
+func (UnimplementedAgentEdgeServiceHandler) StreamPings(context.Context, *connect.Request[gen.StreamPingsRequest], *connect.ServerStream[gen.StreamPingsResponse]) error {
+	return connect.NewError(connect.CodeUnimplemented, errors.New("agentedge.v1.AgentEdgeService.StreamPings is not implemented"))
+}
diff --git a/go.mod b/go.mod
index 09a2bfb282..4069266721 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,9 @@ go 1.24.0
 toolchain go1.24.5
 
 require (
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20251209175733-2a1774d88802.1
 	cloud.google.com/go/compute/metadata v0.9.0
+	connectrpc.com/connect v1.19.1
 	drjosh.dev/zzglob v0.4.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
@@ -63,6 +65,7 @@ require (
 	golang.org/x/sys v0.41.0
 	golang.org/x/term v0.40.0
 	google.golang.org/api v0.260.0
+	google.golang.org/protobuf v1.36.11
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.2
@@ -206,7 +209,6 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
 	google.golang.org/grpc v1.78.0 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/gotestsum v1.13.0 // indirect
diff --git a/go.sum b/go.sum
index ce3773acbe..531634e49b 100644
--- a/go.sum
+++ b/go.sum
@@ -1,9 +1,13 @@
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20251209175733-2a1774d88802.1 h1:j9yeqTWEFrtimt8Nng2MIeRrpoCvQzM9/g25XTvqUGg=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20251209175733-2a1774d88802.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM=
 cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0=
 cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
+connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14=
+connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w=
 drjosh.dev/zzglob v0.4.2 h1:q+e5Cp6SFCyz+Yurhk/edSrTKEk3tn60vzoaXLmtiBo=
 drjosh.dev/zzglob v0.4.2/go.mod h1:SbYDdesQC13iyGiEwV8dJfJbyz7/Qiawrd5ODdJQCoo=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc=

From ba82ed73af7a91e8ca4b26f56ae09d328c14be16 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 3 Feb 2026 15:45:12 +1100
Subject: [PATCH 204/242] Add --ping-mode flag/env var/config

---
 agent/agent_configuration.go |  2 ++
 clicommand/agent_start.go    | 13 ++++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go
index 3929b743b9..664c373753 100644
--- a/agent/agent_configuration.go
+++ b/agent/agent_configuration.go
@@ -69,4 +69,6 @@ type AgentConfiguration struct {
 	TraceContextEncoding         string
 	DisableWarningsFor           []string
 	AllowMultipartArtifactUpload bool
+
+	PingMode string
 }
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 8d919ed520..0ebc1b640c 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -184,13 +184,15 @@ type AgentStartConfig struct {
 	TraceContextEncoding      string `cli:"trace-context-encoding"`
 	NoMultipartArtifactUpload bool   `cli:"no-multipart-artifact-upload"`
 
+	// API + agent behaviour
+	PingMode string `cli:"ping-mode"`
+
 	// API config
 	DebugHTTP bool   `cli:"debug-http"`
 	TraceHTTP bool   `cli:"trace-http"`
 	Token     string `cli:"token" validate:"required"`
 	Endpoint  string `cli:"endpoint" validate:"required"`
 	NoHTTP2   bool   `cli:"no-http2"`
-
 	// Deprecated
 	KubernetesLogCollectionGracePeriod time.Duration `cli:"kubernetes-log-collection-grace-period"`
 	NoSSHFingerprintVerification       bool          `cli:"no-automatic-ssh-fingerprint-verification" deprecated-and-renamed-to:"NoSSHKeyscan"`
@@ -759,6 +761,14 @@ var AgentStartCommand = cli.Command{
 			EnvVar: "BUILDKITE_AGENT_DISABLE_WARNINGS_FOR",
 		},
 
+		// API + agent behaviour
+		cli.StringFlag{
+			Name:   "ping-mode",
+			Usage:  "Selects available protocols for dispatching work to this agent. One of auto (default), ping-only, stream-only.",
+			Value:  "auto",
+			EnvVar: "BUILDKITE_AGENT_PING_MODE",
+		},
+
 		// API Flags
 		AgentRegisterTokenFlag, // != AgentAccessToken
 		EndpointFlag,
@@ -1083,6 +1093,7 @@ var AgentStartCommand = cli.Command{
 			TraceContextEncoding:         cfg.TraceContextEncoding,
 			AllowMultipartArtifactUpload: !cfg.NoMultipartArtifactUpload,
 			KubernetesExec:               cfg.KubernetesExec,
+			PingMode:                     cfg.PingMode,
 
 			SigningJWKSFile:  cfg.SigningJWKSFile,
 			SigningJWKSKeyID: cfg.SigningJWKSKeyID,

From 4fffc7311e849c1c8572f120bb86767109edc253 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Feb 2026 17:37:19 +1100
Subject: [PATCH 205/242] Add API client method for streaming

---
 api/pings_streaming.go | 72 ++++++++++++++++++++++++++++++++++++++++++
 api/retryable.go       | 15 ++++-----
 2 files changed, 79 insertions(+), 8 deletions(-)
 create mode 100644 api/pings_streaming.go

diff --git a/api/pings_streaming.go b/api/pings_streaming.go
new file mode 100644
index 0000000000..f591680a30
--- /dev/null
+++ b/api/pings_streaming.go
@@ -0,0 +1,72 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"iter"
+	"net/url"
+
+	"connectrpc.com/connect"
+	agentedgev1 "github.com/buildkite/agent/v3/api/proto/gen"
+	"github.com/buildkite/agent/v3/api/proto/gen/agentedgev1connect"
+)
+
+// StreamPings opens a ConnectRPC channel for streaming pings. It returns an
+// iterator over received messages and any error that occurs.
+func (c *Client) StreamPings(ctx context.Context, agentID string, opts ...connect.ClientOption) (iter.Seq2[*agentedgev1.StreamPingsResponse, error], error) {
+	// The streaming endpoint is the same as the main endpoint,
+	// minus the `/v3/`.
+	u, err := url.Parse(c.conf.Endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("parsing endpoint: %w", err)
+	}
+	u.Path = "/"
+
+	cl := agentedgev1connect.NewAgentEdgeServiceClient(
+		c.client,
+		u.String(),
+		connect.WithGRPC(),
+		connect.WithClientOptions(opts...),
+	)
+
+	// In order to set request headers, we need to tweak a value set in the
+	// context. To me, this feels too much like burying optional parameters
+	// in a context, which I think is bad - https://pkg.go.dev/context says:
+	// "Use context Values only for request-scoped data that transits processes
+	// and APIs, not for passing optional parameters to functions."
+	ctx, callInfo := connect.NewClientContext(ctx)
+	h := callInfo.RequestHeader()
+
+	// Add any request headers specified by the server during register/ping
+	for k, values := range c.requestHeaders {
+		for _, v := range values {
+			h.Add(k, v)
+		}
+	}
+
+	// The Authorization header is added by the custom transport.
+	// Other methods add User-Agent in newRequest.
+	// Note that this does not set the entire header.
+	// ConnectRPC takes our value here and adds its own component *before* our
+	// own, which violates the convention of decreasing importance
+	// (see RFC 7231 section 5.5.3).
+	h.Set("User-Agent", c.conf.UserAgent)
+	stream, err := cl.StreamPings(ctx, connect.NewRequest(&agentedgev1.StreamPingsRequest{
+		AgentId: agentID,
+	}))
+	if err != nil {
+		return nil, fmt.Errorf("from StreamPings: %w", err)
+	}
+
+	return func(yield func(*agentedgev1.StreamPingsResponse, error) bool) {
+		defer stream.Close() //nolint:errcheck // Best-effort cleanup
+		for stream.Receive() {
+			if !yield(stream.Msg(), nil) {
+				return
+			}
+		}
+		if err := stream.Err(); err != nil {
+			yield(nil, err)
+		}
+	}, nil
+}
diff --git a/api/retryable.go b/api/retryable.go
index beb5f2d811..4666477aa3 100644
--- a/api/retryable.go
+++ b/api/retryable.go
@@ -5,7 +5,6 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"slices"
 	"strings"
 	"syscall"
 )
@@ -20,17 +19,17 @@ var retrableErrorSuffixes = []string{
 	io.EOF.Error(),
 }
 
-var retryableStatuses = []int{
-	http.StatusTooManyRequests,     // 429
-	http.StatusInternalServerError, // 500
-	http.StatusBadGateway,          // 502
-	http.StatusServiceUnavailable,  // 503
-	http.StatusGatewayTimeout,      // 504
+var retryableStatuses = map[int]bool{
+	http.StatusTooManyRequests:     true, // 429
+	http.StatusInternalServerError: true, // 500
+	http.StatusBadGateway:          true, // 502
+	http.StatusServiceUnavailable:  true, // 503
+	http.StatusGatewayTimeout:      true, // 504
 }
 
 // IsRetryableStatus returns true if the response's StatusCode is one that we should retry.
 func IsRetryableStatus(r *Response) bool {
-	return r.StatusCode >= 400 && slices.Contains(retryableStatuses, r.StatusCode)
+	return retryableStatuses[r.StatusCode]
 }
 
 // Looks at a bunch of connection related errors, and returns true if the error

From cbeb8f09b9e831d09dd1cd124c3afbebf47ee794 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Feb 2026 17:37:19 +1100
Subject: [PATCH 206/242] Focus internal API on jobID string rather than
 *api.Job

---
 agent/agent_worker.go | 36 ++++++++++++++++++------------------
 api/jobs.go           |  4 ++--
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index d988792c71..3740306514 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -411,7 +411,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		setStat("📡 Pinging Buildkite for instructions")
 		pingsSent.Inc()
 		startPing := time.Now()
-		job, action, err := a.Ping(ctx)
+		jobID, action, err := a.Ping(ctx)
 		if err != nil {
 			pingErrors.Inc()
 			if isUnrecoverable(err) {
@@ -447,8 +447,8 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		// For acquire-job agents, registration sets ignore-in-dispatches=true,
 		// so job should be nil. If not nil, complain.
 		if a.agentConfiguration.AcquireJob != "" {
-			if job != nil {
-				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in acquire-job mode!", job.ID)
+			if jobID != "" {
+				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in acquire-job mode!", jobID)
 			}
 			return nil
 		}
@@ -456,8 +456,8 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		// Exit after disconnect-after-job. Finishing the job sets
 		// ignore-in-dispatches=true, so job should be nil. If not, complain.
 		if ranJob && a.agentConfiguration.DisconnectAfterJob {
-			if job != nil {
-				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in disconnect-after-job mode (and already ran a job)!", job.ID)
+			if jobID != "" {
+				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in disconnect-after-job mode (and already ran a job)!", jobID)
 			}
 			a.logger.Info("Job ran, and disconnect-after-job is enabled. Disconnecting...")
 			return nil
@@ -466,8 +466,8 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		// Exit after disconnect-after-uptime is exceeded.
 		if maxUptime := a.agentConfiguration.DisconnectAfterUptime; maxUptime > 0 {
 			if time.Since(a.startTime) >= maxUptime {
-				if job != nil {
-					a.logger.Error("Agent ping dispatched a job (id %q) but agent has exceeded max uptime of %v!", job.ID, maxUptime)
+				if jobID != "" {
+					a.logger.Error("Agent ping dispatched a job (id %q) but agent has exceeded max uptime of %v!", jobID, maxUptime)
 				}
 				a.logger.Info("Agent has exceeded max uptime of %v. Disconnecting...", maxUptime)
 				return nil
@@ -475,7 +475,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		}
 
 		// Note that Ping only returns a job if err == nil.
-		if job == nil {
+		if jobID == "" {
 			idleTimeout := a.agentConfiguration.DisconnectAfterIdleTimeout
 			if idleTimeout == 0 {
 				// No job and no idle timeout.
@@ -499,7 +499,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		setStat("💼 Accepting job")
 
 		// Runs the job, only errors if something goes wrong
-		if err := a.AcceptAndRunJob(ctx, job, idleMon); err != nil {
+		if err := a.AcceptAndRunJob(ctx, jobID, idleMon); err != nil {
 			a.logger.Error("%v", err)
 			setStat(fmt.Sprintf("✅ Finished job with error: %v", err))
 			continue
@@ -621,7 +621,7 @@ func (a *AgentWorker) Heartbeat(ctx context.Context) error {
 
 // Performs a ping that checks Buildkite for a job or action to take
 // Returns a job, or nil if none is found
-func (a *AgentWorker) Ping(ctx context.Context) (job *api.Job, action string, err error) {
+func (a *AgentWorker) Ping(ctx context.Context) (jobID, action string, err error) {
 	ping, resp, pingErr := a.apiClient.Ping(ctx)
 	// wait a minute, where's my if err != nil block? TL;DR look for pingErr ~20 lines down
 	// the api client returns an error if the response code isn't a 2xx, but there's still information in resp and ping
@@ -642,7 +642,7 @@ func (a *AgentWorker) Ping(ctx context.Context) (job *api.Job, action string, er
 		// The reason we do this after the disconnect check is because the backend can (and does) send disconnect actions in
 		// responses with non-retryable statuses
 		if resp != nil && !api.IsRetryableStatus(resp) {
-			return nil, action, &errUnrecoverable{action: "Ping", response: resp, err: pingErr}
+			return "", action, &errUnrecoverable{action: "Ping", response: resp, err: pingErr}
 		}
 
 		// Get the last ping time to the nearest microsecond
@@ -652,9 +652,9 @@ func (a *AgentWorker) Ping(ctx context.Context) (job *api.Job, action string, er
 		// If a ping fails, we don't really care, because it'll
 		// ping again after the interval.
 		if a.stats.lastPing.IsZero() {
-			return nil, action, fmt.Errorf("Failed to ping: %w (No successful ping yet)", pingErr)
+			return "", action, fmt.Errorf("Failed to ping: %w (No successful ping yet)", pingErr)
 		} else {
-			return nil, action, fmt.Errorf("Failed to ping: %w (Last successful was %v ago)", pingErr, time.Since(a.stats.lastPing))
+			return "", action, fmt.Errorf("Failed to ping: %w (Last successful was %v ago)", pingErr, time.Since(a.stats.lastPing))
 		}
 	}
 
@@ -683,10 +683,10 @@ func (a *AgentWorker) Ping(ctx context.Context) (job *api.Job, action string, er
 	// If we don't have a job, there's nothing to do!
 	// If we're paused, job should be nil, but in case it isn't, ignore it.
 	if ping.Job == nil || action == "pause" {
-		return nil, action, nil
+		return "", action, nil
 	}
 
-	return ping.Job, action, nil
+	return ping.Job.ID, action, nil
 }
 
 // AcquireAndRunJob attempts to acquire a job an run it. It will retry at after the
@@ -718,8 +718,8 @@ func (a *AgentWorker) AcquireAndRunJob(ctx context.Context, jobId string) error
 }
 
 // Accepts a job and runs it, only returns an error if something goes wrong
-func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, job *api.Job, idleMon *idleMonitor) error {
-	a.logger.Info("Assigned job %s. Accepting...", job.ID)
+func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, jobID string, idleMon *idleMonitor) error {
+	a.logger.Info("Assigned job %s. Accepting...", jobID)
 
 	// An agent is busy during a job, and idle when the job is done.
 	idleMon.markBusy(a)
@@ -734,7 +734,7 @@ func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, job *api.Job, idleMon
 	)
 
 	accepted, err := roko.DoFunc(ctx, r, func(r *roko.Retrier) (*api.Job, error) {
-		accepted, _, err := a.apiClient.AcceptJob(ctx, job)
+		accepted, _, err := a.apiClient.AcceptJob(ctx, jobID)
 		if err != nil {
 			if api.IsRetryableError(err) {
 				a.logger.Warn("%s (%s)", err, r)
diff --git a/api/jobs.go b/api/jobs.go
index 7eb0986ae9..17f1a93daa 100644
--- a/api/jobs.go
+++ b/api/jobs.go
@@ -85,8 +85,8 @@ func (c *Client) AcquireJob(ctx context.Context, id string, headers ...Header) (
 // AcceptJob accepts the passed in job. Returns the job with its finalized set of
 // environment variables (when a job is accepted, the agents environment is
 // applied to the job)
-func (c *Client) AcceptJob(ctx context.Context, job *Job) (*Job, *Response, error) {
-	u := fmt.Sprintf("jobs/%s/accept", railsPathEscape(job.ID))
+func (c *Client) AcceptJob(ctx context.Context, jobID string) (*Job, *Response, error) {
+	u := fmt.Sprintf("jobs/%s/accept", railsPathEscape(jobID))
 
 	req, err := c.newRequest(ctx, "PUT", u, nil)
 	if err != nil {

From 15af4b0a6af753f66e7fb2fa07309c1f72ec69ee Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Feb 2026 17:37:19 +1100
Subject: [PATCH 207/242] Reimplement idleMonitor with channels+goroutine

---
 agent/agent_pool.go        |  13 +--
 agent/agent_worker.go      |  17 ++--
 agent/agent_worker_test.go |  31 +++----
 agent/idle_monitor.go      | 167 +++++++++++++++++++++++++++----------
 agent/idle_monitor_test.go |  91 ++++++++++++++++++++
 clicommand/agent_start.go  |   2 +-
 6 files changed, 241 insertions(+), 80 deletions(-)
 create mode 100644 agent/idle_monitor_test.go

diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index 064b344a88..cb0f75393e 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"strconv"
 	"sync"
+	"time"
 
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/status"
@@ -17,13 +18,15 @@ import (
 
 // AgentPool manages multiple parallel AgentWorkers.
 type AgentPool struct {
-	workers []*AgentWorker
+	workers     []*AgentWorker
+	idleTimeout time.Duration
 }
 
 // NewAgentPool returns a new AgentPool.
-func NewAgentPool(workers []*AgentWorker) *AgentPool {
+func NewAgentPool(workers []*AgentWorker, config *AgentConfiguration) *AgentPool {
 	return &AgentPool{
-		workers: workers,
+		workers:     workers,
+		idleTimeout: config.DisconnectAfterIdleTimeout,
 	}
 }
 
@@ -58,7 +61,7 @@ func (r *AgentPool) Start(ctx context.Context) error {
 	defer done()
 	setStat("🏃 Spawning workers...")
 
-	idleMon := newIdleMonitor(len(r.workers))
+	idleMon := NewIdleMonitor(ctx, len(r.workers), r.idleTimeout)
 
 	errCh := make(chan error)
 
@@ -82,7 +85,7 @@ func (r *AgentPool) Start(ctx context.Context) error {
 func runWorker(ctx context.Context, worker *AgentWorker, idleMon *idleMonitor) error {
 	agentWorkersStarted.Inc()
 	defer agentWorkersEnded.Inc()
-	defer idleMon.markDead(worker)
+	defer idleMon.MarkDead(worker)
 
 	// Connect the worker to the API
 	if err := worker.Connect(ctx); err != nil {
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 3740306514..14f60b9c0d 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -254,7 +254,7 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		// there's really no point in letting the idle monitor know
 		// we're busy, but it's probably a good thing to do for good
 		// measure.
-		idleMon.markBusy(a)
+		idleMon.MarkBusy(a)
 
 		if err := a.AcquireAndRunJob(ctx, a.agentConfiguration.AcquireJob); err != nil {
 			// If the job acquisition was rejected, we can exit with an error
@@ -484,16 +484,17 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 
 			// This ensures agents that never receive a job are still tracked
 			// by the idle monitor and can properly trigger disconnect-after-idle-timeout.
-			idleMon.markIdle(a)
+			idleMon.MarkIdle(a)
 
 			// Exit if every agent has been idle for at least the timeout.
-			if idleMon.shouldExit(idleTimeout) {
+			select {
+			case <-idleMon.Exiting():
 				a.logger.Info("All agents have been idle for at least %v. Disconnecting...", idleTimeout)
 				return nil
+			default:
+				// Not idle enough to exit. Wait and ping again.
+				continue
 			}
-
-			// Not idle enough to exit. Wait and ping again.
-			continue
 		}
 
 		setStat("💼 Accepting job")
@@ -722,8 +723,8 @@ func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, jobID string, idleMon
 	a.logger.Info("Assigned job %s. Accepting...", jobID)
 
 	// An agent is busy during a job, and idle when the job is done.
-	idleMon.markBusy(a)
-	defer idleMon.markIdle(a)
+	idleMon.MarkBusy(a)
+	defer idleMon.MarkIdle(a)
 
 	// Accept the job. We'll retry on connection related issues, but if
 	// Buildkite returns a 422 or 500 for example, we'll just bail out,
diff --git a/agent/agent_worker_test.go b/agent/agent_worker_test.go
index ce33d0c878..5c536dc61c 100644
--- a/agent/agent_worker_test.go
+++ b/agent/agent_worker_test.go
@@ -317,11 +317,9 @@ func TestAgentWorker_Start_AcquireJob_JobAcquisitionRejected(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := newIdleMonitor(1)
-
 	// we expect the worker to try to acquire the job, but fail with ErrJobAcquisitionRejected
 	// because the server returns a 422 Unprocessable Entity.
-	err := worker.Start(ctx, idleMonitor)
+	err := worker.Start(ctx, nil)
 	if !errors.Is(err, core.ErrJobAcquisitionRejected) {
 		t.Fatalf("expected worker.AcquireAndRunJob(%q) = core.ErrJobAcquisitionRejected, got %v", jobID, err)
 	}
@@ -400,9 +398,7 @@ func TestAgentWorker_Start_AcquireJob_Pause_Unpause(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := newIdleMonitor(1)
-
-	if err := worker.Start(ctx, idleMonitor); err != nil {
+	if err := worker.Start(ctx, nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -494,9 +490,7 @@ func TestAgentWorker_DisconnectAfterJob_Start_Pause_Unpause(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := newIdleMonitor(1)
-
-	if err := worker.Start(ctx, idleMonitor); err != nil {
+	if err := worker.Start(ctx, nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -581,12 +575,10 @@ func TestAgentWorker_DisconnectAfterUptime(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	idleMonitor := newIdleMonitor(1)
-
 	// Record start time
 	startTime := time.Now()
 
-	if err := worker.Start(ctx, idleMonitor); err != nil {
+	if err := worker.Start(ctx, nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -665,7 +657,7 @@ func TestAgentWorker_SetEndpointDuringRegistration(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -754,7 +746,7 @@ func TestAgentWorker_UpdateEndpointDuringPing(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -834,7 +826,7 @@ func TestAgentWorker_UpdateEndpointDuringPing_FailAndRevert(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -903,7 +895,7 @@ func TestAgentWorker_SetRequestHeadersDuringRegistration(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
+	if err := worker.Start(ctx, nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -915,9 +907,6 @@ func TestAgentWorker_SetRequestHeadersDuringRegistration(t *testing.T) {
 func TestAgentWorker_UpdateRequestHeadersDuringPing(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	const agentSessionToken = "alpacas"
 
 	server := NewFakeAPIServer()
@@ -991,7 +980,7 @@ func TestAgentWorker_UpdateRequestHeadersDuringPing(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, newIdleMonitor(1)); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -1046,7 +1035,7 @@ func TestAgentWorker_UnrecoverableErrorInPing(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, newIdleMonitor(1)); !isUnrecoverable(err) {
+	if err := worker.Start(ctx, nil); !isUnrecoverable(err) {
 		t.Errorf("worker.Start() = %v, want an unrecoverable error", err)
 	}
 
diff --git a/agent/idle_monitor.go b/agent/idle_monitor.go
index 1f3743c676..75c3288c39 100644
--- a/agent/idle_monitor.go
+++ b/agent/idle_monitor.go
@@ -1,7 +1,7 @@
 package agent
 
 import (
-	"sync"
+	"context"
 	"time"
 )
 
@@ -20,69 +20,146 @@ import (
 //                -> Idle --
 */
 type idleMonitor struct {
-	mu          sync.Mutex
-	exiting     bool
+	// exiting is closed when the idle monitor says all agents should exit
+	exiting chan struct{}
+
+	// totalAgents is the total number of agents configured to run
 	totalAgents int
-	idleAt      map[*AgentWorker]time.Time
+
+	// idleTimeout is a copy of the DisconnectAfterIdleTimeout value
+	idleTimeout time.Duration
+
+	// Channels used to update the monitor state
+	becameIdle chan *AgentWorker
+	becameBusy chan *AgentWorker
+	becameDead chan *AgentWorker
+
+	// idleAt tracks when each agent became idle/dead.
+	// Agents not present in the map are busy.
+	idleAt map[*AgentWorker]time.Time
 }
 
-// newIdleMonitor creates a new IdleMonitor.
-func newIdleMonitor(totalAgents int) *idleMonitor {
-	return &idleMonitor{
+// NewIdleMonitor creates a new IdleMonitor.
+func NewIdleMonitor(ctx context.Context, totalAgents int, idleTimeout time.Duration) *idleMonitor {
+	if idleTimeout <= 0 {
+		// Note that the methods handle a nil receiver safely.
+		return nil
+	}
+	i := &idleMonitor{
+		exiting:     make(chan struct{}),
 		totalAgents: totalAgents,
+		idleTimeout: idleTimeout,
+		becameIdle:  make(chan *AgentWorker),
+		becameBusy:  make(chan *AgentWorker),
+		becameDead:  make(chan *AgentWorker),
 		idleAt:      make(map[*AgentWorker]time.Time),
 	}
+	go i.monitor(ctx)
+	return i
 }
 
-// shouldExit reports whether all agents are dead or have been idle for at least
-// minIdle.  If shouldExit returns true, it will return true on all subsequent
-// calls.
-func (i *idleMonitor) shouldExit(minIdle time.Duration) bool {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	// Once the idle monitor decides we're exiting, we're exiting.
-	if i.exiting {
-		return true
+// monitor is the internal goroutine for handling idleness.
+func (i *idleMonitor) monitor(ctx context.Context) {
+	if i == nil {
+		return
 	}
 
-	// Are all alive agents dead or idle for long enough?
-	idle := 0
-	for _, t := range i.idleAt {
-		if !t.IsZero() && time.Since(t) < minIdle {
-			return false
+	// Once the idle monitor returns, all the agents should also exit.
+	defer close(i.exiting)
+
+	var lastTimeout <-chan time.Time
+	for {
+		select {
+		case <-ctx.Done():
+			return
+
+		case <-lastTimeout:
+			return
+
+		case agent := <-i.becameIdle:
+			// Idleness is counted from when the agent first became idle.
+			if _, alreadyIdle := i.idleAt[agent]; alreadyIdle {
+				break
+			}
+			i.idleAt[agent] = time.Now()
+
+		case agent := <-i.becameBusy:
+			delete(i.idleAt, agent)
+
+		case agent := <-i.becameDead:
+			i.idleAt[agent] = time.Time{}
+		}
+
+		// Update the timeout channel based on all the agent states
+		// Are there any busy agents? Then don't time out.
+		if len(i.idleAt) < i.totalAgents {
+			lastTimeout = nil
+			continue
+		}
+
+		// They're all idle or dead. Figure out when the timeout should happen.
+		// If they're all dead, then the timeout happens immediately.
+		// If at least one is idle and _not_ dead, then the timeout happens
+		// however much of idleTimeout remains since the agent that most
+		// recently became idle.
+		var timeout time.Duration
+		for _, t := range i.idleAt {
+			if t.IsZero() {
+				continue
+			}
+			timeout = max(timeout, i.idleTimeout-time.Since(t))
+		}
+		if timeout == 0 {
+			return
 		}
-		idle++
+		lastTimeout = time.After(timeout)
 	}
-	if idle < i.totalAgents {
-		return false
+}
+
+// Exiting returns a channel that is closed when the monitor declares
+// all agents should exit. It is safe to use with a nil pointer.
+func (i *idleMonitor) Exiting() <-chan struct{} {
+	if i == nil {
+		return nil
 	}
-	i.exiting = true
-	return true
+	return i.exiting
 }
 
-// markIdle marks an agent as idle.
-func (i *idleMonitor) markIdle(agent *AgentWorker) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-	// Allow MarkIdle to be called multiple times without updating the idleAt
-	// timestamp.
-	if _, alreadyIdle := i.idleAt[agent]; alreadyIdle {
+// MarkIdle marks an agent as idle. It is safe to use with a nil pointer.
+func (i *idleMonitor) MarkIdle(agent *AgentWorker) {
+	if i == nil {
 		return
 	}
-	i.idleAt[agent] = time.Now()
+	select {
+	case i.becameIdle <- agent:
+		// marked as idle
+	case <-i.exiting:
+		// no goroutine listening on i.becameIdle
+	}
 }
 
-// markDead marks an agent as dead.
-func (i *idleMonitor) markDead(agent *AgentWorker) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-	i.idleAt[agent] = time.Time{}
+// MarkDead marks an agent as dead. It is safe to use with a nil pointer.
+func (i *idleMonitor) MarkDead(agent *AgentWorker) {
+	if i == nil {
+		return
+	}
+	select {
+	case i.becameDead <- agent:
+		// marked as dead
+	case <-i.exiting:
+		// no goroutine listening on i.becameDead
+	}
 }
 
-// markBusy marks an agent as busy.
-func (i *idleMonitor) markBusy(agent *AgentWorker) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-	delete(i.idleAt, agent)
+// MarkBusy marks an agent as busy. It is safe to use with a nil pointer.
+func (i *idleMonitor) MarkBusy(agent *AgentWorker) {
+	if i == nil {
+		return
+	}
+	select {
+	case i.becameBusy <- agent:
+		// marked as busy
+	case <-i.exiting:
+		// no goroutine listening on i.becameBusy
+	}
 }
diff --git a/agent/idle_monitor_test.go b/agent/idle_monitor_test.go
new file mode 100644
index 0000000000..db340e98b7
--- /dev/null
+++ b/agent/idle_monitor_test.go
@@ -0,0 +1,91 @@
+package agent
+
+import (
+	"testing"
+	"time"
+)
+
+func TestIdleMonitor(t *testing.T) {
+	t.Parallel()
+
+	idleTimeout := 100 * time.Millisecond
+	i := NewIdleMonitor(t.Context(), 3, idleTimeout)
+
+	// These "agents" don't actually run, they're just 3 different pointers.
+	agents := []*AgentWorker{
+		new(AgentWorker), new(AgentWorker), new(AgentWorker),
+	}
+
+	i.MarkBusy(agents[0])
+	i.MarkIdle(agents[1])
+	i.MarkDead(agents[2])
+
+	// The idle monitor should start exiting within 1 second of the agents all
+	// being idle or dead.
+	start := time.Now()
+	i.MarkIdle(agents[0])
+	select {
+	case <-i.Exiting():
+		// This case should win, but only after the timeout.
+		if exitedAfter := time.Since(start); exitedAfter < idleTimeout {
+			t.Errorf("exitedAfter = %v, want > %v", exitedAfter, idleTimeout)
+		}
+
+	case <-time.After(2 * idleTimeout):
+		// TODO: use testing/synctest when that becomes available
+		t.Error("timed out waiting on <-i.Exiting()")
+	}
+}
+
+func TestIdleMonitor_AllDead(t *testing.T) {
+	t.Parallel()
+
+	idleTimeout := 100 * time.Millisecond
+	i := NewIdleMonitor(t.Context(), 3, idleTimeout)
+
+	agents := []*AgentWorker{
+		new(AgentWorker), new(AgentWorker), new(AgentWorker),
+	}
+
+	// All agents dead should result in exiting instantly.
+	i.MarkDead(agents[0])
+	i.MarkDead(agents[1])
+
+	start := time.Now()
+	i.MarkDead(agents[2])
+
+	select {
+	case <-i.Exiting():
+		// This case should win, quickly.
+		if exitedAfter := time.Since(start); exitedAfter > idleTimeout {
+			t.Errorf("exitedAfter = %v, want < %v", exitedAfter, idleTimeout)
+		}
+	case <-time.After(idleTimeout):
+		// TODO: use testing/synctest when that becomes available
+		t.Error("timed out waiting on <-i.Exiting()")
+	}
+}
+
+func TestIdleMonitor_Busy(t *testing.T) {
+	t.Parallel()
+
+	idleTimeout := 100 * time.Millisecond
+	i := NewIdleMonitor(t.Context(), 3, idleTimeout)
+
+	agents := []*AgentWorker{
+		new(AgentWorker), new(AgentWorker), new(AgentWorker),
+	}
+
+	// Any agent still busy should not cause an exit.
+	i.MarkDead(agents[0])
+	i.MarkDead(agents[1])
+	i.MarkBusy(agents[2])
+
+	select {
+	case <-i.Exiting():
+		t.Error("<-i.Exiting() happened while at least one agent was still busy")
+
+	case <-time.After(2 * idleTimeout):
+		// This case should win.
+	}
+}
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 0ebc1b640c..bb3ff2d0f6 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1322,7 +1322,7 @@ var AgentStartCommand = cli.Command{
 		}
 
 		// Setup the agent pool that spawns agent workers
-		pool := agent.NewAgentPool(workers)
+		pool := agent.NewAgentPool(workers, &agentConf)
 
 		// Agent-wide shutdown hook. Once per agent, for all workers on the agent.
 		defer agentShutdownHook(l, cfg)

From 25223a7120504c61197a3559b58255994feb69f0 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Feb 2026 17:37:19 +1100
Subject: [PATCH 208/242] Split action handling into separate loop

---
 agent/agent_worker.go | 273 +++++++++++++++++++++++++++++++-----------
 1 file changed, 204 insertions(+), 69 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 14f60b9c0d..75d78bc4cf 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -276,16 +276,42 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		}
 	}
 
-	// Start the ping loop and block until it has stopped.
-	errCh <- a.runPingLoop(ctx, idleMon)
+	// Channels to enable communication between the ping loop and action loop
+	fromPingLoopCh := make(chan actionMessage) // ping loop to action handler
 
-	// The ping loop has ended, so stop the heartbeat loop.
+	// Start the loops and block until they have all stopped.
+	// Based on configuration, we have our choice of ping loop,
+	// streaming loop+debouncer loop, or both.
+	var wg sync.WaitGroup
+
+	pingLoop := func() {
+		defer wg.Done()
+		errCh <- a.runPingLoop(ctx, fromPingLoopCh)
+	}
+	actionLoop := func() {
+		defer wg.Done()
+		errCh <- a.runActionLoop(ctx, idleMon, fromPingLoopCh)
+	}
+
+	loops := []func(){pingLoop, actionLoop}
+
+	wg.Add(len(loops))
+	for _, l := range loops {
+		go l()
+	}
+	wg.Wait()
+
+	// The source loops have ended, so stop the heartbeat loop.
 	stopHeartbeats()
 
-	// Block until both loops have returned, then join the errors.
+	// Block until all loops have returned, then join the errors.
 	// (Note that errors.Join does the right thing with nil.)
-	// Both loops are context aware, so no need to wait on ctx here.
-	return errors.Join(<-errCh, <-errCh)
+	// All loops are context aware, so no need to wait on ctx here.
+	var err error
+	for range len(loops) + 1 { // loops + heartbeat loop
+		err = errors.Join(err, <-errCh)
+	}
+	return err
 }
 
 func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) error {
@@ -335,7 +361,11 @@ func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) error {
 // runPingLoop runs the loop that pings Buildkite for work. It does all critical
 // agent things.
 // The lifetime of an agent is the lifetime of the ping loop.
-func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) error {
+func (a *AgentWorker) runPingLoop(ctx context.Context, outCh chan<- actionMessage) error {
+	// When this loop returns, close the channel to let the action handler loop
+	// stop listening for actions from it.
+	defer close(outCh)
+
 	ctx, setStat, _ := status.AddSimpleItem(ctx, "Ping loop")
 	defer setStat("🛑 Ping loop stopped!")
 	setStat("🏃 Starting...")
@@ -360,9 +390,6 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 
 	a.logger.Info("Waiting for instructions...")
 
-	ranJob := false
-	wasPaused := false
-
 	// Continue this loop until one of:
 	// * the context is cancelled
 	// * the stop channel is closed (a.Stop)
@@ -422,10 +449,144 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 		}
 		pingDurations.Observe(time.Since(startPing).Seconds())
 
-		pingActions.WithLabelValues(action).Inc()
-		switch action {
+		// Send the action to the action loop
+		errCh := make(chan error)
+		msg := actionMessage{
+			action: action,
+			jobID:  jobID,
+			errCh:  errCh,
+		}
+		select {
+		case outCh <- msg:
+			// sent!
+		case <-a.stop:
+			a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+			return ctx.Err()
+		}
+
+		// Wait for completion
+		select {
+		case err := <-errCh:
+			if err != nil || jobID == "" {
+				break
+			}
+			// A job ran (or was at least started) successfully.
+			// Observation: jobs are rarely the last within a pipeline,
+			// thus if this worker just completed a job,
+			// there is likely another immediately available.
+			// Skip waiting for the ping interval until
+			// a ping without a job has occurred,
+			// but in exchange, ensure the next ping must wait at least a full
+			// pingInterval to avoid too much server load.
+			pingTicker.Reset(pingInterval)
+			select {
+			case skipTicker <- struct{}{}:
+				// Ticker will be skipped
+			default:
+				// We're already skipping the ticker, don't block.
+			}
+		case <-a.stop:
+			a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+			return ctx.Err()
+		}
+	}
+}
+
+type actionMessage struct {
+	// Details of the action to execute
+	action, jobID string
+
+	// Results of the action
+	errCh chan<- error
+}
+
+func (a *AgentWorker) runActionLoop(ctx context.Context, idleMon *idleMonitor, fromPingLoop <-chan actionMessage) error {
+	// Once this loop terminates, there's no point continuing the others,
+	// because nothing remains to execute their actions.
+	defer a.internalStop()
+
+	ctx, setStat, _ := status.AddSimpleItem(ctx, "Action loop")
+	defer setStat("🛑 Action loop stopped!")
+	setStat("🏃 Starting...")
+	a.logger.Debug("[runActionLoop] Starting")
+	defer a.logger.Debug("[runActionLoop] Exiting")
+
+	// Start timing disconnect-after-uptime, if configured.
+	var disconnectAfterUptime <-chan time.Time
+	maxUptime := a.agentConfiguration.DisconnectAfterUptime
+	if maxUptime > 0 {
+		disconnectAfterUptime = time.After(maxUptime)
+	}
+
+	exitWhenNotPaused := false // the next time the action isn't "pause", exit
+	ranJob := false
+	paused := false
+
+	for {
+		// Wait for one of the following:
+		// - an action
+		// - the context to be cancelled
+		// - the agent is stopping (a.stop)
+		// - the idle monitor has declared we're all exiting
+		//   (if DisconnectAfterIdleTimeout is configured & we're not paused)
+		// - disconnect after uptime
+		//   (if DisconnectAfterUptime is configured & we're not paused)
+		a.logger.Debug("[runActionLoop] Waiting for an action...")
+		setStat("⌚️ Waiting for an action...")
+		var msg actionMessage
+		select {
+		case m, open := <-fromPingLoop:
+			if !open {
+				// The ping loop has ended, so exit.
+				return nil
+			}
+			a.logger.Debug("[runActionLoop] Got action %q from ping loop", m.action)
+			msg = m
+			// continue below
+
+		case <-ctx.Done():
+			a.logger.Debug("[runActionLoop] Stopping due to context cancel")
+			return ctx.Err()
+
+		case <-a.stop:
+			a.logger.Debug("[runActionLoop] Stopping due to agent stop")
+			return nil
+
+		case <-disconnectAfterUptime:
+			a.logger.Info("Agent has exceeded max uptime of %v", maxUptime)
+			if paused {
+				// Wait to be unpaused before exiting
+				a.logger.Info("Awaiting resume before disconnecting...")
+				exitWhenNotPaused = true
+				continue
+			}
+			a.logger.Info("Disconnecting...")
+			return nil
+
+		case <-idleMon.Exiting():
+			// This should only happen if the agent isn't paused.
+			// (Pausedness is a kind of non-idleness.)
+			a.logger.Info("All agents have been idle for at least %v. Disconnecting...", idleMon.idleTimeout)
+			return nil
+		}
+
+		// Let's handle the action!
+		a.logger.Debug("[runActionLoop] Performing %q action", msg.action)
+		setStat(fmt.Sprintf("🧑‍🍳 Performing %q action...", msg.action))
+		pingActions.WithLabelValues(msg.action).Inc()
+
+		// In cases where we need to disconnect, *don't* send on msg.errCh,
+		// in order to force the <-a.stop branch in the other loops.
+		// Otherwise, be sure to `close(msg.errCh)`!
+		switch msg.action {
 		case "disconnect":
-			a.StopUngracefully()
+			a.logger.Debug("[runActionLoop] Stopping action loop due to disconnect action")
 			return nil
 
 		case "pause":
@@ -433,95 +594,69 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, idleMon *idleMonitor) err
 			// paused agent is expected to remain alive and pinging for
 			// instructions.
 			// *This includes acquire-job and disconnect-after-idle-timeout.*
-			wasPaused = true
+			a.logger.Debug("[runActionLoop] Entering pause state")
+			paused = true
+			// For the purposes of deciding whether or not to exit,
+			// pausedness is a kind of non-idleness.
+			// If there's also no job, agent is marked as idle below.
+			idleMon.MarkBusy(a)
+			close(msg.errCh)
 			continue
 		}
 
 		// At this point, action was neither "disconnect" nor "pause".
-		if wasPaused {
+		if exitWhenNotPaused {
+			a.logger.Debug("[runActionLoop] Stopping action loop because exitWhenNotPaused is true")
+			return nil
+		}
+		if paused {
+			// We're not paused any more! Log a helpful message.
 			a.logger.Info("Agent has resumed after being paused")
-			wasPaused = false
+			paused = false
 		}
 
-		// Exit after acquire-job.
 		// For acquire-job agents, registration sets ignore-in-dispatches=true,
-		// so job should be nil. If not nil, complain.
+		// so jobID should be empty. If not, complain.
 		if a.agentConfiguration.AcquireJob != "" {
-			if jobID != "" {
-				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in acquire-job mode!", jobID)
+			if msg.jobID != "" {
+				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in acquire-job mode! Ignoring the new job", msg.jobID)
 			}
+			// Disconnect after acquire-job.
 			return nil
 		}
 
-		// Exit after disconnect-after-job. Finishing the job sets
-		// ignore-in-dispatches=true, so job should be nil. If not, complain.
+		// In disconnect-after-job mode, finishing the job sets
+		// ignore-in-dispatches=true. So jobID should be empty. If not, complain.
 		if ranJob && a.agentConfiguration.DisconnectAfterJob {
-			if jobID != "" {
-				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in disconnect-after-job mode (and already ran a job)!", jobID)
+			if msg.jobID != "" {
+				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in disconnect-after-job mode (and already ran a job)! Ignoring the new job", msg.jobID)
 			}
 			a.logger.Info("Job ran, and disconnect-after-job is enabled. Disconnecting...")
 			return nil
 		}
 
-		// Exit after disconnect-after-uptime is exceeded.
-		if maxUptime := a.agentConfiguration.DisconnectAfterUptime; maxUptime > 0 {
-			if time.Since(a.startTime) >= maxUptime {
-				if jobID != "" {
-					a.logger.Error("Agent ping dispatched a job (id %q) but agent has exceeded max uptime of %v!", jobID, maxUptime)
-				}
-				a.logger.Info("Agent has exceeded max uptime of %v. Disconnecting...", maxUptime)
-				return nil
-			}
-		}
-
-		// Note that Ping only returns a job if err == nil.
-		if jobID == "" {
-			idleTimeout := a.agentConfiguration.DisconnectAfterIdleTimeout
-			if idleTimeout == 0 {
-				// No job and no idle timeout.
-				continue
-			}
-
+		// If the jobID is empty, then it's an idle message
+		if msg.jobID == "" {
 			// This ensures agents that never receive a job are still tracked
 			// by the idle monitor and can properly trigger disconnect-after-idle-timeout.
 			idleMon.MarkIdle(a)
-
-			// Exit if every agent has been idle for at least the timeout.
-			select {
-			case <-idleMon.Exiting():
-				a.logger.Info("All agents have been idle for at least %v. Disconnecting...", idleTimeout)
-				return nil
-			default:
-				// Not idle enough to exit. Wait and ping again.
-				continue
-			}
+			close(msg.errCh)
+			continue
 		}
 
 		setStat("💼 Accepting job")
 
 		// Runs the job, only errors if something goes wrong
-		if err := a.AcceptAndRunJob(ctx, jobID, idleMon); err != nil {
+		if err := a.AcceptAndRunJob(ctx, msg.jobID, idleMon); err != nil {
 			a.logger.Error("%v", err)
 			setStat(fmt.Sprintf("✅ Finished job with error: %v", err))
+			msg.errCh <- err // so the ping loop can do something special
+			close(msg.errCh)
 			continue
 		}
 
 		ranJob = true
-
-		// Observation: jobs are rarely the last within a pipeline,
-		// thus if this worker just completed a job,
-		// there is likely another immediately available.
-		// Skip waiting for the ping interval until
-		// a ping without a job has occurred,
-		// but in exchange, ensure the next ping must wait at least a full
-		// pingInterval to avoid too much server load.
-		pingTicker.Reset(pingInterval)
-		select {
-		case skipTicker <- struct{}{}:
-			// Ticker will be skipped
-		default:
-			// We're already skipping the ticker, don't block.
-		}
+		close(msg.errCh)
 	}
 }
 

From c3d803e8b2f173700e181c71ff76a33087175a1a Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 2 Mar 2026 10:55:02 +1100
Subject: [PATCH 209/242] Add baton

Co-authored-by: Ming Guo <ming@buildkite.com>
---
 agent/baton.go | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 agent/baton.go

diff --git a/agent/baton.go b/agent/baton.go
new file mode 100644
index 0000000000..2eed183de5
--- /dev/null
+++ b/agent/baton.go
@@ -0,0 +1,72 @@
+package agent
+
+import "sync"
+
+// baton is a channel-based mutex. This allows for using it as part of a select
+// statement.
+type baton struct {
+	mu      sync.Mutex
+	holder  string
+	acquire map[string]chan struct{}
+}
+
+// newBaton creates a new baton for sharing among (non-zero) values of T.
+func newBaton() *baton {
+	return &baton{
+		acquire: make(map[string]chan struct{}),
+	}
+}
+
+// HeldBy reports if the argument holds the baton.
+func (b *baton) HeldBy(by string) bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.holder == by
+}
+
+// Acquire returns a channel that receives when the baton is acquired by the
+// acquirer.
+func (b *baton) Acquire(by string) <-chan struct{} {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	ch := b.acquire[by]
+	if ch == nil {
+		ch = make(chan struct{})
+	}
+
+	if b.holder == "" {
+		b.holder = by
+		close(ch)
+		delete(b.acquire, by) // in case it is in the map
+		return ch
+	}
+
+	b.acquire[by] = ch
+	return ch
+}
+
+// Release releases the baton, if it is held by the argument.
+func (b *baton) Release(by string) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if b.holder != by {
+		return
+	}
+
+	// Attempt to pass the baton.
+	for a, ch := range b.acquire {
+		delete(b.acquire, a)
+		select {
+		case ch <- struct{}{}:
+			// This acquirer has acquired the baton.
+			b.holder = a
+			return
+		default:
+			// This acquirer has stopped waiting, try another.
+		}
+	}
+	// Nothing was waiting, nothing now holds the baton.
+	b.holder = ""
+}

From a730026f6f8464970a879401847a51802c6f1345 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Feb 2026 17:37:19 +1100
Subject: [PATCH 210/242] Implement streaming loop with ping loop fallback

---
 agent/agent_worker.go           | 563 +++++---------------------------
 agent/agent_worker_action.go    | 229 +++++++++++++
 agent/agent_worker_debouncer.go | 144 ++++++++
 agent/agent_worker_heartbeat.go |  52 +++
 agent/agent_worker_ping.go      | 255 +++++++++++++++
 agent/agent_worker_streaming.go | 201 ++++++++++++
 agent/agent_worker_test.go      |  11 +-
 7 files changed, 967 insertions(+), 488 deletions(-)
 create mode 100644 agent/agent_worker_action.go
 create mode 100644 agent/agent_worker_debouncer.go
 create mode 100644 agent/agent_worker_heartbeat.go
 create mode 100644 agent/agent_worker_ping.go
 create mode 100644 agent/agent_worker_streaming.go

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 75d78bc4cf..1cfbc60fd2 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -5,15 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math/rand/v2"
 	"net/http"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"connectrpc.com/connect"
 	"github.com/buildkite/agent/v3/api"
 	"github.com/buildkite/agent/v3/core"
-	"github.com/buildkite/agent/v3/internal/ptr"
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/metrics"
 	"github.com/buildkite/agent/v3/process"
@@ -163,9 +162,22 @@ func (e *errUnrecoverable) Error() string {
 	return fmt.Sprintf("%s failed with unrecoverable status: %s, mesage: %q", e.action, status, e.err)
 }
 
+// See https://connectrpc.com/docs/protocol/#http-to-error-code
+var codeUnrecoverable = map[connect.Code]bool{
+	connect.CodeInternal:         true, // 400
+	connect.CodeUnauthenticated:  true, // 401
+	connect.CodePermissionDenied: true, // 403
+	connect.CodeUnimplemented:    true, // 404
+	// All other codes are implicitly false, but particularly:
+	// Unavailable (429, 502, 503, 504) and Unknown (all other HTTP statuses).
+}
+
 func isUnrecoverable(err error) bool {
 	var u *errUnrecoverable
-	return errors.As(err, &u)
+	if errors.As(err, &u) {
+		return true
+	}
+	return codeUnrecoverable[connect.CodeOf(err)]
 }
 
 func (e *errUnrecoverable) Unwrap() error {
@@ -216,7 +228,7 @@ func (a *AgentWorker) statusCallback(context.Context) (any, error) {
 	}, nil
 }
 
-// Starts the agent worker
+// Start starts the agent worker.
 func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr error) {
 	// Record the start time for max agent lifetime tracking
 	a.startTime = time.Now()
@@ -233,14 +245,14 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	}
 	defer a.metricsCollector.Stop() //nolint:errcheck // Best-effort cleanup
 
-	// errCh receives 1 value from the heartbeat loop, and 1 from the ping loop.
-	errCh := make(chan error, 2)
+	// There are as many as 4 different loops that send 1 error here each.
+	errCh := make(chan error, 4)
 
 	// Use this context to control the heartbeat loop.
 	heartbeatCtx, stopHeartbeats := context.WithCancel(ctx)
 	defer stopHeartbeats()
 
-	// Start the heartbeat loop but don't wait for it to return.
+	// Start the heartbeat loop but don't wait for it to return (yet).
 	go func() {
 		errCh <- a.runHeartbeatLoop(heartbeatCtx)
 	}()
@@ -250,12 +262,6 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	// (Why run a ping loop at all? To find out if the agent is paused, which
 	// affects whether it terminates after the job.)
 	if a.agentConfiguration.AcquireJob != "" {
-		// When in acquisition mode, there can't be any agents, so
-		// there's really no point in letting the idle monitor know
-		// we're busy, but it's probably a good thing to do for good
-		// measure.
-		idleMon.MarkBusy(a)
-
 		if err := a.AcquireAndRunJob(ctx, a.agentConfiguration.AcquireJob); err != nil {
 			// If the job acquisition was rejected, we can exit with an error
 			// so that supervisor knows that the job was not acquired due to the job being rejected.
@@ -276,8 +282,15 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		}
 	}
 
-	// Channels to enable communication between the ping loop and action loop
-	fromPingLoopCh := make(chan actionMessage) // ping loop to action handler
+	// The baton avoids the ping loop pinging Buildkite when the streaming loop
+	// is healthy, but allows the ping loop to take over from the streaming loop
+	// quickly when it becomes unhealthy.
+	bat := newBaton()
+
+	// More channels to enable communication between the various loops.
+	fromPingLoopCh := make(chan actionMessage)      // ping loop to action handler
+	fromStreamingLoopCh := make(chan actionMessage) // streaming loop to debouncer
+	fromDebouncerCh := make(chan actionMessage)     // debouncer to action handler
 
 	// Start the loops and block until they have all stopped.
 	// Based on configuration, we have our choice of ping loop,
@@ -286,15 +299,52 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 
 	pingLoop := func() {
 		defer wg.Done()
-		errCh <- a.runPingLoop(ctx, fromPingLoopCh)
+		errCh <- a.runPingLoop(ctx, bat, fromPingLoopCh)
 	}
-	actionLoop := func() {
+	streamingLoop := func() {
+		defer wg.Done()
+		err := a.runStreamingPingLoop(ctx, fromStreamingLoopCh)
+		if a.agentConfiguration.PingMode != "streaming-only" {
+			// If the ping mode is streaming-only, then an unrecoverable failure
+			// in the streaming loop should be reported.
+			// Otherwise, it should fall back to the ping loop and carry on,
+			// and if that also has an unrecoverable failure we can report that.
+			// Said another way:
+			// Streaming is best-effort but preferred, unless in streaming-only
+			// mode (it's the only available option).
+			err = nil
+		}
+		errCh <- err
+	}
+	debouncerLoop := func() {
 		defer wg.Done()
-		errCh <- a.runActionLoop(ctx, idleMon, fromPingLoopCh)
+		errCh <- a.runDebouncer(ctx, bat, fromDebouncerCh, fromStreamingLoopCh)
+	}
+
+	var loops []func()
+	switch a.agentConfiguration.PingMode {
+	case "", "auto":
+		loops = []func(){pingLoop, streamingLoop, debouncerLoop}
+		bat.Acquire("debouncer")
+
+	case "ping-only":
+		loops = []func(){pingLoop}
+		fromDebouncerCh = nil // prevent action loop listening to streaming side
+
+	case "stream-only":
+		loops = []func(){streamingLoop, debouncerLoop}
+		fromPingLoopCh = nil // prevent action loop listening to ping side
+		bat.Acquire("debouncer")
 	}
 
-	loops := []func(){pingLoop, actionLoop}
+	// There's always an action handler.
+	actionLoop := func() {
+		defer wg.Done()
+		errCh <- a.runActionLoop(ctx, idleMon, fromPingLoopCh, fromDebouncerCh)
+	}
+	loops = append(loops, actionLoop)
 
+	// Go loops!
 	wg.Add(len(loops))
 	for _, l := range loops {
 		go l()
@@ -314,352 +364,6 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	return err
 }
 
-func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) error {
-	ctx, setStat, _ := status.AddSimpleItem(ctx, "Heartbeat loop")
-	defer setStat("💔 Heartbeat loop stopped!")
-	setStat("🏃 Starting...")
-
-	heartbeatInterval := time.Second * time.Duration(a.agent.HeartbeatInterval)
-	heartbeatTicker := time.NewTicker(heartbeatInterval)
-	defer heartbeatTicker.Stop()
-	for {
-		setStat("😴 Sleeping for a bit")
-		select {
-		case <-heartbeatTicker.C:
-			setStat("❤️ Sending heartbeat")
-			if err := a.Heartbeat(ctx); err != nil {
-				if isUnrecoverable(err) {
-					a.logger.Error("%s", err)
-					// unrecoverable heartbeat failure also stops ping loop
-					a.StopUngracefully()
-					return err
-				}
-
-				// Get the last heartbeat time to the nearest microsecond
-				a.stats.Lock()
-				if a.stats.lastHeartbeat.IsZero() {
-					a.logger.Error("Failed to heartbeat %s. Will try again in %v. (No heartbeat yet)",
-						err, heartbeatInterval)
-				} else {
-					a.logger.Error("Failed to heartbeat %s. Will try again in %v. (Last successful was %v ago)",
-						err, heartbeatInterval, time.Since(a.stats.lastHeartbeat))
-				}
-				a.stats.Unlock()
-			}
-
-		case <-ctx.Done():
-			a.logger.Debug("Stopping heartbeats due to context cancel")
-			// An alternative to returning nil would be ctx.Err(), but we use
-			// the context for ordinary termination of this loop.
-			// A context cancellation from outside the agent worker would still
-			// be reflected in the value returned by the ping loop return.
-			return nil
-		}
-	}
-}
-
-// runPingLoop runs the loop that pings Buildkite for work. It does all critical
-// agent things.
-// The lifetime of an agent is the lifetime of the ping loop.
-func (a *AgentWorker) runPingLoop(ctx context.Context, outCh chan<- actionMessage) error {
-	// When this loop returns, close the channel to let the action handler loop
-	// stop listening for actions from it.
-	defer close(outCh)
-
-	ctx, setStat, _ := status.AddSimpleItem(ctx, "Ping loop")
-	defer setStat("🛑 Ping loop stopped!")
-	setStat("🏃 Starting...")
-
-	// Create the ticker
-	pingInterval := time.Second * time.Duration(a.agent.PingInterval)
-	pingTicker := time.NewTicker(pingInterval)
-	defer pingTicker.Stop()
-
-	// testTriggerCh will normally block forever, and so will not affect the for/select loop.
-	var testTriggerCh chan struct{}
-	if a.noWaitBetweenPingsForTesting {
-		// a closed channel will unblock the for/select instantly, for zero-delay ping loop testing.
-		testTriggerCh = make(chan struct{})
-		close(testTriggerCh)
-	}
-
-	// On the first iteration, skip waiting for the pingTicker.
-	// This doesn't skip the jitter, though.
-	skipTicker := make(chan struct{}, 1)
-	skipTicker <- struct{}{}
-
-	a.logger.Info("Waiting for instructions...")
-
-	// Continue this loop until one of:
-	// * the context is cancelled
-	// * the stop channel is closed (a.Stop)
-	// * the agent is in acquire mode and the ping action isn't "pause"
-	// * the agent is in disconnect-after-job mode, the job is finished, and the
-	//   ping action isn't "pause",
-	// * the agent is in disconnect-after-idle-timeout mode, has been idle for
-	//   longer than the idle timeout, and the ping action isn't "pause".
-	// * the agent has exceeded its disconnect-after-uptime and the ping action isn't "pause".
-	for {
-		startWait := time.Now()
-		setStat("😴 Waiting until next ping interval tick")
-		select {
-		case <-testTriggerCh:
-			// instant receive from closed chan when noWaitBetweenPingsForTesting is true
-		case <-skipTicker:
-			// continue below
-		case <-pingTicker.C:
-			// continue below
-		case <-a.stop:
-			a.logger.Debug("Stopping pings due to agent stop")
-			return nil
-		case <-ctx.Done():
-			a.logger.Debug("Stopping pings due to context cancel")
-			return ctx.Err()
-		}
-
-		// Within the interval, wait a random amount of time to avoid
-		// spontaneous synchronisation across agents.
-		jitter := rand.N(pingInterval)
-		setStat(fmt.Sprintf("🫨 Jittering for %v", jitter))
-		select {
-		case <-testTriggerCh:
-			// instant receive from closed chan when noWaitBetweenPingsForTesting is true
-		case <-time.After(jitter):
-			// continue below
-		case <-a.stop:
-			a.logger.Debug("Stopping pings due to agent stop")
-			return nil
-		case <-ctx.Done():
-			a.logger.Debug("Stopping pings due to context cancel")
-			return ctx.Err()
-		}
-		pingWaitDurations.Observe(time.Since(startWait).Seconds())
-
-		setStat("📡 Pinging Buildkite for instructions")
-		pingsSent.Inc()
-		startPing := time.Now()
-		jobID, action, err := a.Ping(ctx)
-		if err != nil {
-			pingErrors.Inc()
-			if isUnrecoverable(err) {
-				a.logger.Error("%v", err)
-				return err
-			}
-			a.logger.Warn("%v", err)
-		}
-		pingDurations.Observe(time.Since(startPing).Seconds())
-
-		// Send the action to the action loop
-		errCh := make(chan error)
-		msg := actionMessage{
-			action: action,
-			jobID:  jobID,
-			errCh:  errCh,
-		}
-		select {
-		case outCh <- msg:
-			// sent!
-		case <-a.stop:
-			a.logger.Debug("[runPingLoop] Stopping due to agent stop")
-			return nil
-		case <-ctx.Done():
-			a.logger.Debug("[runPingLoop] Stopping due to context cancel")
-			return ctx.Err()
-		}
-
-		// Wait for completion
-		select {
-		case err := <-errCh:
-			if err != nil || jobID == "" {
-				break
-			}
-			// A job ran (or was at least started) successfully.
-			// Observation: jobs are rarely the last within a pipeline,
-			// thus if this worker just completed a job,
-			// there is likely another immediately available.
-			// Skip waiting for the ping interval until
-			// a ping without a job has occurred,
-			// but in exchange, ensure the next ping must wait at least a full
-			// pingInterval to avoid too much server load.
-			pingTicker.Reset(pingInterval)
-			select {
-			case skipTicker <- struct{}{}:
-				// Ticker will be skipped
-			default:
-				// We're already skipping the ticker, don't block.
-			}
-		case <-a.stop:
-			a.logger.Debug("[runPingLoop] Stopping due to agent stop")
-			return nil
-		case <-ctx.Done():
-			a.logger.Debug("[runPingLoop] Stopping due to context cancel")
-			return ctx.Err()
-		}
-	}
-}
-
-type actionMessage struct {
-	// Details of the action to execute
-	action, jobID string
-
-	// Results of the action
-	errCh chan<- error
-}
-
-func (a *AgentWorker) runActionLoop(ctx context.Context, idleMon *idleMonitor, fromPingLoop <-chan actionMessage) error {
-	// Once this loop terminates, there's no point continuing the others,
-	// because nothing remains to execute their actions.
-	defer a.internalStop()
-
-	ctx, setStat, _ := status.AddSimpleItem(ctx, "Action loop")
-	defer setStat("🛑 Action loop stopped!")
-	setStat("🏃 Starting...")
-	a.logger.Debug("[runActionLoop] Starting")
-	defer a.logger.Debug("[runActionLoop] Exiting")
-
-	// Start timing disconnect-after-uptime, if configured.
-	var disconnectAfterUptime <-chan time.Time
-	maxUptime := a.agentConfiguration.DisconnectAfterUptime
-	if maxUptime > 0 {
-		disconnectAfterUptime = time.After(maxUptime)
-	}
-
-	exitWhenNotPaused := false // the next time the action isn't "pause", exit
-	ranJob := false
-	paused := false
-
-	for {
-		// Wait for one of the following:
-		// - an action
-		// - the context to be cancelled
-		// - the agent is stopping (a.stop)
-		// - the idle monitor has declared we're all exiting
-		//   (if DisconnectAfterIdleTimeout is configured & we're not paused)
-		// - disconnect after uptime
-		//   (if DisconnectAfterUptime is configured & we're not paused)
-		a.logger.Debug("[runActionLoop] Waiting for an action...")
-		setStat("⌚️ Waiting for an action...")
-		var msg actionMessage
-		select {
-		case m, open := <-fromPingLoop:
-			if !open {
-				// The ping loop has ended, so exit.
-				return nil
-			}
-			a.logger.Debug("[runActionLoop] Got action %q from ping loop", m.action)
-			msg = m
-			// continue below
-
-		case <-ctx.Done():
-			a.logger.Debug("[runActionLoop] Stopping due to context cancel")
-			return ctx.Err()
-
-		case <-a.stop:
-			a.logger.Debug("[runActionLoop] Stopping due to agent stop")
-			return nil
-
-		case <-disconnectAfterUptime:
-			a.logger.Info("Agent has exceeded max uptime of %v", maxUptime)
-			if paused {
-				// Wait to be unpaused before exiting
-				a.logger.Info("Awaiting resume before disconnecting...")
-				exitWhenNotPaused = true
-				continue
-			}
-			a.logger.Info("Disconnecting...")
-			return nil
-
-		case <-idleMon.Exiting():
-			// This should only happen if the agent isn't paused.
-			// (Pausedness is a kind of non-idleness.)
-			a.logger.Info("All agents have been idle for at least %v. Disconnecting...", idleMon.idleTimeout)
-			return nil
-		}
-
-		// Let's handle the action!
-		a.logger.Debug("[runActionLoop] Performing %q action", msg.action)
-		setStat(fmt.Sprintf("🧑‍🍳 Performing %q action...", msg.action))
-		pingActions.WithLabelValues(msg.action).Inc()
-
-		// In cases where we need to disconnect, *don't* send on msg.errCh,
-		// in order to force the <-a.stop branch in the other loops.
-		// Otherwise, be sure to `close(msg.errCh)`!
-		switch msg.action {
-		case "disconnect":
-			a.logger.Debug("[runActionLoop] Stopping action loop due to disconnect action")
-			return nil
-
-		case "pause":
-			// An agent is not dispatched any jobs while it is paused, but the
-			// paused agent is expected to remain alive and pinging for
-			// instructions.
-			// *This includes acquire-job and disconnect-after-idle-timeout.*
-			a.logger.Debug("[runActionLoop] Entering pause state")
-			paused = true
-			// For the purposes of deciding whether or not to exit,
-			// pausedness is a kind of non-idleness.
-			// If there's also no job, agent is marked as idle below.
-			idleMon.MarkBusy(a)
-			close(msg.errCh)
-			continue
-		}
-
-		// At this point, action was neither "disconnect" nor "pause".
-		if exitWhenNotPaused {
-			a.logger.Debug("[runActionLoop] Stopping action loop because exitWhenNotPaused is true")
-			return nil
-		}
-		if paused {
-			// We're not paused any more! Log a helpful message.
-			a.logger.Info("Agent has resumed after being paused")
-			paused = false
-		}
-
-		// For acquire-job agents, registration sets ignore-in-dispatches=true,
-		// so jobID should be empty. If not, complain.
-		if a.agentConfiguration.AcquireJob != "" {
-			if msg.jobID != "" {
-				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in acquire-job mode! Ignoring the new job", msg.jobID)
-			}
-			// Disconnect after acquire-job.
-			return nil
-		}
-
-		// In disconnect-after-job mode, finishing the job sets
-		// ignore-in-dispatches=true. So jobID should be empty. If not, complain.
-		if ranJob && a.agentConfiguration.DisconnectAfterJob {
-			if msg.jobID != "" {
-				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in disconnect-after-job mode (and already ran a job)! Ignoring the new job", msg.jobID)
-			}
-			a.logger.Info("Job ran, and disconnect-after-job is enabled. Disconnecting...")
-			return nil
-		}
-
-		// If the jobID is empty, then it's an idle message
-		if msg.jobID == "" {
-			// This ensures agents that never receive a job are still tracked
-			// by the idle monitor and can properly trigger disconnect-after-idle-timeout.
-			idleMon.MarkIdle(a)
-			close(msg.errCh)
-			continue
-		}
-
-		setStat("💼 Accepting job")
-
-		// Runs the job, only errors if something goes wrong
-		if err := a.AcceptAndRunJob(ctx, msg.jobID, idleMon); err != nil {
-			a.logger.Error("%v", err)
-			setStat(fmt.Sprintf("✅ Finished job with error: %v", err))
-			msg.errCh <- err // so the ping loop can do something special
-			close(msg.errCh)
-			continue
-		}
-
-		ranJob = true
-		close(msg.errCh)
-	}
-}
-
 func (a *AgentWorker) internalStop() {
 	a.stopOnce.Do(func() {
 		// Use the closure of the stop channel as a signal to the main run
@@ -755,76 +459,6 @@ func (a *AgentWorker) Heartbeat(ctx context.Context) error {
 	return nil
 }
 
-// Performs a ping that checks Buildkite for a job or action to take
-// Returns a job, or nil if none is found
-func (a *AgentWorker) Ping(ctx context.Context) (jobID, action string, err error) {
-	ping, resp, pingErr := a.apiClient.Ping(ctx)
-	// wait a minute, where's my if err != nil block? TL;DR look for pingErr ~20 lines down
-	// the api client returns an error if the response code isn't a 2xx, but there's still information in resp and ping
-	// that we need to check out to do special handling for specific error codes or messages in the response body
-	// once we've done that, we can do the error handling for pingErr
-
-	if ping != nil {
-		// Is there a message that should be shown in the logs?
-		if ping.Message != "" {
-			a.logger.Info(ping.Message)
-		}
-
-		action = ping.Action
-	}
-
-	if pingErr != nil {
-		// If the ping has a non-retryable status, we have to kill the agent, there's no way of recovering
-		// The reason we do this after the disconnect check is because the backend can (and does) send disconnect actions in
-		// responses with non-retryable statuses
-		if resp != nil && !api.IsRetryableStatus(resp) {
-			return "", action, &errUnrecoverable{action: "Ping", response: resp, err: pingErr}
-		}
-
-		// Get the last ping time to the nearest microsecond
-		a.stats.Lock()
-		defer a.stats.Unlock()
-
-		// If a ping fails, we don't really care, because it'll
-		// ping again after the interval.
-		if a.stats.lastPing.IsZero() {
-			return "", action, fmt.Errorf("Failed to ping: %w (No successful ping yet)", pingErr)
-		} else {
-			return "", action, fmt.Errorf("Failed to ping: %w (Last successful was %v ago)", pingErr, time.Since(a.stats.lastPing))
-		}
-	}
-
-	// Track a timestamp for the successful ping for better errors
-	a.stats.Lock()
-	a.stats.lastPing = time.Now()
-	a.stats.Unlock()
-
-	// Should we switch endpoints?
-	if ping.Endpoint != "" && ping.Endpoint != a.agent.Endpoint {
-		newAPIClient := a.apiClient.FromPing(ping)
-
-		// Before switching to the new one, do a ping test to make sure it's
-		// valid. If it is, switch and carry on, otherwise ignore the switch
-		newPing, _, err := newAPIClient.Ping(ctx)
-		if err != nil {
-			a.logger.Warn("Failed to ping the new endpoint %s - ignoring switch for now (%s)", ping.Endpoint, err)
-		} else {
-			// Replace the APIClient and process the new ping
-			a.apiClient = newAPIClient
-			a.agent.Endpoint = ping.Endpoint
-			ping = newPing
-		}
-	}
-
-	// If we don't have a job, there's nothing to do!
-	// If we're paused, job should be nil, but in case it isn't, ignore it.
-	if ping.Job == nil || action == "pause" {
-		return "", action, nil
-	}
-
-	return ping.Job.ID, action, nil
-}
-
 // AcquireAndRunJob attempts to acquire a job an run it. It will retry at after the
 // server determined interval (from the Retry-After response header) if the job is in the waiting
 // state. If the job is in an unassignable state, it will return an error immediately.
@@ -853,51 +487,6 @@ func (a *AgentWorker) AcquireAndRunJob(ctx context.Context, jobId string) error
 	return a.RunJob(ctx, job, nil)
 }
 
-// Accepts a job and runs it, only returns an error if something goes wrong
-func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, jobID string, idleMon *idleMonitor) error {
-	a.logger.Info("Assigned job %s. Accepting...", jobID)
-
-	// An agent is busy during a job, and idle when the job is done.
-	idleMon.MarkBusy(a)
-	defer idleMon.MarkIdle(a)
-
-	// Accept the job. We'll retry on connection related issues, but if
-	// Buildkite returns a 422 or 500 for example, we'll just bail out,
-	// re-ping, and try the whole process again.
-	r := roko.NewRetrier(
-		roko.WithMaxAttempts(30),
-		roko.WithStrategy(roko.Constant(5*time.Second)),
-	)
-
-	accepted, err := roko.DoFunc(ctx, r, func(r *roko.Retrier) (*api.Job, error) {
-		accepted, _, err := a.apiClient.AcceptJob(ctx, jobID)
-		if err != nil {
-			if api.IsRetryableError(err) {
-				a.logger.Warn("%s (%s)", err, r)
-			} else {
-				a.logger.Warn("Buildkite rejected the call to accept the job (%s)", err)
-				r.Break()
-			}
-		}
-		return accepted, err
-	})
-
-	// If `accepted` is nil, then the job was never accepted
-	if accepted == nil {
-		return fmt.Errorf("Failed to accept job: %w", err)
-	}
-
-	// If we're disconnecting-after-job, signal back to Buildkite that we're not
-	// interested in jobs after this one.
-	var ignoreAgentInDispatches *bool
-	if a.agentConfiguration.DisconnectAfterJob {
-		ignoreAgentInDispatches = ptr.To(true)
-	}
-
-	// Now that we've accepted the job, let's run it
-	return a.RunJob(ctx, accepted, ignoreAgentInDispatches)
-}
-
 func (a *AgentWorker) RunJob(ctx context.Context, acceptResponse *api.Job, ignoreAgentInDispatches *bool) error {
 	a.setBusy(acceptResponse.ID)
 	defer a.setIdle()
@@ -967,3 +556,17 @@ func (a *AgentWorker) healthHandler() http.HandlerFunc {
 		}
 	}
 }
+
+type actionMessage struct {
+	// Details of the action to execute
+	action, jobID string
+
+	// Results of the action
+	errCh chan<- error
+
+	// Secret internal action between the streaming loop and debouncer:
+	// set to true when the streaming loop is unhealthy
+	// and the baton should be released so the ping loop is unblocked
+	// (once the current action is completed, if that's the case).
+	unhealthy bool
+}
diff --git a/agent/agent_worker_action.go b/agent/agent_worker_action.go
new file mode 100644
index 0000000000..e24ef057df
--- /dev/null
+++ b/agent/agent_worker_action.go
@@ -0,0 +1,229 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/buildkite/agent/v3/api"
+	"github.com/buildkite/agent/v3/internal/ptr"
+	"github.com/buildkite/agent/v3/status"
+	"github.com/buildkite/roko"
+)
+
+func (a *AgentWorker) runActionLoop(ctx context.Context, idleMon *idleMonitor, fromPingLoop, fromDebouncer <-chan actionMessage) error {
+	a.logger.Debug("[runActionLoop] Starting")
+	defer a.logger.Debug("[runActionLoop] Exiting")
+
+	// Once this loop terminates, there's no point continuing the others,
+	// because nothing remains to execute their actions.
+	defer a.internalStop()
+
+	ctx, setStat, _ := status.AddSimpleItem(ctx, "Action loop")
+	defer setStat("🛑 Action loop stopped!")
+	setStat("🏃 Starting...")
+
+	// Start timing disconnect-after-uptime, if configured.
+	var disconnectAfterUptime <-chan time.Time
+	maxUptime := a.agentConfiguration.DisconnectAfterUptime
+	if maxUptime > 0 {
+		disconnectAfterUptime = time.After(maxUptime)
+	}
+
+	exitWhenNotPaused := false // the next time the action isn't "pause", exit
+	ranJob := false
+	paused := false
+
+	for {
+		// Did both sources of actions terminate? Then we're done too.
+		if fromPingLoop == nil && fromDebouncer == nil {
+			a.logger.Debug("[runActionLoop] All action sources channels are closed, exiting")
+			return nil
+		}
+
+		// Wait for one of the following:
+		// - an action
+		// - the context to be cancelled
+		// - the agent is stopping (a.stop)
+		// - the idle monitor has declared we're all exiting
+		//   (if DisconnectAfterIdleTimeout is configured & we're not paused)
+		// - disconnect after uptime
+		//   (if DisconnectAfterUptime is configured & we're not paused)
+		a.logger.Debug("[runActionLoop] Waiting for an action...")
+		setStat("⌚️ Waiting for an action...")
+		var msg actionMessage
+		select {
+		case m, open := <-fromPingLoop:
+			if !open {
+				// Setting to nil prevents this branch of the select from
+				// happening again.
+				fromPingLoop = nil
+				continue
+			}
+			a.logger.Debug("[runActionLoop] Got action %q, jobID %q from ping loop", m.action, m.jobID)
+			msg = m
+			// continue below
+
+		case m, open := <-fromDebouncer:
+			if !open {
+				fromDebouncer = nil
+				continue
+			}
+			a.logger.Debug("[runActionLoop] Got action %q, jobID %q from streaming loop debouncer", m.action, m.jobID)
+			msg = m
+			// continue below
+
+		case <-ctx.Done():
+			a.logger.Debug("[runActionLoop] Stopping due to context cancel")
+			return ctx.Err()
+
+		case <-a.stop:
+			a.logger.Debug("[runActionLoop] Stopping due to agent stop")
+			return nil
+
+		case <-disconnectAfterUptime:
+			a.logger.Info("Agent has exceeded max uptime of %v", maxUptime)
+			if paused {
+				// Wait to be unpaused before exiting
+				a.logger.Info("Awaiting resume before disconnecting...")
+				exitWhenNotPaused = true
+				continue
+			}
+			a.logger.Info("Disconnecting...")
+			return nil
+
+		case <-idleMon.Exiting():
+			// This should only happen if the agent isn't paused.
+			// (Pausedness is a kind of non-idleness.)
+			a.logger.Info("All agents have been idle for at least %v. Disconnecting...", idleMon.idleTimeout)
+			return nil
+		}
+
+		// Let's handle the action!
+		a.logger.Debug("[runActionLoop] Performing action %q, jobID %q", msg.action, msg.jobID)
+		setStat(fmt.Sprintf("🧑‍🍳 Performing %q action...", msg.action))
+		pingActions.WithLabelValues(msg.action).Inc()
+
+		// In cases where we need to disconnect, *don't* send on msg.errCh,
+		// in order to force the <-a.stop branch in the other loops.
+		// Otherwise, be sure to `close(msg.errCh)`!
+		switch msg.action {
+		case "disconnect":
+			a.logger.Debug("[runActionLoop] Stopping action loop due to disconnect action")
+			return nil
+
+		case "pause":
+			// An agent is not dispatched any jobs while it is paused, but the
+			// paused agent is expected to remain alive and pinging for
+			// instructions.
+			// *This includes acquire-job and disconnect-after-idle-timeout.*
+			a.logger.Debug("[runActionLoop] Entering pause state")
+			paused = true
+			// For the purposes of deciding whether or not to exit,
+			// pausedness is a kind of non-idleness.
+			// If there's also no job, agent is marked as idle below.
+			idleMon.MarkBusy(a)
+			close(msg.errCh)
+			continue
+		}
+
+		// At this point, action was neither "disconnect" nor "pause".
+		if exitWhenNotPaused {
+			a.logger.Debug("[runActionLoop] Stopping action loop because exitWhenNotPaused is true")
+			return nil
+		}
+		if paused {
+			// We're not paused any more! Log a helpful message.
+			a.logger.Info("Agent has resumed after being paused")
+			paused = false
+		}
+
+		// For acquire-job agents, registration sets ignore-in-dispatches=true,
+		// so jobID should be empty. If not, complain.
+		if a.agentConfiguration.AcquireJob != "" {
+			if msg.jobID != "" {
+				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in acquire-job mode! Ignoring the new job", msg.jobID)
+			}
+			// Disconnect after acquire-job.
+			return nil
+		}
+
+		// In disconnect-after-job mode, finishing the job sets
+		// ignore-in-dispatches=true. So jobID should be empty. If not, complain.
+		if ranJob && a.agentConfiguration.DisconnectAfterJob {
+			if msg.jobID != "" {
+				a.logger.Error("Agent ping dispatched a job (id %q) but agent is in disconnect-after-job mode (and already ran a job)! Ignoring the new job", msg.jobID)
+			}
+			a.logger.Info("Job ran, and disconnect-after-job is enabled. Disconnecting...")
+			return nil
+		}
+
+		// If the jobID is empty, then it's an idle message
+		if msg.jobID == "" {
+			// This ensures agents that never receive a job are still tracked
+			// by the idle monitor and can properly trigger disconnect-after-idle-timeout.
+			idleMon.MarkIdle(a)
+			close(msg.errCh)
+			continue
+		}
+
+		setStat("💼 Accepting job")
+
+		// Runs the job, only errors if something goes wrong
+		if err := a.AcceptAndRunJob(ctx, msg.jobID, idleMon); err != nil {
+			a.logger.Error("%v", err)
+			setStat(fmt.Sprintf("✅ Finished job with error: %v", err))
+			msg.errCh <- err // so the ping loop can do something special
+			close(msg.errCh)
+			continue
+		}
+
+		ranJob = true
+		close(msg.errCh)
+	}
+}
+
+// Accepts a job and runs it, only returns an error if something goes wrong
+func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, jobID string, idleMon *idleMonitor) error {
+	a.logger.Info("Assigned job %s. Accepting...", jobID)
+
+	// An agent is busy during a job, and idle when the job is done.
+	idleMon.MarkBusy(a)
+	defer idleMon.MarkIdle(a)
+
+	// Accept the job. We'll retry on connection related issues, but if
+	// Buildkite returns a 422 or 500 for example, we'll just bail out,
+	// re-ping, and try the whole process again.
+	r := roko.NewRetrier(
+		roko.WithMaxAttempts(30),
+		roko.WithStrategy(roko.Constant(5*time.Second)),
+	)
+
+	accepted, err := roko.DoFunc(ctx, r, func(r *roko.Retrier) (*api.Job, error) {
+		accepted, _, err := a.apiClient.AcceptJob(ctx, jobID)
+		if err != nil {
+			if api.IsRetryableError(err) {
+				a.logger.Warn("%s (%s)", err, r)
+			} else {
+				a.logger.Warn("Buildkite rejected the call to accept the job (%s)", err)
+				r.Break()
+			}
+		}
+		return accepted, err
+	})
+
+	// If `accepted` is nil, then the job was never accepted
+	if accepted == nil {
+		return fmt.Errorf("Failed to accept job: %w", err)
+	}
+
+	// If we're disconnecting-after-job, signal back to Buildkite that we're not
+	// interested in jobs after this one.
+	var ignoreAgentInDispatches *bool
+	if a.agentConfiguration.DisconnectAfterJob {
+		ignoreAgentInDispatches = ptr.To(true)
+	}
+
+	// Now that we've accepted the job, let's run it
+	return a.RunJob(ctx, accepted, ignoreAgentInDispatches)
+}
diff --git a/agent/agent_worker_debouncer.go b/agent/agent_worker_debouncer.go
new file mode 100644
index 0000000000..e928d107b2
--- /dev/null
+++ b/agent/agent_worker_debouncer.go
@@ -0,0 +1,144 @@
+package agent
+
+import (
+	"context"
+)
+
+// runDebouncer is an event debouncing loop between the streaming loop and the
+// action handler loop.
+//
+// There are two *big* differences between the streaming loop and the
+// classical ping loop:
+//
+//  1. When pings happen, they happen "regularly". Actions are only sent
+//     in response. But when the streaming loop receives messages is up to
+//     the backend.
+//  2. Pings can be put on hold while a job is running. But streaming
+//     messages can keep arriving during a job.
+//
+// Firstly, we want to get back to receiving from the stream
+// as soon as possible, rather than blocking until the action is handled,
+// so that the stream remains healthy.
+// Secondly, we need to reduce consecutive messages down to only 0 or 1 correct
+// next action(s) following a job.
+// For example, say during a job someone clicks "pause" and "resume"
+// and "pause" again on this agent. This may cause three distinct
+// events to be sent to the streaming loop. If we pass them all on to the
+// action handler directly, then the "resume" may cause the agent to
+// exit in a one-shot mode, even though the second "pause" means the
+// user actually *did* want the agent to be paused.
+func (a *AgentWorker) runDebouncer(ctx context.Context, bat *baton, outCh chan<- actionMessage, inCh <-chan actionMessage) error {
+	a.logger.Debug("[runDebouncer] Starting")
+	defer a.logger.Debug("[runDebouncer] Exiting")
+
+	// When the debouncer returns, close the output channel to let the next
+	// loop know to stop listening to it.
+	defer close(outCh)
+
+	// We begin not running an action.
+	actionInProgress := false
+
+	// We begin holding the baton, ensure it is released when we exit.
+	defer func() {
+		a.logger.Debug("[runDebouncer] Releasing the baton")
+		bat.Release("debouncer")
+	}()
+
+	// lastActionResult is closed when the action handler is done handling the
+	// last action we sent.
+	// It starts nil because at the beginning, there is no previous action.
+	var lastActionResult chan error
+
+	// pending is the next message to send, when able.
+	var pending *actionMessage
+
+	// Is the stream healthy?
+	// If so, take the baton (which blocks the ping loop).
+	// If not, return the baton (unblocking the ping loop).
+	// Returning the baton may have to wait for the current action to complete.
+	healthy := true
+
+	for {
+		select {
+		case <-a.stop:
+			a.logger.Debug("[runDebouncer] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runDebouncer] Stopping due to context cancel")
+			return ctx.Err()
+
+		case <-iif(healthy, bat.Acquire("debouncer")): // if the stream is healthy, take the baton if available
+			a.logger.Debug("[runDebouncer] Took the baton")
+			// We now have the baton!
+			// continue below to send any pending message, if able
+
+		case msg, open := <-inCh: // streaming loop has produced an event
+			if !open {
+				a.logger.Debug("[runDebouncer] Stopping due to input channel closing")
+				return nil
+			}
+
+			healthy = !msg.unhealthy
+
+			if !healthy {
+				a.logger.Debug("[runDebouncer] Streaming loop is unhealthy")
+
+				// It is not healthy, so release the baton as soon as we can
+				// (when the current action is done).
+				if !actionInProgress {
+					// We can release the baton now.
+					a.logger.Debug("[runDebouncer] Releasing the baton")
+					bat.Release("debouncer")
+				}
+				break // out of the select
+			}
+
+			// The next message to send is, currently, always the most recent
+			// healthy message.
+			pending = &msg
+
+			// continue below to send it
+
+		case <-lastActionResult: // most recent action has completed
+			a.logger.Debug("[runDebouncer] Last action has completed")
+			// Set the channel variable to nil so we don't spinloop.
+			// (Operations on a nil channel block forever.)
+			lastActionResult = nil
+			actionInProgress = false
+
+			// continue below to send a pending message
+		}
+
+		// If we're healthy, have the baton, there's no action in progress,
+		// and there's a pending message, then send that message.
+		if !healthy || !bat.HeldBy("debouncer") || actionInProgress || pending == nil {
+			continue
+		}
+		a.logger.Debug("[runDebouncer] Sending action %q, jobID %q", pending.action, pending.jobID)
+		lastActionResult = make(chan error)
+		pending.errCh = lastActionResult
+		select {
+		case outCh <- *pending:
+			// sent!
+			pending = nil
+			actionInProgress = true
+		case <-a.stop:
+			a.logger.Debug("[runDebouncer] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runDebouncer] Stopping due to context cancel")
+			return ctx.Err()
+		}
+	}
+}
+
+// iif returns t if b is true, otherwise it returns the zero value of T.
+// This is useful for enabling or disabling a select case based on a test
+// evaluated at the start of the select.
+func iif[T any](b bool, t T) T {
+	if b {
+		return t
+	}
+	var f T
+	return f
+}
diff --git a/agent/agent_worker_heartbeat.go b/agent/agent_worker_heartbeat.go
new file mode 100644
index 0000000000..ca4f751b6c
--- /dev/null
+++ b/agent/agent_worker_heartbeat.go
@@ -0,0 +1,52 @@
+package agent
+
+import (
+	"context"
+	"time"
+
+	"github.com/buildkite/agent/v3/status"
+)
+
+func (a *AgentWorker) runHeartbeatLoop(ctx context.Context) error {
+	ctx, setStat, _ := status.AddSimpleItem(ctx, "Heartbeat loop")
+	defer setStat("💔 Heartbeat loop stopped!")
+	setStat("🏃 Starting...")
+
+	heartbeatInterval := time.Second * time.Duration(a.agent.HeartbeatInterval)
+	heartbeatTicker := time.NewTicker(heartbeatInterval)
+	defer heartbeatTicker.Stop()
+	for {
+		setStat("😴 Sleeping for a bit")
+		select {
+		case <-heartbeatTicker.C:
+			setStat("❤️ Sending heartbeat")
+			if err := a.Heartbeat(ctx); err != nil {
+				if isUnrecoverable(err) {
+					a.logger.Error("%s", err)
+					// unrecoverable heartbeat failure also stops everything else
+					a.StopUngracefully()
+					return err
+				}
+
+				// Get the last heartbeat time to the nearest microsecond
+				a.stats.Lock()
+				if a.stats.lastHeartbeat.IsZero() {
+					a.logger.Error("Failed to heartbeat %s. Will try again in %v. (No heartbeat yet)",
+						err, heartbeatInterval)
+				} else {
+					a.logger.Error("Failed to heartbeat %s. Will try again in %v. (Last successful was %v ago)",
+						err, heartbeatInterval, time.Since(a.stats.lastHeartbeat))
+				}
+				a.stats.Unlock()
+			}
+
+		case <-ctx.Done():
+			a.logger.Debug("Stopping heartbeats due to context cancel")
+			// An alternative to returning nil would be ctx.Err(), but we use
+			// the context for ordinary termination of this loop.
+			// A context cancellation from outside the agent worker would still
+			// be reflected in the value returned by the ping loop return.
+			return nil
+		}
+	}
+}
diff --git a/agent/agent_worker_ping.go b/agent/agent_worker_ping.go
new file mode 100644
index 0000000000..64b9f110a7
--- /dev/null
+++ b/agent/agent_worker_ping.go
@@ -0,0 +1,255 @@
+package agent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"time"
+
+	"github.com/buildkite/agent/v3/api"
+	"github.com/buildkite/agent/v3/status"
+)
+
+// runPingLoop runs the (classical) loop that pings Buildkite for work.
+func (a *AgentWorker) runPingLoop(ctx context.Context, bat *baton, outCh chan<- actionMessage) error {
+	a.logger.Debug("[runPingLoop] Starting")
+	defer a.logger.Debug("[runPingLoop] Exiting")
+
+	// When this loop returns, close the channel to let the action handler loop
+	// stop listening for actions from it.
+	defer close(outCh)
+
+	ctx, setStat, _ := status.AddSimpleItem(ctx, "Ping loop")
+	defer setStat("🛑 Ping loop stopped!")
+	setStat("🏃 Starting...")
+
+	// Create the ticker
+	pingInterval := time.Second * time.Duration(a.agent.PingInterval)
+	pingTicker := time.NewTicker(pingInterval)
+	defer pingTicker.Stop()
+
+	// On the first iteration, skip waiting for the pingTicker.
+	// This doesn't skip the jitter, though.
+	skipWait := make(chan struct{}, 1)
+	skipWait <- struct{}{}
+	if a.noWaitBetweenPingsForTesting {
+		// a closed channel will unblock the for/select instantly, for zero-delay ping loop testing.
+		close(skipWait)
+	}
+
+	a.logger.Info("Waiting for instructions...")
+
+	for {
+		startWait := time.Now()
+		a.logger.Debug("[runPingLoop] Waiting for pingTicker")
+		setStat("😴 Waiting until next ping interval tick")
+		select {
+		case <-skipWait:
+			// continue below
+		case <-pingTicker.C:
+			// continue below
+		case <-a.stop:
+			a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+			return ctx.Err()
+		}
+
+		// Within the interval, wait a random amount of time to avoid
+		// spontaneous synchronisation across agents.
+		jitter := rand.N(pingInterval)
+		a.logger.Debug("[runPingLoop] Waiting for jitter %v", jitter)
+		setStat(fmt.Sprintf("🫨 Jittering for %v", jitter))
+		select {
+		case <-skipWait:
+			// continue below
+		case <-time.After(jitter):
+			// continue below
+		case <-a.stop:
+			a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+			return ctx.Err()
+		}
+		pingWaitDurations.Observe(time.Since(startWait).Seconds())
+
+		// stop is only used internally when stopping.
+		stop := errors.New("stop")
+		err := func() error {
+			// Wait until the baton is available. If this takes forever, that's
+			// a good thing because it should mean the streaming loop is
+			// healthy.
+			// Once acquired, only release the baton after any work is complete,
+			// to prevent the streaming loop from resuming control until then,
+			// but we always release the baton, because the streaming loop is
+			// preferred.
+			a.logger.Debug("[runPingLoop] Waiting for baton")
+			select {
+			case <-bat.Acquire("ping"): // the baton is ours!
+				a.logger.Debug("[runPingLoop] Acquired the baton")
+				defer func() { // <- this is why the loop body is in a func
+					a.logger.Debug("[runPingLoop] Releasing the baton")
+					bat.Release("ping")
+				}()
+
+			case <-a.stop:
+				a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+				return stop
+			case <-ctx.Done():
+				a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+				return ctx.Err()
+			}
+
+			a.logger.Debug("[runPingLoop] Pinging buildkite for instructions")
+			setStat("📡 Pinging Buildkite for instructions")
+			pingsSent.Inc()
+			startPing := time.Now()
+			jobID, action, err := a.Ping(ctx)
+			if err != nil {
+				pingErrors.Inc()
+				if isUnrecoverable(err) {
+					a.logger.Error("%v", err)
+					return err
+				}
+				a.logger.Warn("%v", err)
+			}
+			pingDurations.Observe(time.Since(startPing).Seconds())
+
+			a.logger.Debug("[runPingLoop] Sending action")
+
+			// Send the action to the action loop
+			errCh := make(chan error)
+			msg := actionMessage{
+				action: action,
+				jobID:  jobID,
+				errCh:  errCh,
+			}
+			select {
+			case outCh <- msg:
+				// sent!
+			case <-a.stop:
+				a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+				return stop
+			case <-ctx.Done():
+				a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+				return ctx.Err()
+			}
+
+			// Wait for completion
+			select {
+			case err := <-errCh:
+				if err != nil || jobID == "" {
+					// We don't terminate the ping loop just because the
+					// action (usually a job) has failed.
+					return nil
+				}
+				if a.noWaitBetweenPingsForTesting {
+					// Don't bother resetting the ticker,
+					// don't try to send on a closed channel (skipWait).
+					return nil
+				}
+				// A job ran (or was at least started) successfully.
+				// Observation: jobs are rarely the last within a pipeline,
+				// thus if this worker just completed a job,
+				// there is likely another immediately available.
+				// Skip waiting for the ping interval until
+				// a ping without a job has occurred,
+				// but in exchange, ensure the next ping must wait at least a full
+				// pingInterval to avoid too much server load.
+				pingTicker.Reset(pingInterval)
+				select {
+				case skipWait <- struct{}{}:
+					// Ticker will be skipped
+				default:
+					// We're already skipping the ticker, don't block.
+				}
+				return nil
+			case <-a.stop:
+				a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+				return stop
+			case <-ctx.Done():
+				a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+				return ctx.Err()
+			}
+		}()
+		if err == stop {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// Performs a ping that checks Buildkite for a job or action to take
+// Returns a job, or nil if none is found
+func (a *AgentWorker) Ping(ctx context.Context) (jobID, action string, err error) {
+	ping, resp, pingErr := a.apiClient.Ping(ctx)
+	// wait a minute, where's my if err != nil block? TL;DR look for pingErr ~20 lines down
+	// the api client returns an error if the response code isn't a 2xx, but there's still information in resp and ping
+	// that we need to check out to do special handling for specific error codes or messages in the response body
+	// once we've done that, we can do the error handling for pingErr
+
+	if ping != nil {
+		// Is there a message that should be shown in the logs?
+		if ping.Message != "" {
+			a.logger.Info(ping.Message)
+		}
+
+		action = ping.Action
+	}
+
+	if pingErr != nil {
+		// If the ping has a non-retryable status, we have to kill the agent, there's no way of recovering
+		// The reason we do this after the disconnect check is because the backend can (and does) send disconnect actions in
+		// responses with non-retryable statuses
+		if resp != nil && !api.IsRetryableStatus(resp) {
+			return "", action, &errUnrecoverable{action: "Ping", response: resp, err: pingErr}
+		}
+
+		// Get the last ping time to the nearest microsecond
+		a.stats.Lock()
+		defer a.stats.Unlock()
+
+		// If a ping fails, we don't really care, because it'll
+		// ping again after the interval.
+		if a.stats.lastPing.IsZero() {
+			return "", action, fmt.Errorf("Failed to ping: %w (No successful ping yet)", pingErr)
+		} else {
+			return "", action, fmt.Errorf("Failed to ping: %w (Last successful was %v ago)", pingErr, time.Since(a.stats.lastPing))
+		}
+	}
+
+	// Track a timestamp for the successful ping for better errors
+	a.stats.Lock()
+	a.stats.lastPing = time.Now()
+	a.stats.Unlock()
+
+	// Should we switch endpoints?
+	if ping.Endpoint != "" && ping.Endpoint != a.agent.Endpoint {
+		newAPIClient := a.apiClient.FromPing(ping)
+
+		// Before switching to the new one, do a ping test to make sure it's
+		// valid. If it is, switch and carry on, otherwise ignore the switch
+		newPing, _, err := newAPIClient.Ping(ctx)
+		if err != nil {
+			a.logger.Warn("Failed to ping the new endpoint %s - ignoring switch for now (%s)", ping.Endpoint, err)
+		} else {
+			// Replace the APIClient and process the new ping
+			a.apiClient = newAPIClient
+			a.agent.Endpoint = ping.Endpoint
+			ping = newPing
+		}
+	}
+
+	// If we don't have a job, there's nothing to do!
+	// If we're paused, job should be nil, but in case it isn't, ignore it.
+	if ping.Job == nil || action == "pause" {
+		return "", action, nil
+	}
+
+	return ping.Job.ID, action, nil
+}
diff --git a/agent/agent_worker_streaming.go b/agent/agent_worker_streaming.go
new file mode 100644
index 0000000000..1a45451078
--- /dev/null
+++ b/agent/agent_worker_streaming.go
@@ -0,0 +1,201 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"math/rand/v2"
+	"time"
+
+	"connectrpc.com/connect"
+	agentedgev1 "github.com/buildkite/agent/v3/api/proto/gen"
+	"github.com/buildkite/agent/v3/status"
+)
+
+// runStreamingPingLoop runs the streaming loop. It is best-effort
+// (allowed to fail and fall back to the regular ping loop) but when it works
+// it is preferred because there is less waiting around.
+func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- actionMessage) error {
+	a.logger.Debug("[runStreamingPingLoop] Starting")
+	defer a.logger.Debug("[runStreamingPingLoop] Exiting")
+
+	// When this loop returns, close the channel to let the next loop stop
+	// listening to it.
+	defer close(outCh)
+
+	ctx, setStat, _ := status.AddSimpleItem(ctx, "Streaming ping loop")
+	defer setStat("🛑 Ping stream loop stopped!")
+	setStat("🏃 Starting...")
+
+	// The stream Receive call blocks until a message is received - we can't
+	// select on it. streamCtx exists to end the stream on agent stop.
+	streamCtx, cancelStream := context.WithCancel(ctx)
+	defer cancelStream()
+	go func() {
+		<-a.stop
+		cancelStream()
+	}()
+
+	// Because we expect the streaming connection to last much longer than a
+	// ping, we should use a different doctrine compared with the ping loop.
+	//
+	// This loop is a repeated fuzzed exponential backoff:
+	//
+	// If the connection is successful, once it closes, the next connection will
+	// begin after a minimal jitter.
+	// While the connection fails, each attempt will jitter over double the
+	// previous interval before attempting reconnection.
+	//
+	// Note: This _could_ be implemented with an infinite loop containing a roko
+	// retrier, but it looked a bit messier to me.
+	initialMaxJitter := 1 * time.Second
+	attempts := 0
+
+	var skipWait chan struct{}
+	if a.noWaitBetweenPingsForTesting {
+		// a closed channel will unblock the select instantly, for zero-delay loop testing.
+		skipWait = make(chan struct{})
+		close(skipWait)
+	}
+
+	for {
+		// Backoff exponentially, up to initialMaxJitter * 2^6.
+		// (Repeated failures may jitter up to 64 seconds between attempts.)
+		maxJitter := initialMaxJitter << min(attempts, 6)
+		attempts++
+
+		// Within the interval, wait a random amount of time to avoid
+		// spontaneous synchronisation across agents.
+		jitter := rand.N(maxJitter)
+		setStat(fmt.Sprintf("🫨 Jittering for %v (max %v)", jitter, maxJitter))
+		a.logger.Debug("[runStreamingPingLoop] Waiting for jitter %v (max %v)", jitter, maxJitter)
+		select {
+		case <-skipWait:
+			// continue below
+		case <-time.After(jitter):
+			// continue below
+		case <-a.stop:
+			a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
+			return ctx.Err()
+		}
+
+		setStat(fmt.Sprintf("📱 Connecting to ping stream (attempt %d)...", attempts))
+		a.logger.Debug("[runStreamingPingLoop] Connecting (attempt %d)", attempts)
+		stream, err := a.apiClient.StreamPings(streamCtx, a.agent.UUID)
+		if err != nil {
+			a.logger.Error("Connection to ping stream failed: %v", err)
+			if isUnrecoverable(err) {
+				a.logger.Error("Stopping ping stream because the error is unrecoverable")
+				return err
+			}
+			// Fast fallback to the ping loop
+			a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+			select {
+			case outCh <- actionMessage{unhealthy: true}:
+				a.logger.Debug("[runStreamingPingLoop] Unhealthy message sent to debouncer")
+				// sent!
+			case <-a.stop:
+				a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
+				return nil
+			case <-ctx.Done():
+				a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
+				return ctx.Err()
+			}
+			continue
+		}
+
+		firstMsg := true // used for the "connection established" log
+
+		setStat("🏞️ Streaming actions from Buildkite")
+		a.logger.Debug("[runStreamingPingLoop] Waiting for a message...")
+	streamLoop:
+		for msg, err := range stream {
+			a.logger.Debug("[runStreamingPingLoop] Received msg %v, err %v", msg, err)
+
+			var amsg actionMessage
+			switch {
+			case err != nil:
+				a.logger.Debug("[runStreamingPingLoop] Connection to ping stream failed or ended: %v", err)
+				if isUnrecoverable(err) {
+					a.logger.Error("Stopping ping stream loop because the error is unrecoverable: %v", err)
+					return err
+				}
+				// Go unhealthy, unless the error is deadline-exceeded.
+				// (The connection timed out, which we want to happen every so often).
+				if connect.CodeOf(err) == connect.CodeDeadlineExceeded {
+					a.logger.Debug("[runStreamingPingLoop] Breaking streamLoop to reconnect after deadline-exceeded")
+					break streamLoop
+				}
+				a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+				amsg.unhealthy = true
+
+			case msg == nil:
+				a.logger.Error("Ping stream yielded a nil message, so assuming the stream is broken")
+				a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+				amsg.unhealthy = true
+
+			default:
+				if firstMsg {
+					a.logger.Info("Ping stream connection established")
+					firstMsg = false
+				}
+
+				switch act := msg.Action.(type) {
+				case *agentedgev1.StreamPingsResponse_Resume: // a.k.a. "idle"
+					// continue below
+
+				case *agentedgev1.StreamPingsResponse_Pause:
+					if reason := act.Pause.GetReason(); reason != "" {
+						a.logger.Info("%s", reason)
+					}
+					amsg.action = "pause"
+
+				case *agentedgev1.StreamPingsResponse_Disconnect:
+					if reason := act.Disconnect.GetReason(); reason != "" {
+						a.logger.Info("%s", reason)
+					}
+					amsg.action = "disconnect"
+
+				case *agentedgev1.StreamPingsResponse_JobAssigned:
+					amsg.jobID = act.JobAssigned.GetJob().GetId()
+					if amsg.jobID == "" {
+						a.logger.Error("Ping stream yielded a JobAssigned message with nil job or empty job ID, so assuming the stream is broken")
+						a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+						amsg.unhealthy = true
+					}
+				}
+			}
+
+			// Send the message to the debouncer.
+			select {
+			case outCh <- amsg:
+				a.logger.Debug("[runStreamingPingLoop] Message sent to debouncer")
+				// sent!
+			case <-a.stop:
+				a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
+				return nil
+			case <-ctx.Done():
+				a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
+				return ctx.Err()
+			}
+
+			// In case the server sends a disconnect but doesn't close the
+			// stream, be sure to exit.
+			if amsg.action == "disconnect" {
+				a.logger.Debug("[runStreamingPingLoop] Stopping due to disconnect action")
+				a.internalStop()
+				return nil
+			}
+
+			if amsg.unhealthy {
+				a.logger.Debug("[runStreamingPingLoop] Breaking streamLoop to reconnect because the stream is unhealthy")
+				break streamLoop
+			} else {
+				// Stream is healthy, reset the retry counter
+				attempts = 0
+			}
+		}
+	}
+}
diff --git a/agent/agent_worker_test.go b/agent/agent_worker_test.go
index 5c536dc61c..b23aeb5c1c 100644
--- a/agent/agent_worker_test.go
+++ b/agent/agent_worker_test.go
@@ -588,14 +588,9 @@ func TestAgentWorker_DisconnectAfterUptime(t *testing.T) {
 		t.Errorf("Agent should have disconnected after ~1 second, but took %v", elapsed)
 	}
 
-	// The agent should have made at least one ping before disconnecting
-	if pingCount == 0 {
-		t.Error("Agent should have made at least one ping before disconnecting")
-	}
-
-	// The agent should have made at least one ping and should have disconnected
-	// due to max uptime being exceeded. The important thing is that the agent
-	// disconnected properly with the uptime check, which we verified above.
+	// The agent may not get around to pinging before the uptime is exceeded.
+	// The important thing is that the agent disconnected properly with the
+	// uptime check, which we verified above.
 }
 
 func TestAgentWorker_SetEndpointDuringRegistration(t *testing.T) {

From c2940e6b269e1f4d33fe78f15d8eb857c5f28dcb Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 18 Feb 2026 16:59:52 +1100
Subject: [PATCH 211/242] Use t.Context in worker tests

---
 agent/agent_worker_test.go | 66 ++++++++------------------------------
 1 file changed, 14 insertions(+), 52 deletions(-)

diff --git a/agent/agent_worker_test.go b/agent/agent_worker_test.go
index b23aeb5c1c..68ea47b217 100644
--- a/agent/agent_worker_test.go
+++ b/agent/agent_worker_test.go
@@ -1,7 +1,6 @@
 package agent
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"math"
@@ -49,8 +48,6 @@ func TestDisconnect(t *testing.T) {
 	}))
 	defer server.Close()
 
-	ctx := context.Background()
-
 	apiClient := api.NewClient(logger.Discard, api.Config{
 		Endpoint: server.URL,
 		Token:    "llamas",
@@ -73,7 +70,7 @@ func TestDisconnect(t *testing.T) {
 		agentConfiguration: AgentConfiguration{},
 	}
 
-	err := worker.Disconnect(ctx)
+	err := worker.Disconnect(t.Context())
 	require.NoError(t, err)
 
 	assert.Equal(t, []string{"[info] Disconnecting...", "[info] Disconnected"}, l.Messages)
@@ -100,8 +97,6 @@ func TestDisconnectRetry(t *testing.T) {
 	}))
 	defer server.Close()
 
-	ctx := context.Background()
-
 	apiClient := api.NewClient(logger.Discard, api.Config{
 		Endpoint: server.URL,
 		Token:    "llamas",
@@ -126,7 +121,7 @@ func TestDisconnectRetry(t *testing.T) {
 		agentConfiguration: AgentConfiguration{},
 	}
 
-	err := worker.Disconnect(ctx)
+	err := worker.Disconnect(t.Context())
 	assert.NoError(t, err)
 
 	// 2 failed attempts sleep 1 second each
@@ -142,9 +137,6 @@ func TestDisconnectRetry(t *testing.T) {
 func TestAcquireJobReturnsWrappedError_WhenServerResponds422(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	jobID := "some-uuid"
 
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
@@ -175,7 +167,7 @@ func TestAcquireJobReturnsWrappedError_WhenServerResponds422(t *testing.T) {
 		agentConfiguration: AgentConfiguration{},
 	}
 
-	err := worker.AcquireAndRunJob(ctx, jobID)
+	err := worker.AcquireAndRunJob(t.Context(), jobID)
 	if !errors.Is(err, core.ErrJobAcquisitionRejected) {
 		t.Fatalf("expected worker.AcquireAndRunJob(%q) = core.ErrJobAcquisitionRejected, got %v", jobID, err)
 	}
@@ -184,9 +176,6 @@ func TestAcquireJobReturnsWrappedError_WhenServerResponds422(t *testing.T) {
 func TestAcquireAndRunJobWaiting(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		switch req.URL.Path {
 		case "/jobs/waitinguuid/acquire":
@@ -233,7 +222,7 @@ func TestAcquireAndRunJobWaiting(t *testing.T) {
 		agentConfiguration: AgentConfiguration{},
 	}
 
-	err := worker.AcquireAndRunJob(ctx, "waitinguuid")
+	err := worker.AcquireAndRunJob(t.Context(), "waitinguuid")
 	assert.ErrorContains(t, err, "423")
 
 	if !errors.Is(err, core.ErrJobLocked) {
@@ -251,9 +240,6 @@ func TestAcquireAndRunJobWaiting(t *testing.T) {
 func TestAgentWorker_Start_AcquireJob_JobAcquisitionRejected(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		switch req.URL.Path {
 		case "/jobs/waitinguuid/acquire":
@@ -319,7 +305,7 @@ func TestAgentWorker_Start_AcquireJob_JobAcquisitionRejected(t *testing.T) {
 
 	// we expect the worker to try to acquire the job, but fail with ErrJobAcquisitionRejected
 	// because the server returns a 422 Unprocessable Entity.
-	err := worker.Start(ctx, nil)
+	err := worker.Start(t.Context(), nil)
 	if !errors.Is(err, core.ErrJobAcquisitionRejected) {
 		t.Fatalf("expected worker.AcquireAndRunJob(%q) = core.ErrJobAcquisitionRejected, got %v", jobID, err)
 	}
@@ -328,9 +314,6 @@ func TestAgentWorker_Start_AcquireJob_JobAcquisitionRejected(t *testing.T) {
 func TestAgentWorker_Start_AcquireJob_Pause_Unpause(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	buildPath := filepath.Join(os.TempDir(), t.Name(), "build")
 	hooksPath := filepath.Join(os.TempDir(), t.Name(), "hooks")
 	if err := errors.Join(os.MkdirAll(buildPath, 0o777), os.MkdirAll(hooksPath, 0o777)); err != nil {
@@ -398,7 +381,7 @@ func TestAgentWorker_Start_AcquireJob_Pause_Unpause(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, nil); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -413,9 +396,6 @@ func TestAgentWorker_Start_AcquireJob_Pause_Unpause(t *testing.T) {
 func TestAgentWorker_DisconnectAfterJob_Start_Pause_Unpause(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	buildPath := filepath.Join(os.TempDir(), t.Name(), "build")
 	hooksPath := filepath.Join(os.TempDir(), t.Name(), "hooks")
 	if err := errors.Join(os.MkdirAll(buildPath, 0o777), os.MkdirAll(hooksPath, 0o777)); err != nil {
@@ -490,7 +470,7 @@ func TestAgentWorker_DisconnectAfterJob_Start_Pause_Unpause(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, nil); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -508,9 +488,6 @@ func TestAgentWorker_DisconnectAfterJob_Start_Pause_Unpause(t *testing.T) {
 func TestAgentWorker_DisconnectAfterUptime(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	buildPath := filepath.Join(os.TempDir(), t.Name(), "build")
 	hooksPath := filepath.Join(os.TempDir(), t.Name(), "hooks")
 	if err := errors.Join(os.MkdirAll(buildPath, 0o777), os.MkdirAll(hooksPath, 0o777)); err != nil {
@@ -578,7 +555,7 @@ func TestAgentWorker_DisconnectAfterUptime(t *testing.T) {
 	// Record start time
 	startTime := time.Now()
 
-	if err := worker.Start(ctx, nil); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -598,9 +575,6 @@ func TestAgentWorker_SetEndpointDuringRegistration(t *testing.T) {
 	// is passed into agent.NewAgentWorker(...), so we'll just test the response handling.
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	server := NewFakeAPIServer()
 	defer server.Close()
 	targetEndpoint := server.URL
@@ -652,7 +626,7 @@ func TestAgentWorker_SetEndpointDuringRegistration(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, nil); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -664,9 +638,6 @@ func TestAgentWorker_SetEndpointDuringRegistration(t *testing.T) {
 func TestAgentWorker_UpdateEndpointDuringPing(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	const agentSessionToken = "alpacas"
 
 	// the first endpoint, to be redirected from
@@ -741,7 +712,7 @@ func TestAgentWorker_UpdateEndpointDuringPing(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, nil); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -754,9 +725,6 @@ func TestAgentWorker_UpdateEndpointDuringPing(t *testing.T) {
 func TestAgentWorker_UpdateEndpointDuringPing_FailAndRevert(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	const agentSessionToken = "alpacas"
 
 	// A working endpoint for the original ping
@@ -821,7 +789,7 @@ func TestAgentWorker_UpdateEndpointDuringPing_FailAndRevert(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, nil); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -833,9 +801,6 @@ func TestAgentWorker_UpdateEndpointDuringPing_FailAndRevert(t *testing.T) {
 func TestAgentWorker_SetRequestHeadersDuringRegistration(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	const headerKey = "Buildkite-Hello"
 	const headerValue = "world"
 
@@ -875,7 +840,7 @@ func TestAgentWorker_SetRequestHeadersDuringRegistration(t *testing.T) {
 	})
 	client := &core.Client{APIClient: apiClient, Logger: l}
 	// the underlying api.Client will capture & store the server-specified request headers here...
-	reg, err := client.Register(ctx, api.AgentRegisterRequest{})
+	reg, err := client.Register(t.Context(), api.AgentRegisterRequest{})
 	if err != nil {
 		t.Fatalf("failed to register: %v", err)
 	}
@@ -890,7 +855,7 @@ func TestAgentWorker_SetRequestHeadersDuringRegistration(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, nil); err != nil {
+	if err := worker.Start(t.Context(), nil); err != nil {
 		t.Errorf("worker.Start() = %v", err)
 	}
 
@@ -987,9 +952,6 @@ func TestAgentWorker_UpdateRequestHeadersDuringPing(t *testing.T) {
 func TestAgentWorker_UnrecoverableErrorInPing(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-
 	const agentSessionToken = "alpacas"
 
 	server := NewFakeAPIServer()
@@ -1030,7 +992,7 @@ func TestAgentWorker_UnrecoverableErrorInPing(t *testing.T) {
 	)
 	worker.noWaitBetweenPingsForTesting = true
 
-	if err := worker.Start(ctx, nil); !isUnrecoverable(err) {
+	if err := worker.Start(t.Context(), nil); !isUnrecoverable(err) {
 		t.Errorf("worker.Start() = %v, want an unrecoverable error", err)
 	}
 

From f16a910bea8d04eac1bd54a41f8a34bd790526b8 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 18 Feb 2026 16:59:52 +1100
Subject: [PATCH 212/242] Add streaming-mode tests

---
 agent/agent_worker_test.go    | 384 ++++++++++++++++++++++++++++++++++
 agent/fake_api_server_test.go |  47 ++++-
 2 files changed, 429 insertions(+), 2 deletions(-)

diff --git a/agent/agent_worker_test.go b/agent/agent_worker_test.go
index 68ea47b217..e6d4ddcd7a 100644
--- a/agent/agent_worker_test.go
+++ b/agent/agent_worker_test.go
@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math"
@@ -15,7 +16,9 @@ import (
 	"testing"
 	"time"
 
+	"connectrpc.com/connect"
 	"github.com/buildkite/agent/v3/api"
+	agentedgev1 "github.com/buildkite/agent/v3/api/proto/gen"
 	"github.com/buildkite/agent/v3/core"
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/metrics"
@@ -1000,3 +1003,384 @@ func TestAgentWorker_UnrecoverableErrorInPing(t *testing.T) {
 		t.Errorf("agent.Pings = %d, want %d", got, want)
 	}
 }
+
+func TestAgentWorker_Streaming_Disconnect(t *testing.T) {
+	t.Parallel()
+
+	server := NewFakeAPIServer(WithStreaming)
+	defer server.Close()
+
+	const agentSessionToken = "alpacas"
+	agent := server.AddAgent(agentSessionToken)
+	agent.PingHandler = func(*http.Request) (api.Ping, error) {
+		return api.Ping{}, errors.New("too many pings")
+	}
+	go func() {
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Disconnect{},
+		}
+		close(agent.PingStream)
+	}()
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(int) {})
+
+	worker := NewAgentWorker(
+		l,
+		&api.AgentRegisterResponse{
+			UUID:              uuid.New().String(),
+			Name:              "agent-1",
+			AccessToken:       agentSessionToken,
+			Endpoint:          server.URL,
+			PingInterval:      1,
+			JobStatusInterval: 5,
+			HeartbeatInterval: 60,
+		},
+		metrics.NewCollector(logger.Discard, metrics.CollectorConfig{}),
+		api.NewClient(logger.Discard, api.Config{
+			Endpoint: server.URL,
+			Token:    "llamas",
+		}),
+		AgentWorkerConfig{},
+	)
+
+	if err := worker.Start(t.Context(), nil); err != nil {
+		t.Errorf("worker.Start() error = %v, want nil", err)
+	}
+	if got, want := agent.Pings, 0; got != want {
+		t.Errorf("agent.Pings = %d, want %d", got, want)
+	}
+}
+
+func TestAgentWorker_Streaming_Pause_Resume_Disconnect(t *testing.T) {
+	t.Parallel()
+
+	server := NewFakeAPIServer(WithStreaming)
+	defer server.Close()
+
+	const agentSessionToken = "alpacas"
+	agent := server.AddAgent(agentSessionToken)
+	agent.PingHandler = func(*http.Request) (api.Ping, error) {
+		return api.Ping{}, errors.New("too many pings")
+	}
+	go func() {
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Pause{
+				Pause: &agentedgev1.PauseAction{
+					Reason: "Agent has been paused",
+				},
+			},
+		}
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Resume{},
+		}
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Disconnect{},
+		}
+		close(agent.PingStream)
+	}()
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(int) {})
+
+	worker := NewAgentWorker(
+		l,
+		&api.AgentRegisterResponse{
+			UUID:              uuid.New().String(),
+			Name:              "agent-1",
+			AccessToken:       agentSessionToken,
+			Endpoint:          server.URL,
+			PingInterval:      1,
+			JobStatusInterval: 5,
+			HeartbeatInterval: 60,
+		},
+		metrics.NewCollector(logger.Discard, metrics.CollectorConfig{}),
+		api.NewClient(logger.Discard, api.Config{
+			Endpoint: server.URL,
+			Token:    "llamas",
+		}),
+		AgentWorkerConfig{},
+	)
+
+	if err := worker.Start(t.Context(), nil); err != nil {
+		t.Errorf("worker.Start() error = %v, want nil", err)
+	}
+	if got, want := agent.Pings, 0; got != want {
+		t.Errorf("agent.Pings = %d, want %d", got, want)
+	}
+}
+
+func TestAgentWorker_Streaming_Start_AcquireJob_Pause_Unpause(t *testing.T) {
+	t.Parallel()
+
+	buildPath := filepath.Join(os.TempDir(), t.Name(), "build")
+	hooksPath := filepath.Join(os.TempDir(), t.Name(), "hooks")
+	if err := errors.Join(os.MkdirAll(buildPath, 0o777), os.MkdirAll(hooksPath, 0o777)); err != nil {
+		t.Fatalf("Couldn't create directories: %v", err)
+	}
+	t.Cleanup(func() {
+		os.RemoveAll(filepath.Join(os.TempDir(), t.Name())) //nolint:errcheck // Best-effort cleanup
+	})
+
+	server := NewFakeAPIServer(WithStreaming)
+	defer server.Close()
+
+	job := server.AddJob(map[string]string{
+		"BUILDKITE_COMMAND": "echo echo",
+	})
+
+	// Pre-register the agent.
+	const agentSessionToken = "alpacas"
+	agent := server.AddAgent(agentSessionToken)
+	agent.PingHandler = func(*http.Request) (api.Ping, error) {
+		return api.Ping{}, errors.New("too many pings")
+	}
+	go func() {
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Pause{},
+		}
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Resume{},
+		}
+		close(agent.PingStream)
+	}()
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(int) {})
+
+	worker := NewAgentWorker(
+		l,
+		&api.AgentRegisterResponse{
+			UUID:              uuid.New().String(),
+			Name:              "agent-1",
+			AccessToken:       agentSessionToken,
+			Endpoint:          server.URL,
+			PingInterval:      1,
+			JobStatusInterval: 1,
+			HeartbeatInterval: 10,
+		},
+		metrics.NewCollector(logger.Discard, metrics.CollectorConfig{}),
+		api.NewClient(logger.Discard, api.Config{
+			Endpoint: server.URL,
+			Token:    "llamas",
+		}),
+		AgentWorkerConfig{
+			SpawnIndex: 1,
+			AgentConfiguration: AgentConfiguration{
+				BootstrapScript: dummyBootstrap,
+				BuildPath:       buildPath,
+				HooksPath:       hooksPath,
+				AcquireJob:      job.Job.ID,
+			},
+		},
+	)
+	worker.noWaitBetweenPingsForTesting = true
+
+	if err := worker.Start(t.Context(), nil); err != nil {
+		t.Errorf("worker.Start() = %v", err)
+	}
+
+	if got, want := agent.Pings, 0; got != want {
+		t.Errorf("agent.Pings = %d, want %d", got, want)
+	}
+	if got, want := job.State, JobStateFinished; got != want {
+		t.Errorf("job.State = %q, want %q", got, want)
+	}
+}
+
+func TestAgentWorker_Streaming_DisconnectAfterJob_Start_Pause_Unpause(t *testing.T) {
+	t.Parallel()
+
+	buildPath := filepath.Join(os.TempDir(), t.Name(), "build")
+	hooksPath := filepath.Join(os.TempDir(), t.Name(), "hooks")
+	if err := errors.Join(os.MkdirAll(buildPath, 0o777), os.MkdirAll(hooksPath, 0o777)); err != nil {
+		t.Fatalf("Couldn't create directories: %v", err)
+	}
+	t.Cleanup(func() {
+		os.RemoveAll(filepath.Join(os.TempDir(), t.Name())) //nolint:errcheck // Best-effort cleanup
+	})
+
+	server := NewFakeAPIServer(WithStreaming)
+	defer server.Close()
+
+	job := server.AddJob(map[string]string{
+		"BUILDKITE_COMMAND": "echo echo",
+	})
+
+	// Pre-register the agent.
+	const agentSessionToken = "alpacas"
+	agent := server.AddAgent(agentSessionToken)
+	agent.PingHandler = func(*http.Request) (api.Ping, error) {
+		return api.Ping{}, errors.New("too many pings")
+	}
+	go func() {
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_JobAssigned{
+				JobAssigned: &agentedgev1.JobAssignedAction{
+					Job: &agentedgev1.Job{
+						Id: job.Job.ID,
+					},
+				},
+			},
+		}
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Pause{},
+		}
+		agent.PingStream <- &agentedgev1.StreamPingsResponse{
+			Action: &agentedgev1.StreamPingsResponse_Resume{},
+		}
+		close(agent.PingStream)
+	}()
+
+	server.Assign(agent, job)
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(int) {})
+
+	worker := NewAgentWorker(
+		l,
+		&api.AgentRegisterResponse{
+			UUID:              uuid.New().String(),
+			Name:              "agent-1",
+			AccessToken:       "alpacas",
+			Endpoint:          server.URL,
+			PingInterval:      1,
+			JobStatusInterval: 1,
+			HeartbeatInterval: 10,
+		},
+		metrics.NewCollector(logger.Discard, metrics.CollectorConfig{}),
+		api.NewClient(logger.Discard, api.Config{
+			Endpoint: server.URL,
+			Token:    "llamas",
+		}),
+		AgentWorkerConfig{
+			SpawnIndex: 1,
+			AgentConfiguration: AgentConfiguration{
+				BootstrapScript:    dummyBootstrap,
+				BuildPath:          buildPath,
+				HooksPath:          hooksPath,
+				DisconnectAfterJob: true,
+			},
+		},
+	)
+	worker.noWaitBetweenPingsForTesting = true
+
+	if err := worker.Start(t.Context(), nil); err != nil {
+		t.Errorf("worker.Start() = %v", err)
+	}
+
+	if got, want := agent.Pings, 0; got != want {
+		t.Errorf("agent.Pings = %d, want %d", got, want)
+	}
+	if got, want := agent.IgnoreInDispatches, true; got != want {
+		t.Errorf("agent.IgnoreInDispatches = %t, want %t", got, want)
+	}
+	if got, want := job.State, JobStateFinished; got != want {
+		t.Errorf("job.State = %q, want %q", got, want)
+	}
+}
+
+func TestAgentWorker_Streaming_UnrecoverableError_Fallback(t *testing.T) {
+	t.Parallel()
+
+	const agentSessionToken = "alpacas"
+
+	server := NewFakeAPIServer(WithStreaming)
+	defer server.Close()
+
+	agent := server.AddAgent(agentSessionToken)
+	agent.PingHandler = func(req *http.Request) (api.Ping, error) {
+		switch agent.Pings {
+		case 0:
+			return api.Ping{Action: "disconnect"}, nil
+		default:
+			return api.Ping{}, fmt.Errorf("unexpected ping #%d", agent.Pings)
+		}
+	}
+	agent.PingStreamHandler = func(ctx context.Context, r *connect.Request[agentedgev1.StreamPingsRequest], ss *connect.ServerStream[agentedgev1.StreamPingsResponse]) error {
+		return connect.NewError(connect.CodePermissionDenied, errors.New("flagrant system error"))
+	}
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(int) {})
+
+	worker := NewAgentWorker(
+		l,
+		&api.AgentRegisterResponse{
+			UUID:              uuid.New().String(),
+			Name:              "agent-1",
+			AccessToken:       agentSessionToken,
+			Endpoint:          server.URL,
+			PingInterval:      1,
+			JobStatusInterval: 5,
+			HeartbeatInterval: 60,
+		},
+		metrics.NewCollector(logger.Discard, metrics.CollectorConfig{}),
+		api.NewClient(logger.Discard, api.Config{
+			Endpoint: server.URL,
+			Token:    "llamas",
+		}),
+		AgentWorkerConfig{},
+	)
+	worker.noWaitBetweenPingsForTesting = true
+
+	if err := worker.Start(t.Context(), nil); err != nil {
+		t.Errorf("worker.Start() = %v, want nil", err)
+	}
+
+	if got, want := agent.Pings, 1; got != want {
+		t.Errorf("agent.Pings = %d, want %d", got, want)
+	}
+}
+
+func TestAgentWorker_Streaming_RecoverableError_Fallback_Resume(t *testing.T) {
+	t.Parallel()
+
+	const agentSessionToken = "alpacas"
+
+	server := NewFakeAPIServer(WithStreaming)
+	defer server.Close()
+
+	agent := server.AddAgent(agentSessionToken)
+	// Default ping handler - idle
+
+	connections := 0
+	agent.PingStreamHandler = func(ctx context.Context, req *connect.Request[agentedgev1.StreamPingsRequest], resp *connect.ServerStream[agentedgev1.StreamPingsResponse]) error {
+		connections++
+		switch connections {
+		case 1:
+			return connect.NewError(connect.CodeUnavailable, errors.New("demure system error"))
+		case 2:
+			return resp.Send(&agentedgev1.StreamPingsResponse{
+				Action: &agentedgev1.StreamPingsResponse_Disconnect{},
+			})
+		default:
+			return connect.NewError(connect.CodeInternal, errors.New("too many connections"))
+		}
+	}
+
+	l := logger.NewConsoleLogger(logger.NewTestPrinter(t), func(int) {})
+
+	worker := NewAgentWorker(
+		l,
+		&api.AgentRegisterResponse{
+			UUID:              uuid.New().String(),
+			Name:              "agent-1",
+			AccessToken:       agentSessionToken,
+			Endpoint:          server.URL,
+			PingInterval:      1,
+			JobStatusInterval: 5,
+			HeartbeatInterval: 60,
+		},
+		metrics.NewCollector(logger.Discard, metrics.CollectorConfig{}),
+		api.NewClient(logger.Discard, api.Config{
+			Endpoint: server.URL,
+			Token:    "llamas",
+		}),
+		AgentWorkerConfig{},
+	)
+	worker.noWaitBetweenPingsForTesting = true
+
+	if err := worker.Start(t.Context(), nil); err != nil {
+		t.Errorf("worker.Start() = %v, want nil", err)
+	}
+
+	if connections != 2 {
+		t.Errorf("StreamPings connections = %d, want %d", connections, 2)
+	}
+}
diff --git a/agent/fake_api_server_test.go b/agent/fake_api_server_test.go
index 295cd070cb..29e574d4ce 100644
--- a/agent/fake_api_server_test.go
+++ b/agent/fake_api_server_test.go
@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,7 +11,10 @@ import (
 	"sync"
 	"time"
 
+	"connectrpc.com/connect"
 	"github.com/buildkite/agent/v3/api"
+	agentedgev1 "github.com/buildkite/agent/v3/api/proto/gen"
+	"github.com/buildkite/agent/v3/api/proto/gen/agentedgev1connect"
 	"github.com/google/uuid"
 )
 
@@ -46,6 +50,14 @@ type FakeAgent struct {
 	IgnoreInDispatches bool
 
 	PingHandler func(*http.Request) (api.Ping, error)
+
+	// PingStream is a simple way of providing streaming responses concurrently.
+	// It is used for the default handler.
+	PingStream chan *agentedgev1.StreamPingsResponse
+
+	// PingStreamHandler provides more flexibility in how the streaming request
+	// is handled. Setting PingStreamHandler overrides the default handler.
+	PingStreamHandler func(context.Context, *connect.Request[agentedgev1.StreamPingsRequest], *connect.ServerStream[agentedgev1.StreamPingsResponse]) error
 }
 
 // agentJob is just an agent/job tuple.
@@ -54,6 +66,8 @@ type agentJob struct {
 	job   *FakeJob
 }
 
+type fakeAPIServerOption = func(*FakeAPIServer, *http.ServeMux)
+
 // FakeAPIServer implements a fake Agent REST API server for testing.
 type FakeAPIServer struct {
 	*httptest.Server
@@ -68,7 +82,7 @@ type FakeAPIServer struct {
 }
 
 // NewFakeAPIServer constructs a new FakeAPIServer for testing.
-func NewFakeAPIServer() *FakeAPIServer {
+func NewFakeAPIServer(opts ...fakeAPIServerOption) *FakeAPIServer {
 	fs := &FakeAPIServer{
 		agents:        make(map[string]*FakeAgent),
 		jobs:          make(map[string]*FakeJob),
@@ -76,6 +90,9 @@ func NewFakeAPIServer() *FakeAPIServer {
 		registrations: make(map[string]*api.AgentRegisterResponse),
 	}
 	mux := http.NewServeMux()
+	for _, opt := range opts {
+		opt(fs, mux)
+	}
 	mux.HandleFunc("PUT /jobs/{job_uuid}/acquire", fs.handleJobAcquire)
 	mux.HandleFunc("PUT /jobs/{job_uuid}/accept", fs.handleJobAccept)
 	mux.HandleFunc("PUT /jobs/{job_uuid}/start", fs.handleJobStart)
@@ -88,10 +105,36 @@ func NewFakeAPIServer() *FakeAPIServer {
 	return fs
 }
 
+// WithStreaming enables the ping streaming API for the fake server.
+func WithStreaming(fs *FakeAPIServer, mux *http.ServeMux) {
+	mux.Handle(agentedgev1connect.NewAgentEdgeServiceHandler(fs))
+}
+
+func (fs *FakeAPIServer) StreamPings(ctx context.Context, req *connect.Request[agentedgev1.StreamPingsRequest], resp *connect.ServerStream[agentedgev1.StreamPingsResponse]) error {
+	auth := req.Header().Get("Authorization")
+	agent := fs.agentForAuth(auth)
+	if agent == nil {
+		return connect.NewError(connect.CodePermissionDenied, fmt.Errorf("invalid Authorization header value %q", auth))
+	}
+
+	if agent.PingStreamHandler != nil {
+		return agent.PingStreamHandler(ctx, req, resp)
+	}
+
+	for p := range agent.PingStream {
+		if err := resp.Send(p); err != nil {
+			return connect.NewError(connect.CodeUnknown, err)
+		}
+	}
+	return nil
+}
+
 func (fs *FakeAPIServer) AddAgent(token string) *FakeAgent {
 	fs.agentsMu.Lock()
 	defer fs.agentsMu.Unlock()
-	a := &FakeAgent{}
+	a := &FakeAgent{
+		PingStream: make(chan *agentedgev1.StreamPingsResponse),
+	}
 	fs.agents["Token "+token] = a
 	return a
 }

From a71996f301332593f06ac19ba9fef93fb02ddef4 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 11 Feb 2026 16:39:47 +1100
Subject: [PATCH 213/242] Add E2E tests for new ping modes

---
 internal/e2e/basic_test.go | 56 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 55 insertions(+), 1 deletion(-)

diff --git a/internal/e2e/basic_test.go b/internal/e2e/basic_test.go
index 9862fb16b4..d8cc9434ab 100644
--- a/internal/e2e/basic_test.go
+++ b/internal/e2e/basic_test.go
@@ -3,8 +3,10 @@
 package e2e
 
 import (
+	"context"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestBasicE2E(t *testing.T) {
@@ -13,7 +15,59 @@ func TestBasicE2E(t *testing.T) {
 
 	tc.startAgent()
 	build := tc.triggerBuild()
-	state := tc.waitForBuild(ctx, build)
+
+	// It should take much less time than 1 minute to successfully run the job.
+	waitCtx, canc := context.WithTimeout(ctx, 1*time.Minute)
+	defer canc()
+
+	state := tc.waitForBuild(waitCtx, build)
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+
+	logs := tc.fetchLogs(ctx, build)
+	if !strings.Contains(logs, "hello world") {
+		t.Errorf("tc.fetchLogs(ctx, build %q) logs as follows, did not contain 'hello world'\n%s", build.ID, logs)
+	}
+}
+
+func TestBasicE2E_PingOnly(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, "basic_e2e.yaml")
+
+	tc.startAgent("--ping-mode=ping-only")
+	build := tc.triggerBuild()
+
+	// It should take much less time than 1 minute to successfully run the job.
+	waitCtx, canc := context.WithTimeout(ctx, 1*time.Minute)
+	defer canc()
+
+	state := tc.waitForBuild(waitCtx, build)
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+
+	logs := tc.fetchLogs(ctx, build)
+	if !strings.Contains(logs, "hello world") {
+		t.Errorf("tc.fetchLogs(ctx, build %q) logs as follows, did not contain 'hello world'\n%s", build.ID, logs)
+	}
+}
+
+func TestBasicE2E_StreamOnly(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, "basic_e2e.yaml")
+
+	tc.startAgent(
+		"--ping-mode=stream-only",
+		"--endpoint=https://agent-edge.buildkite.com/v3",
+	)
+	build := tc.triggerBuild()
+
+	// It should take much less time than 1 minute to successfully run the job.
+	waitCtx, canc := context.WithTimeout(ctx, 2*time.Minute)
+	defer canc()
+
+	state := tc.waitForBuild(waitCtx, build)
 	if got, want := state, "passed"; got != want {
 		t.Errorf("Build state = %q, want %q", got, want)
 	}

From d102a50cc56aaa041c12315d063686ffab675b9d Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 11 Feb 2026 16:39:47 +1100
Subject: [PATCH 214/242] Fix coverage reports missing many tests

---
 .buildkite/pipeline.yml                  | 23 ++++++++++-------------
 .buildkite/steps/test-coverage-report.sh |  2 +-
 .buildkite/steps/tests.sh                |  9 +++++++--
 3 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f2c9d90818..0debb0967b 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -50,7 +50,7 @@ steps:
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       plugins:
         - docker-compose#v4.14.0:
             config: .buildkite/docker-compose.yml
@@ -71,7 +71,7 @@ steps:
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
       plugins:
@@ -94,7 +94,7 @@ steps:
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_WINDOWS_QUEUE
       plugins:
@@ -113,7 +113,7 @@ steps:
       parallelism: 3
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
       plugins:
@@ -132,7 +132,7 @@ steps:
 
     - name: ":coverage: Test coverage report Linux ARM64"
       key: test-coverage-linux-arm64
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-arm64"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -144,12 +144,11 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-linux-arm64
+            download: "coverage-linux-arm64/**"
 
     - name: ":coverage: Test coverage report Linux AMD64"
       key: test-coverage-linux-amd64
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-amd64"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -161,12 +160,11 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-linux-amd64
+            download: "coverage-linux-amd64/**"
 
     - name: ":coverage: Test coverage report Linux ARM64 Race"
       key: test-coverage-linux-arm64-race
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-arm64-race"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -178,8 +176,7 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-race-linux-arm64
+            download: "coverage-linux-arm64-race/**"
 
     - label: ":writing_hand: Annotate with Test Failures"
       depends_on:
diff --git a/.buildkite/steps/test-coverage-report.sh b/.buildkite/steps/test-coverage-report.sh
index 28991f44e4..5294b7af79 100755
--- a/.buildkite/steps/test-coverage-report.sh
+++ b/.buildkite/steps/test-coverage-report.sh
@@ -2,5 +2,5 @@
 set -euo pipefail
 
 echo 'Producing coverage report'
-go tool covdata textfmt -i "coverage" -o cover.out
+go tool covdata textfmt -i "$1" -o cover.out
 go tool cover -html cover.out -o cover.html
diff --git a/.buildkite/steps/tests.sh b/.buildkite/steps/tests.sh
index 9ee0706bbd..eb949f116a 100755
--- a/.buildkite/steps/tests.sh
+++ b/.buildkite/steps/tests.sh
@@ -4,6 +4,11 @@ set -euo pipefail
 go version
 echo arch is "$(uname -m)"
 
+RACE=''
+if [[ $* == *-race* ]] ; then
+  RACE='-race'
+fi
+
 export BUILDKITE_TEST_ENGINE_SUITE_SLUG=buildkite-agent
 export BUILDKITE_TEST_ENGINE_TEST_RUNNER=gotest
 export BUILDKITE_TEST_ENGINE_RESULT_PATH="junit-${BUILDKITE_JOB_ID}.xml"
@@ -13,8 +18,8 @@ if [[ "$(go env GOOS)" == "windows" ]]; then
   # need a Windows VM to debug.
   export BUILDKITE_TEST_ENGINE_TEST_CMD="go tool gotestsum --junitfile={{resultPath}} -- -count=1 $* {{packages}}"
 else
-  mkdir -p coverage
-  COVERAGE_DIR="$PWD/coverage"
+  COVERAGE_DIR="${PWD}/coverage-$(go env GOOS)-$(go env GOARCH)${RACE}"
+  mkdir -p "${COVERAGE_DIR}"
   export BUILDKITE_TEST_ENGINE_TEST_CMD="go tool gotestsum --junitfile={{resultPath}} -- -count=1 -cover $* {{packages}} -test.gocoverdir=${COVERAGE_DIR}"
 fi
 

From 91f732c99a5c85a9388754639aa059e9d5afcb7e Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 11 Feb 2026 16:39:47 +1100
Subject: [PATCH 215/242] Split ping loop inner into new method

---
 agent/agent_worker_ping.go | 243 ++++++++++++++++++++-----------------
 1 file changed, 133 insertions(+), 110 deletions(-)

diff --git a/agent/agent_worker_ping.go b/agent/agent_worker_ping.go
index 64b9f110a7..2a10b22178 100644
--- a/agent/agent_worker_ping.go
+++ b/agent/agent_worker_ping.go
@@ -24,18 +24,24 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, bat *baton, outCh chan<-
 	defer setStat("🛑 Ping loop stopped!")
 	setStat("🏃 Starting...")
 
-	// Create the ticker
 	pingInterval := time.Second * time.Duration(a.agent.PingInterval)
-	pingTicker := time.NewTicker(pingInterval)
-	defer pingTicker.Stop()
+	state := &pingLoopState{
+		AgentWorker:  a,
+		bat:          bat,
+		outCh:        outCh,
+		pingInterval: pingInterval,
+		pingTicker:   time.NewTicker(pingInterval),
+		skipWait:     make(chan struct{}, 1),
+		setStat:      setStat,
+	}
+	defer state.pingTicker.Stop()
 
 	// On the first iteration, skip waiting for the pingTicker.
-	// This doesn't skip the jitter, though.
-	skipWait := make(chan struct{}, 1)
-	skipWait <- struct{}{}
+	// One buffered value won't skip the jitter, though.
+	state.skipWait <- struct{}{}
 	if a.noWaitBetweenPingsForTesting {
 		// a closed channel will unblock the for/select instantly, for zero-delay ping loop testing.
-		close(skipWait)
+		close(state.skipWait)
 	}
 
 	a.logger.Info("Waiting for instructions...")
@@ -45,9 +51,9 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, bat *baton, outCh chan<-
 		a.logger.Debug("[runPingLoop] Waiting for pingTicker")
 		setStat("😴 Waiting until next ping interval tick")
 		select {
-		case <-skipWait:
+		case <-state.skipWait:
 			// continue below
-		case <-pingTicker.C:
+		case <-state.pingTicker.C:
 			// continue below
 		case <-a.stop:
 			a.logger.Debug("[runPingLoop] Stopping due to agent stop")
@@ -63,7 +69,7 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, bat *baton, outCh chan<-
 		a.logger.Debug("[runPingLoop] Waiting for jitter %v", jitter)
 		setStat(fmt.Sprintf("🫨 Jittering for %v", jitter))
 		select {
-		case <-skipWait:
+		case <-state.skipWait:
 			// continue below
 		case <-time.After(jitter):
 			// continue below
@@ -76,106 +82,8 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, bat *baton, outCh chan<-
 		}
 		pingWaitDurations.Observe(time.Since(startWait).Seconds())
 
-		// stop is only used internally when stopping.
-		stop := errors.New("stop")
-		err := func() error {
-			// Wait until the baton is available. If this takes forever, that's
-			// a good thing because it should mean the streaming loop is
-			// healthy.
-			// Once acquired, only release the baton after any work is complete,
-			// to prevent the streaming loop from resuming control until then,
-			// but we always release the baton, because the streaming loop is
-			// preferred.
-			a.logger.Debug("[runPingLoop] Waiting for baton")
-			select {
-			case <-bat.Acquire("ping"): // the baton is ours!
-				a.logger.Debug("[runPingLoop] Acquired the baton")
-				defer func() { // <- this is why the loop body is in a func
-					a.logger.Debug("[runPingLoop] Releasing the baton")
-					bat.Release("ping")
-				}()
-
-			case <-a.stop:
-				a.logger.Debug("[runPingLoop] Stopping due to agent stop")
-				return stop
-			case <-ctx.Done():
-				a.logger.Debug("[runPingLoop] Stopping due to context cancel")
-				return ctx.Err()
-			}
-
-			a.logger.Debug("[runPingLoop] Pinging buildkite for instructions")
-			setStat("📡 Pinging Buildkite for instructions")
-			pingsSent.Inc()
-			startPing := time.Now()
-			jobID, action, err := a.Ping(ctx)
-			if err != nil {
-				pingErrors.Inc()
-				if isUnrecoverable(err) {
-					a.logger.Error("%v", err)
-					return err
-				}
-				a.logger.Warn("%v", err)
-			}
-			pingDurations.Observe(time.Since(startPing).Seconds())
-
-			a.logger.Debug("[runPingLoop] Sending action")
-
-			// Send the action to the action loop
-			errCh := make(chan error)
-			msg := actionMessage{
-				action: action,
-				jobID:  jobID,
-				errCh:  errCh,
-			}
-			select {
-			case outCh <- msg:
-				// sent!
-			case <-a.stop:
-				a.logger.Debug("[runPingLoop] Stopping due to agent stop")
-				return stop
-			case <-ctx.Done():
-				a.logger.Debug("[runPingLoop] Stopping due to context cancel")
-				return ctx.Err()
-			}
-
-			// Wait for completion
-			select {
-			case err := <-errCh:
-				if err != nil || jobID == "" {
-					// We don't terminate the ping loop just because the
-					// action (usually a job) has failed.
-					return nil
-				}
-				if a.noWaitBetweenPingsForTesting {
-					// Don't bother resetting the ticker,
-					// don't try to send on a closed channel (skipWait).
-					return nil
-				}
-				// A job ran (or was at least started) successfully.
-				// Observation: jobs are rarely the last within a pipeline,
-				// thus if this worker just completed a job,
-				// there is likely another immediately available.
-				// Skip waiting for the ping interval until
-				// a ping without a job has occurred,
-				// but in exchange, ensure the next ping must wait at least a full
-				// pingInterval to avoid too much server load.
-				pingTicker.Reset(pingInterval)
-				select {
-				case skipWait <- struct{}{}:
-					// Ticker will be skipped
-				default:
-					// We're already skipping the ticker, don't block.
-				}
-				return nil
-			case <-a.stop:
-				a.logger.Debug("[runPingLoop] Stopping due to agent stop")
-				return stop
-			case <-ctx.Done():
-				a.logger.Debug("[runPingLoop] Stopping due to context cancel")
-				return ctx.Err()
-			}
-		}()
-		if err == stop {
+		err := state.pingLoopInner(ctx)
+		if err == internalStop {
 			return nil
 		}
 		if err != nil {
@@ -184,6 +92,121 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, bat *baton, outCh chan<-
 	}
 }
 
+// Internal error values that should not escape to the user.
+var (
+	// internalStop is used when stopping.
+	internalStop = errors.New("stop")
+)
+
+// pingLoopState exists to pass parameters to pingLoopInner.
+type pingLoopState struct {
+	*AgentWorker
+	bat          *baton
+	outCh        chan<- actionMessage
+	setStat      func(string)
+	pingTicker   *time.Ticker
+	pingInterval time.Duration
+	skipWait     chan struct{}
+}
+
+func (a *pingLoopState) pingLoopInner(ctx context.Context) error {
+	// Wait until the baton is available. If this takes forever, that's
+	// a good thing because it should mean the streaming loop is
+	// healthy.
+	// Once acquired, only release the baton after any work is complete,
+	// to prevent the streaming loop from resuming control until then,
+	// but we always release the baton, because the streaming loop is
+	// preferred.
+	a.logger.Debug("[runPingLoop] Waiting for baton")
+	select {
+	case <-a.bat.Acquire("ping"): // the baton is ours!
+		a.logger.Debug("[runPingLoop] Acquired the baton")
+		defer func() { // <- this is why the ping loop body is in a func
+			a.logger.Debug("[runPingLoop] Releasing the baton")
+			a.bat.Release("ping")
+		}()
+
+	case <-a.stop:
+		a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+		return internalStop
+	case <-ctx.Done():
+		a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+		return ctx.Err()
+	}
+
+	a.logger.Debug("[runPingLoop] Pinging buildkite for instructions")
+	a.setStat("📡 Pinging Buildkite for instructions")
+	pingsSent.Inc()
+	startPing := time.Now()
+	jobID, action, err := a.Ping(ctx)
+	if err != nil {
+		pingErrors.Inc()
+		if isUnrecoverable(err) {
+			a.logger.Error("%v", err)
+			return err
+		}
+		a.logger.Warn("%v", err)
+	}
+	pingDurations.Observe(time.Since(startPing).Seconds())
+
+	a.logger.Debug("[runPingLoop] Sending action")
+
+	// Send the action to the action loop
+	errCh := make(chan error)
+	msg := actionMessage{
+		action: action,
+		jobID:  jobID,
+		errCh:  errCh,
+	}
+	select {
+	case a.outCh <- msg:
+		// sent!
+	case <-a.stop:
+		a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+		return internalStop
+	case <-ctx.Done():
+		a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+		return ctx.Err()
+	}
+
+	// Wait for completion
+	select {
+	case err := <-errCh:
+		if err != nil || jobID == "" {
+			// We don't terminate the ping loop just because the
+			// action (usually a job) has failed.
+			return nil
+		}
+		if a.noWaitBetweenPingsForTesting {
+			// Don't bother resetting the ticker,
+			// don't try to send on a closed channel (skipWait).
+			return nil
+		}
+		// A job ran (or was at least started) successfully.
+		// Observation: jobs are rarely the last within a pipeline,
+		// thus if this worker just completed a job,
+		// there is likely another immediately available.
+		// Skip waiting for the ping interval until
+		// a ping without a job has occurred,
+		// but in exchange, ensure the next ping must wait at least a full
+		// pingInterval to avoid too much server load.
+		a.pingTicker.Reset(a.pingInterval)
+		select {
+		case a.skipWait <- struct{}{}:
+			// Ticker will be skipped
+		default:
+			// We're already skipping the ticker, don't block.
+		}
+		return nil
+	case <-a.stop:
+		a.logger.Debug("[runPingLoop] Stopping due to agent stop")
+		return internalStop
+	case <-ctx.Done():
+		a.logger.Debug("[runPingLoop] Stopping due to context cancel")
+		return ctx.Err()
+	}
+}
+
 // Performs a ping that checks Buildkite for a job or action to take
 // Returns a job, or nil if none is found
 func (a *AgentWorker) Ping(ctx context.Context) (jobID, action string, err error) {

From de05c088350785ef7f678f473fc1a24c39e99375 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 11 Feb 2026 16:39:47 +1100
Subject: [PATCH 216/242] Make "ping" and "debouncer" constants

---
 agent/agent_worker.go           | 9 +++++++--
 agent/agent_worker_debouncer.go | 8 ++++----
 agent/agent_worker_ping.go      | 4 ++--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 1cfbc60fd2..1cee72ac7d 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -325,7 +325,7 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	switch a.agentConfiguration.PingMode {
 	case "", "auto":
 		loops = []func(){pingLoop, streamingLoop, debouncerLoop}
-		bat.Acquire("debouncer")
+		bat.Acquire(actorDebouncer)
 
 	case "ping-only":
 		loops = []func(){pingLoop}
@@ -334,7 +334,7 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	case "stream-only":
 		loops = []func(){streamingLoop, debouncerLoop}
 		fromPingLoopCh = nil // prevent action loop listening to ping side
-		bat.Acquire("debouncer")
+		bat.Acquire(actorDebouncer)
 	}
 
 	// There's always an action handler.
@@ -557,6 +557,11 @@ func (a *AgentWorker) healthHandler() http.HandlerFunc {
 	}
 }
 
+const (
+	actorPingLoop  = "ping"
+	actorDebouncer = "debouncer"
+)
+
 type actionMessage struct {
 	// Details of the action to execute
 	action, jobID string
diff --git a/agent/agent_worker_debouncer.go b/agent/agent_worker_debouncer.go
index e928d107b2..f09cc0b000 100644
--- a/agent/agent_worker_debouncer.go
+++ b/agent/agent_worker_debouncer.go
@@ -41,7 +41,7 @@ func (a *AgentWorker) runDebouncer(ctx context.Context, bat *baton, outCh chan<-
 	// We begin holding the baton, ensure it is released when we exit.
 	defer func() {
 		a.logger.Debug("[runDebouncer] Releasing the baton")
-		bat.Release("debouncer")
+		bat.Release(actorDebouncer)
 	}()
 
 	// lastActionResult is closed when the action handler is done handling the
@@ -67,7 +67,7 @@ func (a *AgentWorker) runDebouncer(ctx context.Context, bat *baton, outCh chan<-
 			a.logger.Debug("[runDebouncer] Stopping due to context cancel")
 			return ctx.Err()
 
-		case <-iif(healthy, bat.Acquire("debouncer")): // if the stream is healthy, take the baton if available
+		case <-iif(healthy, bat.Acquire(actorDebouncer)): // if the stream is healthy, take the baton if available
 			a.logger.Debug("[runDebouncer] Took the baton")
 			// We now have the baton!
 			// continue below to send any pending message, if able
@@ -88,7 +88,7 @@ func (a *AgentWorker) runDebouncer(ctx context.Context, bat *baton, outCh chan<-
 				if !actionInProgress {
 					// We can release the baton now.
 					a.logger.Debug("[runDebouncer] Releasing the baton")
-					bat.Release("debouncer")
+					bat.Release(actorDebouncer)
 				}
 				break // out of the select
 			}
@@ -111,7 +111,7 @@ func (a *AgentWorker) runDebouncer(ctx context.Context, bat *baton, outCh chan<-
 
 		// If we're healthy, have the baton, there's no action in progress,
 		// and there's a pending message, then send that message.
-		if !healthy || !bat.HeldBy("debouncer") || actionInProgress || pending == nil {
+		if !healthy || !bat.HeldBy(actorDebouncer) || actionInProgress || pending == nil {
 			continue
 		}
 		a.logger.Debug("[runDebouncer] Sending action %q, jobID %q", pending.action, pending.jobID)
diff --git a/agent/agent_worker_ping.go b/agent/agent_worker_ping.go
index 2a10b22178..22030db62f 100644
--- a/agent/agent_worker_ping.go
+++ b/agent/agent_worker_ping.go
@@ -119,11 +119,11 @@ func (a *pingLoopState) pingLoopInner(ctx context.Context) error {
 	// preferred.
 	a.logger.Debug("[runPingLoop] Waiting for baton")
 	select {
-	case <-a.bat.Acquire("ping"): // the baton is ours!
+	case <-a.bat.Acquire(actorPingLoop): // the baton is ours!
 		a.logger.Debug("[runPingLoop] Acquired the baton")
 		defer func() { // <- this is why the ping loop body is in a func
 			a.logger.Debug("[runPingLoop] Releasing the baton")
-			a.bat.Release("ping")
+			a.bat.Release(actorPingLoop)
 		}()
 
 	case <-a.stop:

From 9116080e1f4b586aee47dddf44ca05f3ef402c83 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 11 Feb 2026 16:39:47 +1100
Subject: [PATCH 217/242] Add more comments to explain baton implementation

---
 agent/baton.go | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/agent/baton.go b/agent/baton.go
index 2eed183de5..0603935a75 100644
--- a/agent/baton.go
+++ b/agent/baton.go
@@ -10,14 +10,15 @@ type baton struct {
 	acquire map[string]chan struct{}
 }
 
-// newBaton creates a new baton for sharing among (non-zero) values of T.
+// newBaton creates a new baton for sharing among actors, each identified
+// by a non-empty string.
 func newBaton() *baton {
 	return &baton{
 		acquire: make(map[string]chan struct{}),
 	}
 }
 
-// HeldBy reports if the argument holds the baton.
+// HeldBy reports if the actor specified by the argument holds the baton.
 func (b *baton) HeldBy(by string) bool {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -30,11 +31,15 @@ func (b *baton) Acquire(by string) <-chan struct{} {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
+	// If there's an existing channel for this actor, reuse it.
 	ch := b.acquire[by]
 	if ch == nil {
 		ch = make(chan struct{})
 	}
 
+	// If nothing holds the baton currently, assign it to the caller.
+	// The caller won't be receiving on the channel until after we
+	// return it, so make the channel receivable by closing it.
 	if b.holder == "" {
 		b.holder = by
 		close(ch)
@@ -42,6 +47,8 @@ func (b *baton) Acquire(by string) <-chan struct{} {
 		return ch
 	}
 
+	// Something holds the baton, so record that this actor is
+	// waiting for the baton.
 	b.acquire[by] = ch
 	return ch
 }
@@ -51,22 +58,29 @@ func (b *baton) Release(by string) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
+	// Only release if its the same actor, to prevent bugs due
+	// to double-releasing.
 	if b.holder != by {
 		return
 	}
 
-	// Attempt to pass the baton.
+	// Attempt to pass the baton to anything still waiting for it.
 	for a, ch := range b.acquire {
 		delete(b.acquire, a)
 		select {
 		case ch <- struct{}{}:
-			// This acquirer has acquired the baton.
+			// We were able to send a value to the channel,
+			// so this actor was still waiting to receive.
+			// Therefore this actor has acquired the baton.
 			b.holder = a
 			return
 		default:
-			// This acquirer has stopped waiting, try another.
+			// This actor has stopped waiting to receive,
+			// so try another.
 		}
 	}
-	// Nothing was waiting, nothing now holds the baton.
+
+	// Nothing was still waiting on its channel,
+	// so now nothing holds the baton.
 	b.holder = ""
 }

From dde74653678986c366d995e4c739fc767632ac78 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 11 Feb 2026 16:39:47 +1100
Subject: [PATCH 218/242] Contextualise streaming logs on pause and disconnect
 actions

---
 agent/agent_worker_streaming.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/agent/agent_worker_streaming.go b/agent/agent_worker_streaming.go
index 1a45451078..351306f478 100644
--- a/agent/agent_worker_streaming.go
+++ b/agent/agent_worker_streaming.go
@@ -148,13 +148,13 @@ func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- act
 
 				case *agentedgev1.StreamPingsResponse_Pause:
 					if reason := act.Pause.GetReason(); reason != "" {
-						a.logger.Info("%s", reason)
+						a.logger.Info("Pause reason: %s", reason)
 					}
 					amsg.action = "pause"
 
 				case *agentedgev1.StreamPingsResponse_Disconnect:
 					if reason := act.Disconnect.GetReason(); reason != "" {
-						a.logger.Info("%s", reason)
+						a.logger.Info("Disconnect reason: %s", reason)
 					}
 					amsg.action = "disconnect"
 

From 8b0bbcbd26249693317a9642d55abdee72f034cf Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 3 Mar 2026 18:10:13 +1100
Subject: [PATCH 219/242] Split up streaming loop a bit

---
 agent/agent_worker.go           |  10 ++
 agent/agent_worker_ping.go      |   7 -
 agent/agent_worker_streaming.go | 254 ++++++++++++++++++--------------
 3 files changed, 157 insertions(+), 114 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 1cee72ac7d..c218af65c2 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -557,6 +557,16 @@ func (a *AgentWorker) healthHandler() http.HandlerFunc {
 	}
 }
 
+// Internal error values that should not escape to the user.
+var (
+	// internalStop is used when stopping.
+	internalStop = errors.New("stop")
+
+	// internalBreak is used to stop an inner loop but continue
+	// an outer loop.
+	internalBreak = errors.New("break")
+)
+
 const (
 	actorPingLoop  = "ping"
 	actorDebouncer = "debouncer"
diff --git a/agent/agent_worker_ping.go b/agent/agent_worker_ping.go
index 22030db62f..7e4957ad03 100644
--- a/agent/agent_worker_ping.go
+++ b/agent/agent_worker_ping.go
@@ -2,7 +2,6 @@ package agent
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"math/rand/v2"
 	"time"
@@ -92,12 +91,6 @@ func (a *AgentWorker) runPingLoop(ctx context.Context, bat *baton, outCh chan<-
 	}
 }
 
-// Internal error values that should not escape to the user.
-var (
-	// internalStop is used when stopping.
-	internalStop = errors.New("stop")
-)
-
 // pingLoopState exists to pass parameters to pingLoopInner.
 type pingLoopState struct {
 	*AgentWorker
diff --git a/agent/agent_worker_streaming.go b/agent/agent_worker_streaming.go
index 351306f478..f36c3564f5 100644
--- a/agent/agent_worker_streaming.go
+++ b/agent/agent_worker_streaming.go
@@ -48,7 +48,6 @@ func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- act
 	// Note: This _could_ be implemented with an infinite loop containing a roko
 	// retrier, but it looked a bit messier to me.
 	initialMaxJitter := 1 * time.Second
-	attempts := 0
 
 	var skipWait chan struct{}
 	if a.noWaitBetweenPingsForTesting {
@@ -57,11 +56,17 @@ func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- act
 		close(skipWait)
 	}
 
+	state := &streamLoopState{
+		AgentWorker: a,
+		outCh:       outCh,
+		setStat:     setStat,
+	}
+
 	for {
 		// Backoff exponentially, up to initialMaxJitter * 2^6.
 		// (Repeated failures may jitter up to 64 seconds between attempts.)
-		maxJitter := initialMaxJitter << min(attempts, 6)
-		attempts++
+		maxJitter := initialMaxJitter << min(state.attempts, 6)
+		state.attempts++
 
 		// Within the interval, wait a random amount of time to avoid
 		// spontaneous synchronisation across agents.
@@ -81,121 +86,156 @@ func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- act
 			return ctx.Err()
 		}
 
-		setStat(fmt.Sprintf("📱 Connecting to ping stream (attempt %d)...", attempts))
-		a.logger.Debug("[runStreamingPingLoop] Connecting (attempt %d)", attempts)
-		stream, err := a.apiClient.StreamPings(streamCtx, a.agent.UUID)
+		err := state.startStream(ctx, streamCtx)
+		if err == internalStop {
+			return nil
+		}
 		if err != nil {
-			a.logger.Error("Connection to ping stream failed: %v", err)
-			if isUnrecoverable(err) {
-				a.logger.Error("Stopping ping stream because the error is unrecoverable")
-				return err
-			}
-			// Fast fallback to the ping loop
-			a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
-			select {
-			case outCh <- actionMessage{unhealthy: true}:
-				a.logger.Debug("[runStreamingPingLoop] Unhealthy message sent to debouncer")
-				// sent!
-			case <-a.stop:
-				a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
-				return nil
-			case <-ctx.Done():
-				a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
-				return ctx.Err()
-			}
-			continue
+			return err
 		}
+	}
+}
 
-		firstMsg := true // used for the "connection established" log
-
-		setStat("🏞️ Streaming actions from Buildkite")
-		a.logger.Debug("[runStreamingPingLoop] Waiting for a message...")
-	streamLoop:
-		for msg, err := range stream {
-			a.logger.Debug("[runStreamingPingLoop] Received msg %v, err %v", msg, err)
-
-			var amsg actionMessage
-			switch {
-			case err != nil:
-				a.logger.Debug("[runStreamingPingLoop] Connection to ping stream failed or ended: %v", err)
-				if isUnrecoverable(err) {
-					a.logger.Error("Stopping ping stream loop because the error is unrecoverable: %v", err)
-					return err
-				}
-				// Go unhealthy, unless the error is deadline-exceeded.
-				// (The connection timed out, which we want to happen every so often).
-				if connect.CodeOf(err) == connect.CodeDeadlineExceeded {
-					a.logger.Debug("[runStreamingPingLoop] Breaking streamLoop to reconnect after deadline-exceeded")
-					break streamLoop
-				}
-				a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
-				amsg.unhealthy = true
+// streamLoopState holds stream loop specific state for startStream
+// and streamLoopInner.
+type streamLoopState struct {
+	*AgentWorker
+	outCh    chan<- actionMessage
+	attempts int
+	firstMsg bool
+	setStat  func(string)
+}
 
-			case msg == nil:
-				a.logger.Error("Ping stream yielded a nil message, so assuming the stream is broken")
-				a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
-				amsg.unhealthy = true
+// startStream attempts 1 connection to the stream and handles its messages.
+func (a *streamLoopState) startStream(ctx, streamCtx context.Context) error {
+	a.setStat(fmt.Sprintf("📱 Connecting to ping stream (attempt %d)...", a.attempts))
+	a.logger.Debug("[runStreamingPingLoop] Connecting (attempt %d)", a.attempts)
+	stream, err := a.apiClient.StreamPings(streamCtx, a.agent.UUID)
+	if err != nil {
+		a.logger.Error("Connection to ping stream failed: %v", err)
+		if isUnrecoverable(err) {
+			a.logger.Error("Stopping ping stream because the error is unrecoverable")
+			return err
+		}
+		// Fast fallback to the ping loop
+		a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+		select {
+		case a.outCh <- actionMessage{unhealthy: true}:
+			a.logger.Debug("[runStreamingPingLoop] Unhealthy message sent to debouncer")
+			// sent!
+		case <-a.stop:
+			a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
+			return internalStop
+		case <-ctx.Done():
+			a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
+			return ctx.Err()
+		}
+		return nil // continue outer streaming loop
+	}
 
-			default:
-				if firstMsg {
-					a.logger.Info("Ping stream connection established")
-					firstMsg = false
-				}
-
-				switch act := msg.Action.(type) {
-				case *agentedgev1.StreamPingsResponse_Resume: // a.k.a. "idle"
-					// continue below
-
-				case *agentedgev1.StreamPingsResponse_Pause:
-					if reason := act.Pause.GetReason(); reason != "" {
-						a.logger.Info("Pause reason: %s", reason)
-					}
-					amsg.action = "pause"
-
-				case *agentedgev1.StreamPingsResponse_Disconnect:
-					if reason := act.Disconnect.GetReason(); reason != "" {
-						a.logger.Info("Disconnect reason: %s", reason)
-					}
-					amsg.action = "disconnect"
-
-				case *agentedgev1.StreamPingsResponse_JobAssigned:
-					amsg.jobID = act.JobAssigned.GetJob().GetId()
-					if amsg.jobID == "" {
-						a.logger.Error("Ping stream yielded a JobAssigned message with nil job or empty job ID, so assuming the stream is broken")
-						a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
-						amsg.unhealthy = true
-					}
-				}
-			}
+	a.firstMsg = true // used for the "connection established" log
 
-			// Send the message to the debouncer.
-			select {
-			case outCh <- amsg:
-				a.logger.Debug("[runStreamingPingLoop] Message sent to debouncer")
-				// sent!
-			case <-a.stop:
-				a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
-				return nil
-			case <-ctx.Done():
-				a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
-				return ctx.Err()
+	a.setStat("🏞️ Streaming actions from Buildkite")
+	a.logger.Debug("[runStreamingPingLoop] Waiting for a message...")
+	for msg, streamErr := range stream {
+		err := a.handle(ctx, msg, streamErr)
+		if err == internalBreak {
+			break
+		}
+		if err == internalStop {
+			return internalStop
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *streamLoopState) handle(ctx context.Context, msg *agentedgev1.StreamPingsResponse, streamErr error) error {
+	a.logger.Debug("[runStreamingPingLoop] Received msg %v, err %v", msg, streamErr)
+
+	var amsg actionMessage
+	switch {
+	case streamErr != nil:
+		a.logger.Debug("[runStreamingPingLoop] Connection to ping stream failed or ended: %v", streamErr)
+		if isUnrecoverable(streamErr) {
+			a.logger.Error("Stopping ping stream loop because the error is unrecoverable: %v", streamErr)
+			return streamErr
+		}
+		// Stay healthy if the error is deadline-exceeded.
+		// (The connection timed out, which we want to happen every so often).
+		if connect.CodeOf(streamErr) == connect.CodeDeadlineExceeded {
+			a.logger.Debug("[runStreamingPingLoop] Breaking stream loop to reconnect following deadline-exceeded")
+			return internalBreak
+		}
+		// It's some other error. Go unhealthy, which unblocks the ping loop.
+		a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+		amsg.unhealthy = true
+
+	case msg == nil:
+		a.logger.Error("Ping stream yielded a nil message, so assuming the stream is broken")
+		a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+		amsg.unhealthy = true
+
+	default:
+		if a.firstMsg {
+			a.logger.Info("Ping stream connection established")
+			a.firstMsg = false
+		}
+
+		switch act := msg.Action.(type) {
+		case *agentedgev1.StreamPingsResponse_Resume: // a.k.a. "idle"
+			// continue below
+
+		case *agentedgev1.StreamPingsResponse_Pause:
+			if reason := act.Pause.GetReason(); reason != "" {
+				a.logger.Info("Pause reason: %s", reason)
 			}
+			amsg.action = "pause"
 
-			// In case the server sends a disconnect but doesn't close the
-			// stream, be sure to exit.
-			if amsg.action == "disconnect" {
-				a.logger.Debug("[runStreamingPingLoop] Stopping due to disconnect action")
-				a.internalStop()
-				return nil
+		case *agentedgev1.StreamPingsResponse_Disconnect:
+			if reason := act.Disconnect.GetReason(); reason != "" {
+				a.logger.Info("Disconnect reason: %s", reason)
 			}
+			amsg.action = "disconnect"
 
-			if amsg.unhealthy {
-				a.logger.Debug("[runStreamingPingLoop] Breaking streamLoop to reconnect because the stream is unhealthy")
-				break streamLoop
-			} else {
-				// Stream is healthy, reset the retry counter
-				attempts = 0
+		case *agentedgev1.StreamPingsResponse_JobAssigned:
+			amsg.jobID = act.JobAssigned.GetJob().GetId()
+			if amsg.jobID == "" {
+				a.logger.Error("Ping stream yielded a JobAssigned message with nil job or empty job ID, so assuming the stream is broken")
+				a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
+				amsg.unhealthy = true
 			}
 		}
 	}
+
+	// Send the message to the debouncer.
+	select {
+	case a.outCh <- amsg:
+		a.logger.Debug("[runStreamingPingLoop] Message sent to debouncer")
+		// sent!
+	case <-a.stop:
+		a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
+		return internalStop
+	case <-ctx.Done():
+		a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
+		return ctx.Err()
+	}
+
+	// In case the server sends a disconnect but doesn't close the
+	// stream, be sure to exit.
+	if amsg.action == "disconnect" {
+		a.logger.Debug("[runStreamingPingLoop] Stopping due to disconnect action")
+		a.internalStop()
+		return internalStop
+	}
+
+	if amsg.unhealthy {
+		a.logger.Debug("[runStreamingPingLoop] Breaking stream loop to reconnect because the stream is unhealthy")
+		return internalBreak
+	}
+	// Stream is healthy, reset the retry counter
+	a.attempts = 0
+	return nil
 }

From 7a4b492d25142a0e4d5d670a954b3a4bb8284ce6 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 3 Mar 2026 18:10:13 +1100
Subject: [PATCH 220/242] Use proper jitter windowing

---
 agent/agent_worker_streaming.go | 35 +++++++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/agent/agent_worker_streaming.go b/agent/agent_worker_streaming.go
index f36c3564f5..3107c3d505 100644
--- a/agent/agent_worker_streaming.go
+++ b/agent/agent_worker_streaming.go
@@ -47,7 +47,7 @@ func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- act
 	//
 	// Note: This _could_ be implemented with an infinite loop containing a roko
 	// retrier, but it looked a bit messier to me.
-	initialMaxJitter := 1 * time.Second
+	initialWindow := 1 * time.Second
 
 	var skipWait chan struct{}
 	if a.noWaitBetweenPingsForTesting {
@@ -63,16 +63,17 @@ func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- act
 	}
 
 	for {
-		// Backoff exponentially, up to initialMaxJitter * 2^6.
+		// Backoff exponentially, up to initialWindow * 2^6.
 		// (Repeated failures may jitter up to 64 seconds between attempts.)
-		maxJitter := initialMaxJitter << min(state.attempts, 6)
+		window := initialWindow << min(state.attempts, 6)
+		windowEnd := time.After(window)
 		state.attempts++
 
 		// Within the interval, wait a random amount of time to avoid
 		// spontaneous synchronisation across agents.
-		jitter := rand.N(maxJitter)
-		setStat(fmt.Sprintf("🫨 Jittering for %v (max %v)", jitter, maxJitter))
-		a.logger.Debug("[runStreamingPingLoop] Waiting for jitter %v (max %v)", jitter, maxJitter)
+		jitter := rand.N(window)
+		setStat(fmt.Sprintf("🫨 Jittering for %v (max %v)", jitter, window))
+		a.logger.Debug("[runStreamingPingLoop] Waiting for jitter %v (max %v)", jitter, window)
 		select {
 		case <-skipWait:
 			// continue below
@@ -93,6 +94,28 @@ func (a *AgentWorker) runStreamingPingLoop(ctx context.Context, outCh chan<- act
 		if err != nil {
 			return err
 		}
+
+		// Wait the remainder of the jitter window.
+		// Windowing the jitter this way avoids statistical effects.
+		// (If we started a new jitter right away, the Nth request would
+		// happen at an approximately Normally-distributed time after start,
+		// because that's a sum of random variables each with finite variance.
+		// Central Limit Theorem! We'd rather have a uniform distribution
+		// over a window.)
+		setStat("😴 Waiting for remainder of window")
+		a.logger.Debug("[runStreamingPingLoop] Waiting for remainder of window")
+		select {
+		case <-skipWait:
+			// continue next iteration
+		case <-windowEnd:
+			// continue next iteration
+		case <-a.stop:
+			a.logger.Debug("[runStreamingPingLoop] Stopping due to agent stop")
+			return nil
+		case <-ctx.Done():
+			a.logger.Debug("[runStreamingPingLoop] Stopping due to context cancel")
+			return ctx.Err()
+		}
 	}
 }
 

From 1a7db49e12300e4042cc7f1793b71b87e32c28a8 Mon Sep 17 00:00:00 2001
From: CerealBoy <allan@buildkite.com>
Date: Wed, 4 Mar 2026 12:56:25 +1100
Subject: [PATCH 221/242] Bump version and CHANGELOG for v3.119.0

---
 CHANGELOG.md    | 15 +++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e9d86a43d7..286a598c1b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.119.0](https://github.com/buildkite/agent/tree/v3.119.0) (2026-03-03)
+
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.118.1...v3.119.0)
+
+### Added
+
+- Streaming pings [#3697](https://github.com/buildkite/agent/pull/3697) (@DrJosh9000)
+- PS-1663: log s3 credential source for visibility [#3723](https://github.com/buildkite/agent/pull/3723) (@zhming0)
+
+### Fixed
+
+- Fix false URL mismatch detection with insteadOf [#3718](https://github.com/buildkite/agent/pull/3718) (@rajatvig)
+- A-970: Skip git fetch for already-present commits during checkout [#3725](https://github.com/buildkite/agent/pull/3725) (@zhming0)
+- Fix codeowner, add one more owner [#3724](https://github.com/buildkite/agent/pull/3724) (@zhming0)
+
 ## [v3.118.1](https://github.com/buildkite/agent/tree/v3.118.1) (2026-02-25)
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.118.0...v3.118.1)
 
diff --git a/version/VERSION b/version/VERSION
index 5826da02b3..3c8b5bd6de 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.118.1
+3.119.0

From aa2e16e53ce5dbb9d32b8eeb256a9717d942a065 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Wed, 4 Mar 2026 13:42:12 +1100
Subject: [PATCH 222/242] Default ping-mode to ping-only for now

---
 clicommand/agent_start.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index bb3ff2d0f6..f0aaca4482 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -764,8 +764,8 @@ var AgentStartCommand = cli.Command{
 		// API + agent behaviour
 		cli.StringFlag{
 			Name:   "ping-mode",
-			Usage:  "Selects available protocols for dispatching work to this agent. One of auto (default), ping-only, stream-only.",
-			Value:  "auto",
+			Usage:  "Selects available protocols for dispatching work to this agent. One of ping-only (default), auto (prefer streaming, but fall back to polling when necessary) or stream-only.",
+			Value:  "ping-only",
 			EnvVar: "BUILDKITE_AGENT_PING_MODE",
 		},
 

From ed231f8756a17dc85a14b6a94a788606306cfc02 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Mar 2026 14:32:13 +1100
Subject: [PATCH 223/242] Validate ping mode flag, change ping-only to
 poll-only

---
 agent/agent_worker.go      | 15 ++++++++++++---
 clicommand/agent_start.go  | 22 ++++++++++++++++++++--
 internal/e2e/basic_test.go |  4 ++--
 3 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index c218af65c2..9bbbc725c7 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -20,6 +20,12 @@ import (
 	"github.com/buildkite/roko"
 )
 
+const (
+	PingModeAuto       = "auto" // empty string can be used from tests
+	PingModeStreamOnly = "stream-only"
+	PingModePollOnly   = "poll-only"
+)
+
 type AgentWorkerConfig struct {
 	// Whether to set debug in the job
 	Debug bool
@@ -323,18 +329,21 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 
 	var loops []func()
 	switch a.agentConfiguration.PingMode {
-	case "", "auto":
+	case "", PingModeAuto: // note: "" can happen in some tests
 		loops = []func(){pingLoop, streamingLoop, debouncerLoop}
 		bat.Acquire(actorDebouncer)
 
-	case "ping-only":
+	case PingModePollOnly:
 		loops = []func(){pingLoop}
 		fromDebouncerCh = nil // prevent action loop listening to streaming side
 
-	case "stream-only":
+	case PingModeStreamOnly:
 		loops = []func(){streamingLoop, debouncerLoop}
 		fromPingLoopCh = nil // prevent action loop listening to ping side
 		bat.Acquire(actorDebouncer)
+
+	default:
+		return fmt.Errorf("unknown ping mode %q", a.agentConfiguration.PingMode)
 	}
 
 	// There's always an action handler.
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index f0aaca4482..baf490d186 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -60,9 +60,18 @@ Example:
 
     $ buildkite-agent start --token xxx`
 
+const pingModePingOnly = "ping-only"
+
 var (
 	verificationFailureBehaviors = []string{agent.VerificationBehaviourBlock, agent.VerificationBehaviourWarn}
 
+	pingModes = []string{
+		agent.PingModeAuto,
+		agent.PingModePollOnly,
+		pingModePingOnly, // canonicalises to agent.PingModePollOnly
+		agent.PingModeStreamOnly,
+	}
+
 	buildkiteSetEnvironmentVariables = []*regexp.Regexp{
 		regexp.MustCompile("^BUILDKITE$"),
 		regexp.MustCompile("^BUILDKITE_.*$"),
@@ -764,8 +773,8 @@ var AgentStartCommand = cli.Command{
 		// API + agent behaviour
 		cli.StringFlag{
 			Name:   "ping-mode",
-			Usage:  "Selects available protocols for dispatching work to this agent. One of ping-only (default), auto (prefer streaming, but fall back to polling when necessary) or stream-only.",
-			Value:  "ping-only",
+			Usage:  "Selects available protocols for dispatching work to this agent. One of auto (default, prefer streaming, but fall back to polling when necessary), poll-only, or stream-only.",
+			Value:  "auto",
 			EnvVar: "BUILDKITE_AGENT_PING_MODE",
 		},
 
@@ -852,6 +861,15 @@ var AgentStartCommand = cli.Command{
 			return fmt.Errorf("failed to unset config from environment: %w", err)
 		}
 
+		if !slices.Contains(pingModes, cfg.PingMode) {
+			return fmt.Errorf("invalid ping mode %q, must be one of %v", cfg.PingMode, pingModes)
+		}
+		// Calling it "ping-only" was a mistake, so canonicalise it to "poll-only"
+		// on the very remote chance someone is using that.
+		if cfg.PingMode == pingModePingOnly {
+			cfg.PingMode = agent.PingModePollOnly
+		}
+
 		if cfg.VerificationJWKSFile != "" {
 			if !slices.Contains(verificationFailureBehaviors, cfg.VerificationFailureBehavior) {
 				return fmt.Errorf(
diff --git a/internal/e2e/basic_test.go b/internal/e2e/basic_test.go
index d8cc9434ab..39054dc744 100644
--- a/internal/e2e/basic_test.go
+++ b/internal/e2e/basic_test.go
@@ -31,11 +31,11 @@ func TestBasicE2E(t *testing.T) {
 	}
 }
 
-func TestBasicE2E_PingOnly(t *testing.T) {
+func TestBasicE2E_PollOnly(t *testing.T) {
 	ctx := t.Context()
 	tc := newTestCase(t, "basic_e2e.yaml")
 
-	tc.startAgent("--ping-mode=ping-only")
+	tc.startAgent("--ping-mode=poll-only")
 	build := tc.triggerBuild()
 
 	// It should take much less time than 1 minute to successfully run the job.

From 2480514c889328b53030b6aba85e994d978b949a Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Mar 2026 14:32:13 +1100
Subject: [PATCH 224/242] Log unrecoverable at error level in stream-only mode,
 info otherwise

---
 agent/agent_worker.go           | 26 +++++++++++++++++---------
 agent/agent_worker_streaming.go | 12 ++++++++----
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 9bbbc725c7..5625351056 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -310,15 +310,23 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	streamingLoop := func() {
 		defer wg.Done()
 		err := a.runStreamingPingLoop(ctx, fromStreamingLoopCh)
-		if a.agentConfiguration.PingMode != "streaming-only" {
-			// If the ping mode is streaming-only, then an unrecoverable failure
-			// in the streaming loop should be reported.
-			// Otherwise, it should fall back to the ping loop and carry on,
-			// and if that also has an unrecoverable failure we can report that.
-			// Said another way:
-			// Streaming is best-effort but preferred, unless in streaming-only
-			// mode (it's the only available option).
-			err = nil
+		if err != nil {
+			switch a.agentConfiguration.PingMode {
+			case PingModeStreamOnly:
+				// In streaming-only mode, an unrecoverable failure
+				// in the streaming loop should be reported and should
+				// terminate the agent worker.
+				a.logger.Error("Streaming ping mode failed due to an unrecoverable error: %v", err)
+			default:
+				// In auto mode, the worker should fall back to the ping loop
+				// and carry on. The user might find that interesting (especially if
+				// they are expecting streaming to work).
+				a.logger.Info("Streaming ping mode is unavailable, permanently falling back to polling-based ping mode (the underlying error was: %v)", err)
+				// If the ping loop then has its own unrecoverable error, then
+				// *that* will terminate the worker. But the streaming loop shouldn't.
+				// So treat the error from the streaming loop as "business as usual".
+				err = nil
+			}
 		}
 		errCh <- err
 	}
diff --git a/agent/agent_worker_streaming.go b/agent/agent_worker_streaming.go
index 3107c3d505..2d6fe4fd3f 100644
--- a/agent/agent_worker_streaming.go
+++ b/agent/agent_worker_streaming.go
@@ -135,9 +135,11 @@ func (a *streamLoopState) startStream(ctx, streamCtx context.Context) error {
 	a.logger.Debug("[runStreamingPingLoop] Connecting (attempt %d)", a.attempts)
 	stream, err := a.apiClient.StreamPings(streamCtx, a.agent.UUID)
 	if err != nil {
-		a.logger.Error("Connection to ping stream failed: %v", err)
+		// TODO: after we've made streaming endpoints generally available,
+		// think about making some of these logs error or warning level.
+		a.logger.Debug("[runStreamingPingLoop] Connection to ping stream failed: %v", err)
 		if isUnrecoverable(err) {
-			a.logger.Error("Stopping ping stream because the error is unrecoverable")
+			a.logger.Debug("[runStreamingPingLoop] Stopping because the error is unrecoverable")
 			return err
 		}
 		// Fast fallback to the ping loop
@@ -181,9 +183,11 @@ func (a *streamLoopState) handle(ctx context.Context, msg *agentedgev1.StreamPin
 	var amsg actionMessage
 	switch {
 	case streamErr != nil:
+		// TODO: after we've made streaming endpoints generally available,
+		// think about making some of these logs error or warning level.
 		a.logger.Debug("[runStreamingPingLoop] Connection to ping stream failed or ended: %v", streamErr)
 		if isUnrecoverable(streamErr) {
-			a.logger.Error("Stopping ping stream loop because the error is unrecoverable: %v", streamErr)
+			a.logger.Debug("[runStreamingPingLoop] Stopping because the error is unrecoverable")
 			return streamErr
 		}
 		// Stay healthy if the error is deadline-exceeded.
@@ -197,7 +201,7 @@ func (a *streamLoopState) handle(ctx context.Context, msg *agentedgev1.StreamPin
 		amsg.unhealthy = true
 
 	case msg == nil:
-		a.logger.Error("Ping stream yielded a nil message, so assuming the stream is broken")
+		a.logger.Debug("[runStreamingPingLoop] Ping stream yielded a nil message, so assuming the stream is broken")
 		a.logger.Debug("[runStreamingPingLoop] Becoming unhealthy")
 		amsg.unhealthy = true
 

From da439072a25e0bd948e8d4c9b426d9a9d76271de Mon Sep 17 00:00:00 2001
From: CerealBoy <allan@buildkite.com>
Date: Wed, 4 Mar 2026 17:11:04 +1100
Subject: [PATCH 225/242] Bump version and CHANGELOG for v3.119.1

---
 CHANGELOG.md    | 9 +++++++++
 version/VERSION | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 286a598c1b..3aa6a62b08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.119.1](https://github.com/buildkite/agent/tree/v3.119.1) (2026-03-04)
+
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.119.0...v3.119.1)
+
+### Fixed
+
+- Validate ping mode flag, tweak log levels [#3734](https://github.com/buildkite/agent/pull/3734) (@DrJosh9000)
+- Default ping-mode to ping-only for now [#3733](https://github.com/buildkite/agent/pull/3733) (@moskyb)
+
 ## [v3.119.0](https://github.com/buildkite/agent/tree/v3.119.0) (2026-03-03)
 
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.118.1...v3.119.0)
diff --git a/version/VERSION b/version/VERSION
index 3c8b5bd6de..e74c064446 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.119.0
+3.119.1

From 150deaf201f36bae12ca211ee50417e299cfbb2a Mon Sep 17 00:00:00 2001
From: Lachlan Donald <lachlan@buildkite.com>
Date: Fri, 6 Mar 2026 13:29:15 +1100
Subject: [PATCH 226/242] chore: add mise config for go and golangci-lint

Amp-Thread-ID: https://ampcode.com/threads/T-019cc01c-1a01-765f-817d-5cddee9a79dc
---
 .mise.toml | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 .mise.toml

diff --git a/.mise.toml b/.mise.toml
new file mode 100644
index 0000000000..7b19a6359c
--- /dev/null
+++ b/.mise.toml
@@ -0,0 +1,3 @@
+[tools]
+go = "1.24.5"
+golangci-lint = "2.9.0"

From 71282a813ef3ab6c5294c4b57ab93bec448a1041 Mon Sep 17 00:00:00 2001
From: Lachlan Donald <lachlan@buildkite.com>
Date: Fri, 6 Mar 2026 16:50:28 +1100
Subject: [PATCH 227/242] test: add regression for header times scan after stop

Amp-Thread-ID: https://ampcode.com/threads/T-019cc164-d488-73cc-83a9-527ff4485676
---
 agent/header_times_streamer_test.go | 72 +++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 agent/header_times_streamer_test.go

diff --git a/agent/header_times_streamer_test.go b/agent/header_times_streamer_test.go
new file mode 100644
index 0000000000..7dbcf09f64
--- /dev/null
+++ b/agent/header_times_streamer_test.go
@@ -0,0 +1,72 @@
+package agent
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/buildkite/agent/v3/logger"
+)
+
+func TestHeaderTimesStreamerScanAfterStopDoesNotPanic(t *testing.T) {
+	t.Parallel()
+
+	h := newHeaderTimesStreamer(logger.Discard, func(context.Context, int, int, map[string]string) {})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runDone := make(chan struct{})
+	go func() {
+		h.Run(ctx)
+		close(runDone)
+	}()
+
+	deadline := time.After(500 * time.Millisecond)
+	for {
+		h.streamingMu.Lock()
+		streaming := h.streaming
+		h.streamingMu.Unlock()
+
+		if streaming {
+			break
+		}
+
+		select {
+		case <-deadline:
+			t.Fatal("timed out waiting for header times streamer to start")
+		default:
+			time.Sleep(1 * time.Millisecond)
+		}
+	}
+
+	stopDone := make(chan struct{})
+	go func() {
+		h.Stop()
+		close(stopDone)
+	}()
+
+	select {
+	case <-stopDone:
+	case <-time.After(500 * time.Millisecond):
+		cancel()
+		t.Fatal("timed out waiting for header times streamer to stop")
+	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("Scan panicked after Stop: %v", r)
+		}
+	}()
+
+	if got := h.Scan("--- a header"); !got {
+		t.Fatalf("Scan() = %t, want true", got)
+	}
+
+	select {
+	case <-runDone:
+	case <-time.After(500 * time.Millisecond):
+		cancel()
+		t.Fatal("timed out waiting for header times streamer run loop to exit")
+	}
+}

From 7d3eb2399a7b028e6dfd8f0a00756689c64e520a Mon Sep 17 00:00:00 2001
From: Lachlan Donald <lachlan@buildkite.com>
Date: Fri, 6 Mar 2026 16:50:58 +1100
Subject: [PATCH 228/242] fix: prevent header times scan panic after stop

Amp-Thread-ID: https://ampcode.com/threads/T-019cc164-d488-73cc-83a9-527ff4485676
---
 agent/header_times_streamer.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/agent/header_times_streamer.go b/agent/header_times_streamer.go
index 230daef798..5324d2400c 100644
--- a/agent/header_times_streamer.go
+++ b/agent/header_times_streamer.go
@@ -163,6 +163,7 @@ func (h *headerTimesStreamer) Stop() {
 		h.streamingMu.Unlock()
 		return
 	}
+	h.streaming = false
 	close(h.timesCh)
 	h.streamingMu.Unlock()
 

From eb849c3d92bc1348b5f9ebbcb3c5bf3eb38ae315 Mon Sep 17 00:00:00 2001
From: Lachlan Donald <lachlan@buildkite.com>
Date: Fri, 6 Mar 2026 16:58:41 +1100
Subject: [PATCH 229/242] test: re-enable additional lifecycle hooks regression

Amp-Thread-ID: https://ampcode.com/threads/T-019cc164-d488-73cc-83a9-527ff4485676
---
 clicommand/agent_start_test.go | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/clicommand/agent_start_test.go b/clicommand/agent_start_test.go
index cf143869b8..2ca933d496 100644
--- a/clicommand/agent_start_test.go
+++ b/clicommand/agent_start_test.go
@@ -100,19 +100,6 @@ func TestAgentStartupHook(t *testing.T) {
 }
 
 func TestAgentStartupHookWithAdditionalPaths(t *testing.T) {
-	t.SkipNow()
-	// This test was added to validate that multiple global hooks can be added
-	// by using the AdditionalHooksPaths configuration option. When this test
-	// runs however, there's a timing issue where the second hook errors at
-	// execution time as the file is not available.
-	//
-	// Error:          Received unexpected error:
-	//                 error running "/opt/homebrew/bin/bash /var/folders/x3/rsj92m015tdcby8gz2j_25ym0000gn/T/471662504/agent-startup": unexpected error type *errors.errorString: io: read/write on closed pipe
-	// Test:           TestAgentStartupHookWithAdditionalPaths/with_additional_agent-startup_hook
-	// Messages:       [[info] $ /var/folders/x3/rsj92m015tdcby8gz2j_25ym0000gn/T/982974833/agent-startup [info] hello new world [error] "agent-startup" hook: error running "/opt/homebrew/bin/bash /var/folders/x3/rsj92m015tdcby8gz2j_25ym0000gn/T/471662504/agent-startup": unexpected error type *errors.errorString: io: read/write on closed pipe]
-	//
-	// For now it is skipped, and left as a placeholder!
-
 	t.Parallel()
 
 	cfg := func(hooksPath, additionalHooksPath string) AgentStartConfig {

From e018c28de7d17931141f2361c34a0a55785301ab Mon Sep 17 00:00:00 2001
From: Lachlan Donald <lachlan@buildkite.com>
Date: Fri, 6 Mar 2026 16:59:20 +1100
Subject: [PATCH 230/242] fix: keep lifecycle hook output pipe open across
 hooks

Amp-Thread-ID: https://ampcode.com/threads/T-019cc164-d488-73cc-83a9-527ff4485676
---
 clicommand/agent_start.go | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index baf490d186..6a655f9187 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1549,6 +1549,10 @@ func agentLifecycleHook(hookName string, log logger.Logger, cfg AgentStartConfig
 		}
 	}
 
+	if len(hooks) == 0 {
+		return nil
+	}
+
 	// pipe from hook output to logger
 	r, w := io.Pipe()
 	sh, err := shell.New(
@@ -1570,6 +1574,10 @@ func agentLifecycleHook(hookName string, log logger.Logger, cfg AgentStartConfig
 			log.Info(scan.Text())
 		}
 	}()
+	defer func() {
+		_ = w.Close() // closing the writer ends scan.Scan and lets wg.Wait return
+		wg.Wait()
+	}()
 
 	// run hooks
 	for _, p = range hooks {
@@ -1584,10 +1592,6 @@ func agentLifecycleHook(hookName string, log logger.Logger, cfg AgentStartConfig
 			log.Error("%q hook: %v", hookName, err)
 			return err
 		}
-		w.Close() // goroutine scans until pipe is closed
-
-		// wait for hook to finish and output to flush to logger
-		wg.Wait()
 	}
 	return nil
 }

From d124a1011572501f2b66c41b1014ef841112cb42 Mon Sep 17 00:00:00 2001
From: CerealBoy <allan@buildkite.com>
Date: Mon, 9 Mar 2026 11:12:55 +1100
Subject: [PATCH 231/242] Generate a warning when cache is specified on
 self-hosted jobs

---
 .../job_environment_integration_test.go       | 129 ++++++++++++++++++
 agent/run_job.go                              |  10 ++
 2 files changed, 139 insertions(+)

diff --git a/agent/integration/job_environment_integration_test.go b/agent/integration/job_environment_integration_test.go
index 7cd36bde2f..cb20646e8e 100644
--- a/agent/integration/job_environment_integration_test.go
+++ b/agent/integration/job_environment_integration_test.go
@@ -2,6 +2,7 @@ package integration
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	"github.com/buildkite/agent/v3/agent"
@@ -52,6 +53,134 @@ func TestWhenCachePathsSetInJobStep_CachePathsEnvVarIsSet(t *testing.T) {
 	}
 }
 
+func TestCacheSettingsOnSelfHosted_LogsMessage(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	jobID := "cache-self-hosted-job"
+	job := &api.Job{
+		ID:                 jobID,
+		ChunksMaxSizeBytes: 1024,
+		Env: map[string]string{
+			"BUILDKITE_COMPUTE_TYPE": "self-hosted",
+		},
+		Step: pipeline.CommandStep{
+			Cache: &pipeline.Cache{
+				Paths: []string{"vendor", "node_modules"},
+			},
+		},
+		Token: "bkaj_job-token",
+	}
+
+	mb := mockBootstrap(t)
+	defer mb.CheckAndClose(t) //nolint:errcheck // bintest logs to t
+	mb.Expect().Once().AndExitWith(0)
+
+	e := createTestAgentEndpoint()
+	server := e.server()
+	defer server.Close()
+
+	err := runJob(t, ctx, testRunJobConfig{
+		job:           job,
+		server:        server,
+		agentCfg:      agent.AgentConfiguration{},
+		mockBootstrap: mb,
+	})
+	if err != nil {
+		t.Fatalf("runJob() error = %v", err)
+	}
+
+	logs := e.logsFor(t, jobID)
+	if !strings.Contains(logs, "Cache settings detected on self-hosted agent") {
+		t.Errorf("expected logs to contain cache warning for self-hosted agent, got %q", logs)
+	}
+	if !strings.Contains(logs, "vendor, node_modules") {
+		t.Errorf("expected logs to contain cache paths, got %q", logs)
+	}
+}
+
+func TestCacheSettingsOnHosted_DoesNotLogMessage(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	jobID := "cache-hosted-job"
+	job := &api.Job{
+		ID:                 jobID,
+		ChunksMaxSizeBytes: 1024,
+		Env: map[string]string{
+			"BUILDKITE_COMPUTE_TYPE": "hosted",
+		},
+		Step: pipeline.CommandStep{
+			Cache: &pipeline.Cache{
+				Paths: []string{"vendor", "node_modules"},
+			},
+		},
+		Token: "bkaj_job-token",
+	}
+
+	mb := mockBootstrap(t)
+	defer mb.CheckAndClose(t) //nolint:errcheck // bintest logs to t
+	mb.Expect().Once().AndExitWith(0)
+
+	e := createTestAgentEndpoint()
+	server := e.server()
+	defer server.Close()
+
+	err := runJob(t, ctx, testRunJobConfig{
+		job:           job,
+		server:        server,
+		agentCfg:      agent.AgentConfiguration{},
+		mockBootstrap: mb,
+	})
+	if err != nil {
+		t.Fatalf("runJob() error = %v", err)
+	}
+
+	logs := e.logsFor(t, jobID)
+	if strings.Contains(logs, "Cache settings detected on self-hosted agent") {
+		t.Errorf("expected logs to NOT contain cache warning for hosted agent, got %q", logs)
+	}
+}
+
+func TestNoCacheSettings_DoesNotLogMessage(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	jobID := "no-cache-job"
+	job := &api.Job{
+		ID:                 jobID,
+		ChunksMaxSizeBytes: 1024,
+		Env: map[string]string{
+			"BUILDKITE_COMPUTE_TYPE": "self-hosted",
+		},
+		Step:  pipeline.CommandStep{},
+		Token: "bkaj_job-token",
+	}
+
+	mb := mockBootstrap(t)
+	defer mb.CheckAndClose(t) //nolint:errcheck // bintest logs to t
+	mb.Expect().Once().AndExitWith(0)
+
+	e := createTestAgentEndpoint()
+	server := e.server()
+	defer server.Close()
+
+	err := runJob(t, ctx, testRunJobConfig{
+		job:           job,
+		server:        server,
+		agentCfg:      agent.AgentConfiguration{},
+		mockBootstrap: mb,
+	})
+	if err != nil {
+		t.Fatalf("runJob() error = %v", err)
+	}
+
+	logs := e.logsFor(t, jobID)
+	if strings.Contains(logs, "Cache settings detected on self-hosted agent") {
+		t.Errorf("expected logs to NOT contain cache warning when no cache settings, got %q", logs)
+	}
+}
+
 func TestBuildkiteRequestHeaders(t *testing.T) {
 	t.Parallel()
 
diff --git a/agent/run_job.go b/agent/run_job.go
index 09296ac3be..28eff695c5 100644
--- a/agent/run_job.go
+++ b/agent/run_job.go
@@ -9,6 +9,7 @@ import (
 	"os"
 	"regexp"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -169,6 +170,15 @@ func (r *JobRunner) Run(ctx context.Context, ignoreAgentInDispatches *bool) (err
 		}
 	}
 
+	// Log a message if the job has cache settings but is running on a self-hosted agent.
+	if cache := job.Step.Cache; cache != nil && !cache.Disabled && len(cache.Paths) > 0 {
+		if job.Env["BUILDKITE_COMPUTE_TYPE"] == "self-hosted" {
+			fmt.Fprintln(r.jobLogs, "+++ ⚠️ Cache settings detected on self-hosted agent")
+			fmt.Fprintf(r.jobLogs, "cache paths: %s\n", strings.Join(cache.Paths, ", "))
+			r.agentLogger.Info("Job %s has cache settings but is running on a self-hosted agent", job.ID)
+		}
+	}
+
 	// Validate the repository if the list of allowed repositories is set.
 	if err := r.validateConfigAllowlists(job); err != nil {
 		fmt.Fprintln(r.jobLogs, err.Error())

From bb17f5603474eab01d3d8f94c45ac7b07d6cc717 Mon Sep 17 00:00:00 2001
From: CerealBoy <allan@buildkite.com>
Date: Mon, 9 Mar 2026 15:17:15 +1100
Subject: [PATCH 232/242] Bump version and CHANGELOG for v3.119.2

---
 CHANGELOG.md    | 12 ++++++++++++
 version/VERSION |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3aa6a62b08..cbbf5349f3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.119.2](https://github.com/buildkite/agent/tree/v3.119.2) (2026-03-09)
+
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.119.1...v3.119.2)
+
+### Added
+
+- Generate a warning when cache is specified on self-hosted jobs [#3743](https://github.com/buildkite/agent/pull/3743) (@CerealBoy)
+
+### Internal
+
+- chore: add mise config for go and golangci-lint [#3739](https://github.com/buildkite/agent/pull/3739) (@lox)
+
 ## [v3.119.1](https://github.com/buildkite/agent/tree/v3.119.1) (2026-03-04)
 
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.119.0...v3.119.1)
diff --git a/version/VERSION b/version/VERSION
index e74c064446..03938b9efb 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.119.1
+3.119.2

From 4723317bf66fcc28631d4797e714093e7d96f31c Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 10 Mar 2026 11:03:23 +1100
Subject: [PATCH 233/242] Upgrade to Go 1.25

---
 .buildkite/Dockerfile-compile                |   2 +-
 .buildkite/Dockerfile-e2e                    |   2 +-
 .mise.toml                                   |   2 +-
 MODULE.bazel                                 |   4 +-
 agent/BUILD.bazel                            |  12 +
 api/BUILD.bazel                              |   4 +
 api/proto/BUILD.bazel                        |  26 +
 api/proto/gen/BUILD.bazel                    |  13 +
 api/proto/gen/agentedgev1connect/BUILD.bazel |  12 +
 clicommand/BUILD.bazel                       |   1 +
 go.mod                                       | 219 ++++----
 go.sum                                       | 532 ++++++++++---------
 internal/experiments/BUILD.bazel             |   8 +-
 internal/job/BUILD.bazel                     |   1 -
 internal/secrets/BUILD.bazel                 |   2 +
 kubernetes/BUILD.bazel                       |   1 +
 16 files changed, 461 insertions(+), 380 deletions(-)
 create mode 100644 api/proto/BUILD.bazel
 create mode 100644 api/proto/gen/BUILD.bazel
 create mode 100644 api/proto/gen/agentedgev1connect/BUILD.bazel

diff --git a/.buildkite/Dockerfile-compile b/.buildkite/Dockerfile-compile
index c7b5f2815b..729d0d3097 100644
--- a/.buildkite/Dockerfile-compile
+++ b/.buildkite/Dockerfile-compile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.12@sha256:c2131140c7c29ff277b1c412d524b7f56289513f49672c57a3d992247dd146f8
+FROM public.ecr.aws/docker/library/golang:1.25.8@sha256:779b230b2508037a8095c9e2d223a6405f8426e12233b694dbae50197b9f6d04
 COPY build/ssh.conf /etc/ssh/ssh_config.d/
 
 RUN go install github.com/google/go-licenses@latest
diff --git a/.buildkite/Dockerfile-e2e b/.buildkite/Dockerfile-e2e
index 61225cc883..4f818bf1f3 100644
--- a/.buildkite/Dockerfile-e2e
+++ b/.buildkite/Dockerfile-e2e
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.24.12@sha256:c2131140c7c29ff277b1c412d524b7f56289513f49672c57a3d992247dd146f8
+FROM public.ecr.aws/docker/library/golang:1.25.8@sha256:779b230b2508037a8095c9e2d223a6405f8426e12233b694dbae50197b9f6d04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     unzip \
diff --git a/.mise.toml b/.mise.toml
index 7b19a6359c..41fe1fd2cd 100644
--- a/.mise.toml
+++ b/.mise.toml
@@ -1,3 +1,3 @@
 [tools]
-go = "1.24.5"
+go = "1.25.8"
 golangci-lint = "2.9.0"
diff --git a/MODULE.bazel b/MODULE.bazel
index a0c528acab..7af455bcb9 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -2,7 +2,7 @@ bazel_dep(name = "gazelle", version = "0.47.0")
 bazel_dep(name = "rules_go", version = "0.59.0")
 
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
-go_sdk.download(version = "1.24.10")
+go_sdk.download(version = "1.25.8")
 
 go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//:go.mod")
@@ -14,4 +14,4 @@ go_deps.gazelle_override(
 )
 
 # All *direct* dependencies are required to be listed explicitly
-use_repo(go_deps, "cc_mvdan_gofumpt", "com_github_aws_aws_sdk_go_v2", "com_github_aws_aws_sdk_go_v2_config", "com_github_aws_aws_sdk_go_v2_feature_ec2_imds", "com_github_aws_aws_sdk_go_v2_feature_s3_manager", "com_github_aws_aws_sdk_go_v2_service_ec2", "com_github_aws_aws_sdk_go_v2_service_kms", "com_github_aws_aws_sdk_go_v2_service_s3", "com_github_aws_smithy_go", "com_github_azure_azure_sdk_for_go_sdk_azidentity", "com_github_azure_azure_sdk_for_go_sdk_storage_azblob", "com_github_brunoscheufler_aws_ecs_metadata_go", "com_github_buildkite_bintest_v3", "com_github_buildkite_go_buildkite_v4", "com_github_buildkite_go_pipeline", "com_github_buildkite_interpolate", "com_github_buildkite_roko", "com_github_buildkite_shellwords", "com_github_buildkite_test_engine_client", "com_github_buildkite_zstash", "com_github_creack_pty", "com_github_datadog_datadog_go_v5", "com_github_denisbrodbeck_machineid", "com_github_dustin_go_humanize", "com_github_dustinkirkland_golang_petname", "com_github_gliderlabs_ssh", "com_github_go_chi_chi_v5", "com_github_gofrs_flock", "com_github_google_go_cmp", "com_github_google_go_querystring", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_khan_genqlient", "com_github_lestrrat_go_jwx_v2", "com_github_oleiade_reflections", "com_github_opentracing_opentracing_go", "com_github_pborman_uuid", "com_github_prometheus_client_golang", "com_github_puzpuzpuz_xsync_v2", "com_github_qri_io_jsonschema", "com_github_stretchr_testify", "com_github_urfave_cli", "com_google_cloud_go_compute_metadata", "dev_drjosh_zzglob", "in_gopkg_datadog_dd_trace_go_v1", "in_gopkg_yaml_v3", "io_opentelemetry_go_contrib_propagators_aws", "io_opentelemetry_go_contrib_propagators_b3", "io_opentelemetry_go_contrib_propagators_jaeger", "io_opentelemetry_go_contrib_propagators_ot", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc", "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_trace", "org_golang_google_api", "org_golang_x_crypto", "org_golang_x_net", "org_golang_x_oauth2", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term", "tools_gotest_gotestsum", "tools_gotest_v3")
+use_repo(go_deps, "build_buf_gen_go_bufbuild_protovalidate_protocolbuffers_go", "cc_mvdan_gofumpt", "com_connectrpc_connect", "com_github_aws_aws_sdk_go_v2", "com_github_aws_aws_sdk_go_v2_config", "com_github_aws_aws_sdk_go_v2_feature_ec2_imds", "com_github_aws_aws_sdk_go_v2_feature_s3_manager", "com_github_aws_aws_sdk_go_v2_service_ec2", "com_github_aws_aws_sdk_go_v2_service_kms", "com_github_aws_aws_sdk_go_v2_service_s3", "com_github_aws_smithy_go", "com_github_azure_azure_sdk_for_go_sdk_azidentity", "com_github_azure_azure_sdk_for_go_sdk_storage_azblob", "com_github_brunoscheufler_aws_ecs_metadata_go", "com_github_buildkite_bintest_v3", "com_github_buildkite_go_buildkite_v4", "com_github_buildkite_go_pipeline", "com_github_buildkite_interpolate", "com_github_buildkite_roko", "com_github_buildkite_shellwords", "com_github_buildkite_test_engine_client", "com_github_buildkite_zstash", "com_github_creack_pty", "com_github_datadog_datadog_go_v5", "com_github_denisbrodbeck_machineid", "com_github_dustin_go_humanize", "com_github_dustinkirkland_golang_petname", "com_github_gliderlabs_ssh", "com_github_go_chi_chi_v5", "com_github_gofrs_flock", "com_github_google_go_cmp", "com_github_google_go_querystring", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_khan_genqlient", "com_github_lestrrat_go_jwx_v2", "com_github_oleiade_reflections", "com_github_opentracing_opentracing_go", "com_github_pborman_uuid", "com_github_prometheus_client_golang", "com_github_puzpuzpuz_xsync_v2", "com_github_qri_io_jsonschema", "com_github_stretchr_testify", "com_github_urfave_cli", "com_google_cloud_go_compute_metadata", "dev_drjosh_zzglob", "in_gopkg_datadog_dd_trace_go_v1", "in_gopkg_yaml_v3", "io_opentelemetry_go_contrib_propagators_aws", "io_opentelemetry_go_contrib_propagators_b3", "io_opentelemetry_go_contrib_propagators_jaeger", "io_opentelemetry_go_contrib_propagators_ot", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc", "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_trace", "org_golang_google_api", "org_golang_google_protobuf", "org_golang_x_crypto", "org_golang_x_net", "org_golang_x_oauth2", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term", "tools_gotest_gotestsum", "tools_gotest_v3")
diff --git a/agent/BUILD.bazel b/agent/BUILD.bazel
index 767decc001..aa9924f5eb 100644
--- a/agent/BUILD.bazel
+++ b/agent/BUILD.bazel
@@ -6,6 +6,12 @@ go_library(
         "agent_configuration.go",
         "agent_pool.go",
         "agent_worker.go",
+        "agent_worker_action.go",
+        "agent_worker_debouncer.go",
+        "agent_worker_heartbeat.go",
+        "agent_worker_ping.go",
+        "agent_worker_streaming.go",
+        "baton.go",
         "doc.go",
         "ec2_meta_data.go",
         "ec2_tags.go",
@@ -27,6 +33,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//api",
+        "//api/proto/gen",
         "//core",
         "//env",
         "//internal/agenthttp",
@@ -40,6 +47,7 @@ go_library(
         "//metrics",
         "//process",
         "//status",
+        "@com_connectrpc_connect//:connect",
         "@com_github_aws_aws_sdk_go_v2//aws",
         "@com_github_aws_aws_sdk_go_v2_config//:config",
         "@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds",
@@ -69,6 +77,7 @@ go_test(
         "agent_worker_test.go",
         "fake_api_server_test.go",
         "gcp_meta_data_test.go",
+        "idle_monitor_test.go",
         "job_runner_test.go",
         "k8s_tags_test.go",
         "log_streamer_test.go",
@@ -78,9 +87,12 @@ go_test(
     embed = [":agent"],
     deps = [
         "//api",
+        "//api/proto/gen",
+        "//api/proto/gen/agentedgev1connect",
         "//core",
         "//logger",
         "//metrics",
+        "@com_connectrpc_connect//:connect",
         "@com_github_buildkite_go_pipeline//:go-pipeline",
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_uuid//:uuid",
diff --git a/api/BUILD.bazel b/api/BUILD.bazel
index 3e345febb0..b50d5bc3ca 100644
--- a/api/BUILD.bazel
+++ b/api/BUILD.bazel
@@ -17,6 +17,7 @@ go_library(
         "meta_data.go",
         "oidc.go",
         "pings.go",
+        "pings_streaming.go",
         "pipelines.go",
         "retryable.go",
         "secrets.go",
@@ -27,8 +28,11 @@ go_library(
     importpath = "github.com/buildkite/agent/v3/api",
     visibility = ["//visibility:public"],
     deps = [
+        "//api/proto/gen",
+        "//api/proto/gen/agentedgev1connect",
         "//internal/agenthttp",
         "//logger",
+        "@com_connectrpc_connect//:connect",
         "@com_github_buildkite_go_pipeline//:go-pipeline",
         "@com_github_buildkite_roko//:roko",
         "@com_github_google_go_querystring//query",
diff --git a/api/proto/BUILD.bazel b/api/proto/BUILD.bazel
new file mode 100644
index 0000000000..c60ce05f00
--- /dev/null
+++ b/api/proto/BUILD.bazel
@@ -0,0 +1,26 @@
+load("@rules_go//go:def.bzl", "go_library")
+load("@rules_go//proto:def.bzl", "go_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+proto_library(
+    name = "agentedge_v1_proto",
+    srcs = ["agentedge.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//buf/validate:validate_proto"],
+)
+
+go_proto_library(
+    name = "agentedge_v1_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:go_grpc_v2"],
+    importpath = "github.com/buildkite/agent/v3/api/proto",
+    proto = ":agentedge_v1_proto",
+    visibility = ["//visibility:public"],
+    deps = ["//buf/validate:validate_proto"],
+)
+
+go_library(
+    name = "proto",
+    embed = [":agentedge_v1_go_proto"],
+    importpath = "github.com/buildkite/agent/v3/api/proto",
+    visibility = ["//visibility:public"],
+)
diff --git a/api/proto/gen/BUILD.bazel b/api/proto/gen/BUILD.bazel
new file mode 100644
index 0000000000..b261951747
--- /dev/null
+++ b/api/proto/gen/BUILD.bazel
@@ -0,0 +1,13 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "gen",
+    srcs = ["agentedge.pb.go"],
+    importpath = "github.com/buildkite/agent/v3/api/proto/gen",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@build_buf_gen_go_bufbuild_protovalidate_protocolbuffers_go//buf/validate",
+        "@org_golang_google_protobuf//reflect/protoreflect",
+        "@org_golang_google_protobuf//runtime/protoimpl",
+    ],
+)
diff --git a/api/proto/gen/agentedgev1connect/BUILD.bazel b/api/proto/gen/agentedgev1connect/BUILD.bazel
new file mode 100644
index 0000000000..0b9767e103
--- /dev/null
+++ b/api/proto/gen/agentedgev1connect/BUILD.bazel
@@ -0,0 +1,12 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "agentedgev1connect",
+    srcs = ["agentedge.connect.go"],
+    importpath = "github.com/buildkite/agent/v3/api/proto/gen/agentedgev1connect",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//api/proto/gen",
+        "@com_connectrpc_connect//:connect",
+    ],
+)
diff --git a/clicommand/BUILD.bazel b/clicommand/BUILD.bazel
index f9c992aef8..77557d4787 100644
--- a/clicommand/BUILD.bazel
+++ b/clicommand/BUILD.bazel
@@ -29,6 +29,7 @@ go_library(
         "errors.go",
         "git_credentials_helper.go",
         "global.go",
+        "job_update.go",
         "kubernetes_bootstrap.go",
         "lock_acquire.go",
         "lock_common.go",
diff --git a/go.mod b/go.mod
index 4069266721..932fa9d0ee 100644
--- a/go.mod
+++ b/go.mod
@@ -1,11 +1,11 @@
 module github.com/buildkite/agent/v3
 
-go 1.24.0
+go 1.25.0
 
-toolchain go1.24.5
+toolchain go1.25.8
 
 require (
-	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20251209175733-2a1774d88802.1
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260209202127-80ab13bee0bf.1
 	cloud.google.com/go/compute/metadata v0.9.0
 	connectrpc.com/connect v1.19.1
 	drjosh.dev/zzglob v0.4.2
@@ -13,26 +13,26 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
 	github.com/DataDog/datadog-go/v5 v5.8.3
 	github.com/Khan/genqlient v0.8.1
-	github.com/aws/aws-sdk-go-v2 v1.41.1
-	github.com/aws/aws-sdk-go-v2/config v1.32.7
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.279.2
-	github.com/aws/aws-sdk-go-v2/service/kms v1.49.5
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1
-	github.com/aws/smithy-go v1.24.0
-	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
+	github.com/aws/aws-sdk-go-v2 v1.41.3
+	github.com/aws/aws-sdk-go-v2/config v1.32.11
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.6
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0
+	github.com/aws/aws-sdk-go-v2/service/kms v1.50.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4
+	github.com/aws/smithy-go v1.24.2
+	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20221221133751-67e37ae746cd
 	github.com/buildkite/bintest/v3 v3.3.0
-	github.com/buildkite/go-buildkite/v4 v4.13.1
+	github.com/buildkite/go-buildkite/v4 v4.16.0
 	github.com/buildkite/go-pipeline v0.16.0
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
 	github.com/buildkite/shellwords v1.0.1
 	github.com/buildkite/zstash v0.8.0
-	github.com/creack/pty v1.1.19
+	github.com/creack/pty v1.1.24
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
-	github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2
+	github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b
 	github.com/gliderlabs/ssh v0.3.8
 	github.com/go-chi/chi/v5 v5.2.5
 	github.com/gofrs/flock v0.13.0
@@ -49,22 +49,22 @@ require (
 	github.com/qri-io/jsonschema v0.2.1
 	github.com/stretchr/testify v1.11.1
 	github.com/urfave/cli v1.22.17
-	go.opentelemetry.io/contrib/propagators/aws v1.40.0
-	go.opentelemetry.io/contrib/propagators/b3 v1.40.0
-	go.opentelemetry.io/contrib/propagators/jaeger v1.40.0
-	go.opentelemetry.io/contrib/propagators/ot v1.40.0
-	go.opentelemetry.io/otel v1.40.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0
-	go.opentelemetry.io/otel/sdk v1.40.0
-	go.opentelemetry.io/otel/trace v1.40.0
+	go.opentelemetry.io/contrib/propagators/aws v1.42.0
+	go.opentelemetry.io/contrib/propagators/b3 v1.42.0
+	go.opentelemetry.io/contrib/propagators/jaeger v1.42.0
+	go.opentelemetry.io/contrib/propagators/ot v1.42.0
+	go.opentelemetry.io/otel v1.42.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0
+	go.opentelemetry.io/otel/sdk v1.42.0
+	go.opentelemetry.io/otel/trace v1.42.0
 	golang.org/x/crypto v0.48.0
-	golang.org/x/net v0.50.0
-	golang.org/x/oauth2 v0.35.0
-	golang.org/x/sync v0.19.0
-	golang.org/x/sys v0.41.0
+	golang.org/x/net v0.51.0
+	golang.org/x/oauth2 v0.36.0
+	golang.org/x/sync v0.20.0
+	golang.org/x/sys v0.42.0
 	golang.org/x/term v0.40.0
-	google.golang.org/api v0.260.0
+	google.golang.org/api v0.270.0
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/DataDog/dd-trace-go.v1 v1.74.8
 	gopkg.in/yaml.v3 v3.0.1
@@ -72,144 +72,147 @@ require (
 )
 
 require (
-	cloud.google.com/go/auth v0.18.0 // indirect
+	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
-	github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/obfuscate v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/proto v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.69.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/trace v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/util/log v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/version v0.67.0 // indirect
-	github.com/DataDog/dd-trace-go/v2 v2.3.0 // indirect
-	github.com/DataDog/go-libddwaf/v4 v4.3.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect
+	github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/proto v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/template v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/trace v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/trace/log v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/trace/otel v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/trace/stats v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/log v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.76.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/version v0.76.3 // indirect
+	github.com/DataDog/dd-trace-go/v2 v2.6.0 // indirect
+	github.com/DataDog/go-libddwaf/v4 v4.9.0 // indirect
 	github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20250721125240-fdf1ef85b633 // indirect
-	github.com/DataDog/go-sqllexer v0.1.6 // indirect
-	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
-	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0 // indirect
-	github.com/DataDog/sketches-go v1.4.7 // indirect
-	github.com/Masterminds/semver/v3 v3.3.1 // indirect
+	github.com/DataDog/go-sqllexer v0.2.0 // indirect
+	github.com/DataDog/go-tuf v1.1.1-0.5.2 // indirect
+	github.com/DataDog/sketches-go v1.4.8 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/agnivade/levenshtein v1.2.1 // indirect
 	github.com/alexflint/go-arg v1.5.1 // indirect
 	github.com/alexflint/go-scalar v1.2.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/buildkite/test-engine-client v1.6.0 // indirect
-	github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect
 	github.com/dnephin/pflag v1.0.7 // indirect
-	github.com/ebitengine/purego v0.8.4 // indirect
+	github.com/ebitengine/purego v0.10.0 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
-	github.com/goccy/go-json v0.10.3 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.9 // indirect
-	github.com/googleapis/gax-go/v2 v2.16.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
-	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
+	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
+	github.com/hashicorp/go-version v1.8.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.18.2 // indirect
+	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lestrrat-go/blackmagic v1.0.3 // indirect
+	github.com/lestrrat-go/blackmagic v1.0.4 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/httprc v1.0.6 // indirect
 	github.com/lestrrat-go/iter v1.0.2 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
-	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
+	github.com/linkdata/deadlock v0.5.5 // indirect
+	github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/minio/simdjson-go v0.4.5 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/outcaste-io/ristretto v0.2.3 // indirect
-	github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
+	github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect
+	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/common v0.67.5 // indirect
+	github.com/prometheus/procfs v0.20.1 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/qri-io/jsonpointer v0.1.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 // indirect
-	github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect
-	github.com/segmentio/asm v1.2.0 // indirect
-	github.com/shirou/gopsutil/v4 v4.25.8 // indirect
-	github.com/theckman/httpforwarded v0.4.0 // indirect
-	github.com/tinylib/msgp v1.2.5 // indirect
-	github.com/tklauser/go-sysconf v0.3.15 // indirect
-	github.com/tklauser/numcpus v0.10.0 // indirect
-	github.com/vektah/gqlparser/v2 v2.5.25 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect
+	github.com/segmentio/asm v1.2.1 // indirect
+	github.com/shirou/gopsutil/v4 v4.26.2 // indirect
+	github.com/tinylib/msgp v1.6.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
+	github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561 // indirect
+	github.com/vektah/gqlparser/v2 v2.5.32 // indirect
 	github.com/wolfeidau/quickzip v1.0.2 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/collector/component v1.31.0 // indirect
-	go.opentelemetry.io/collector/featuregate v1.31.0 // indirect
-	go.opentelemetry.io/collector/internal/telemetry v0.125.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.31.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.125.0 // indirect
-	go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
-	go.opentelemetry.io/otel/log v0.11.0 // indirect
-	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.opentelemetry.io/collector/component v1.53.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.53.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.53.0 // indirect
+	go.opentelemetry.io/collector/pdata/pprofile v0.147.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.27.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
-	golang.org/x/mod v0.32.0 // indirect
+	go.uber.org/zap v1.27.1 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
+	golang.org/x/mod v0.33.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	golang.org/x/tools v0.42.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/grpc v1.78.0 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
+	google.golang.org/grpc v1.79.2 // indirect
+	gopkg.in/ini.v1 v1.67.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/gotestsum v1.13.0 // indirect
 	mvdan.cc/gofumpt v0.9.2 // indirect
diff --git a/go.sum b/go.sum
index 531634e49b..963ea7868b 100644
--- a/go.sum
+++ b/go.sum
@@ -1,7 +1,7 @@
-buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20251209175733-2a1774d88802.1 h1:j9yeqTWEFrtimt8Nng2MIeRrpoCvQzM9/g25XTvqUGg=
-buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20251209175733-2a1774d88802.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM=
-cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0=
-cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260209202127-80ab13bee0bf.1 h1:PMmTMyvHScV9Mn8wc6ASge9uRcHy0jtqPd+fM35LmsQ=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260209202127-80ab13bee0bf.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM=
+cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
+cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
@@ -10,8 +10,8 @@ connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14=
 connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w=
 drjosh.dev/zzglob v0.4.2 h1:q+e5Cp6SFCyz+Yurhk/edSrTKEk3tn60vzoaXLmtiBo=
 drjosh.dev/zzglob v0.4.2/go.mod h1:SbYDdesQC13iyGiEwV8dJfJbyz7/Qiawrd5ODdJQCoo=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
@@ -24,47 +24,55 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBAD
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUXdP0ICgAabvq2FYLXrJWKx1fJQ+GxSo3Y=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0 h1:2mEwRWvhIPHMPK4CMD8iKbsrYBxeMBSuuCXumQAwShU=
-github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.67.0/go.mod h1:ejJHsyJTG7NU6c6TDbF7dmckD3g+AUGSdiSXy+ZyaCE=
-github.com/DataDog/datadog-agent/pkg/obfuscate v0.67.0 h1:NcvyDVIUA0NbBDbp7QJnsYhoBv548g8bXq886795mCQ=
-github.com/DataDog/datadog-agent/pkg/obfuscate v0.67.0/go.mod h1:1oPcs3BUTQhiTkmk789rb7ob105MxNV6OuBa28BdukQ=
-github.com/DataDog/datadog-agent/pkg/proto v0.67.0 h1:7dO6mKYRb7qSiXEu7Q2mfeKbhp4hykCAULy4BfMPmsQ=
-github.com/DataDog/datadog-agent/pkg/proto v0.67.0/go.mod h1:bKVXB7pxBg0wqXF6YSJ+KU6PeCWKDyJj83kUH1ab+7o=
-github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.69.0 h1:/DsN4R+IkC6t1+4cHSfkxzLtDl84rBbPC5Wa9srBAoM=
-github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.69.0/go.mod h1:Th2LD/IGid5Rza55pzqGu6nUdOv/Rts6wPwLjTyOSTs=
-github.com/DataDog/datadog-agent/pkg/trace v0.67.0 h1:dqt+/nObo0JKyaEqIMZgfqGZbx9TfEHpCkrjQ/zzH7k=
-github.com/DataDog/datadog-agent/pkg/trace v0.67.0/go.mod h1:zmZoEtKvOnaKHbJGBKH3a4xuyPrSfBaF0ZE3Q3rCoDw=
-github.com/DataDog/datadog-agent/pkg/util/log v0.67.0 h1:xrH15QNqeJZkYoXYi44VCIvGvTwlQ3z2iT2QVTGiT7s=
-github.com/DataDog/datadog-agent/pkg/util/log v0.67.0/go.mod h1:dfVLR+euzEyg1CeiExgJQq1c1dod42S6IeiRPj8H7Yk=
-github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0 h1:aIWF85OKxXGo7rVyqJ7jm7lm2qCQrgyXzYyFuw0T2EQ=
-github.com/DataDog/datadog-agent/pkg/util/scrubber v0.67.0/go.mod h1:Lfap5FuM4b/Pw9IrTuAvWBWZEmXOvZhCya3dYv4G8O0=
-github.com/DataDog/datadog-agent/pkg/version v0.67.0 h1:TB8H8r+laB1Qdttvvc6XJVyLGxp8E6j2f2Mh5IPbYmQ=
-github.com/DataDog/datadog-agent/pkg/version v0.67.0/go.mod h1:kvAw/WbI7qLAsDI2wHabZfM7Cv2zraD3JA3323GEB+8=
+github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.76.3 h1:rWeinj0AJg/c/vu0bjeWkVXOhnT8L3xZv6C1cZdumIA=
+github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.76.3/go.mod h1:EiRMrLmaSQmE4dLlBewlhbqg2J0n2QVhUWrWe/1MHMw=
+github.com/DataDog/datadog-agent/pkg/obfuscate v0.76.3 h1:dYF3X2OFdKba0Qa9Y2bxj2BRZqZIZ4QS1PMiQU6tf4E=
+github.com/DataDog/datadog-agent/pkg/obfuscate v0.76.3/go.mod h1:Q1B0h6Gyk8twdN1huri/UkQvLbUooVmEsooE49cqEFk=
+github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.76.3 h1:0P//Q4MaXmCij+lJeDAR/r/FjX34mxm7rPDgJ+S269I=
+github.com/DataDog/datadog-agent/pkg/opentelemetry-mapping-go/otlp/attributes v0.76.3/go.mod h1:q1X3A1Tnjj+0Jb0wlSuYPDLJRn1G0Jzj127Z/LQ7+u0=
+github.com/DataDog/datadog-agent/pkg/proto v0.76.3 h1:ADTat21KVlR2Vgg99vgclCstfQM1sJhqfISW2A+xJhs=
+github.com/DataDog/datadog-agent/pkg/proto v0.76.3/go.mod h1:RzHSkeIpR6OBEmwGy4fUPtQ1hPgt7ljwYlJmKxMHSX4=
+github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.76.3 h1:X4mSuuxJnKYLVgTtoHQFOWh2sQE1EbSMQ6+p6G9fEFo=
+github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.76.3/go.mod h1:MCaZbLoaLD7kUUmSd5jlIvQ5qPAMM8GI7OZ+0FGWi1E=
+github.com/DataDog/datadog-agent/pkg/template v0.76.3 h1:U2Tt3KA/MXrhIOT/ctxmcL2LezvscRMcCnE4pt40z1s=
+github.com/DataDog/datadog-agent/pkg/template v0.76.3/go.mod h1:mpV3MbF/us0LdM3tvVHDztjApy3VWGeu5RuS/MpGVHQ=
+github.com/DataDog/datadog-agent/pkg/trace v0.76.3 h1:kpPoKiVsCk+MYYLnl8EXOH/C/Kxiq/pHAdXLDPreUiU=
+github.com/DataDog/datadog-agent/pkg/trace v0.76.3/go.mod h1:CYHt36L9PiX5cckWeOaH3E3iWBdGHRpWvAQP8NFyNDc=
+github.com/DataDog/datadog-agent/pkg/trace/log v0.76.3 h1:8EHqwJMTAHS3T7qO4P5VO3oAnbSnuPqxf5pyTB6mTFc=
+github.com/DataDog/datadog-agent/pkg/trace/log v0.76.3/go.mod h1:sHKgo3EBOqLu35qKiJV9lOU8ixLpDzE+vjvqo+abEhs=
+github.com/DataDog/datadog-agent/pkg/trace/otel v0.76.3 h1:CFJWLO2ORycIb/dLTSsE6gF58w40ocYUPUreopl3IDc=
+github.com/DataDog/datadog-agent/pkg/trace/otel v0.76.3/go.mod h1:9796X7mt3DXNgkgppgmJRu46aMh6vTIKxLc6W2Kvi1E=
+github.com/DataDog/datadog-agent/pkg/trace/stats v0.76.3 h1:+CxRY4IIZcbnf0faTxeGoCFj7s9G2amtq444QrZiq/s=
+github.com/DataDog/datadog-agent/pkg/trace/stats v0.76.3/go.mod h1:Dwjtu2AxPhmZdlErnU8l/c/Q7E118wb8KNyw999XiLg=
+github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.76.3 h1:73rQ3y+b6ZyvY68SaZgwjGmNTy2PKneCaDqEmJ+G3TQ=
+github.com/DataDog/datadog-agent/pkg/trace/traceutil v0.76.3/go.mod h1:PZLADfur1R7YUxzNM5HIrKhN9alP4lJt/vkhgKOqZIk=
+github.com/DataDog/datadog-agent/pkg/util/log v0.76.3 h1:UOfvx+9/UZBahteNZ7MIsHRw9p3EwGiNI4rWY0JJ6N0=
+github.com/DataDog/datadog-agent/pkg/util/log v0.76.3/go.mod h1:GvttG8Nh3NuSFqmL/lAjnMGNDNn2NsDtbk56SoRBYSw=
+github.com/DataDog/datadog-agent/pkg/util/scrubber v0.76.3 h1:0i1QhzPb2Fd7gi+yDlhFNxBHkuQM8QRiDlPcTONF2KE=
+github.com/DataDog/datadog-agent/pkg/util/scrubber v0.76.3/go.mod h1:gvgHWXeTBueDDJhDdwtLZTa3BY5qRSc9/ycL9woLVEo=
+github.com/DataDog/datadog-agent/pkg/version v0.76.3 h1:GUDdUomtTaClTFNrLRmLvnoeP6w5M4cNiyh62JugMKk=
+github.com/DataDog/datadog-agent/pkg/version v0.76.3/go.mod h1:4I4x0IqhIr0fXr90G8Qauz9iID0xhonjUB2lAZH9qyI=
 github.com/DataDog/datadog-go/v5 v5.8.3 h1:s58CUJ9s8lezjhTNJO/SxkPBv2qZjS3ktpRSqGF5n0s=
 github.com/DataDog/datadog-go/v5 v5.8.3/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
-github.com/DataDog/dd-trace-go/v2 v2.3.0 h1:0Y5kx+Wbod0z8moY0vUbKl6OM0oIV4zAynsVmsq+XT8=
-github.com/DataDog/dd-trace-go/v2 v2.3.0/go.mod h1:yFomJ/rqKNLDbS9ohIDibdz8q9GK0MUSSkBdVDCibGA=
-github.com/DataDog/go-libddwaf/v4 v4.3.2 h1:YGvW2Of1C4e1yU+p7iibmhN2zEOgi9XEchbhQjBxb/A=
-github.com/DataDog/go-libddwaf/v4 v4.3.2/go.mod h1:/AZqP6zw3qGJK5mLrA0PkfK3UQDk1zCI2fUNCt4xftE=
+github.com/DataDog/dd-trace-go/v2 v2.6.0 h1:hqrgEmDi9atKGU9+xuyMmcsUxfzNAR6+GvhsXOG6qpk=
+github.com/DataDog/dd-trace-go/v2 v2.6.0/go.mod h1:TwIaEXJHXmFWd513U69T1cFPHinI2pDuvlh6DuMGC88=
+github.com/DataDog/go-libddwaf/v4 v4.9.0 h1:a788e37iuH7sR9uIYHkulvTnp2FkXTiZ3yY/kuaHgZE=
+github.com/DataDog/go-libddwaf/v4 v4.9.0/go.mod h1:/AZqP6zw3qGJK5mLrA0PkfK3UQDk1zCI2fUNCt4xftE=
 github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20250721125240-fdf1ef85b633 h1:ZRLR9Lbym748e8RznWzmSoK+OfV+8qW6SdNYA4/IqdA=
 github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20250721125240-fdf1ef85b633/go.mod h1:YFoTl1xsMzdSRFIu33oCSPS/3+HZAPGpO3oOM96wXCM=
-github.com/DataDog/go-sqllexer v0.1.6 h1:skEXpWEVCpeZFIiydoIa2f2rf+ymNpjiIMqpW4w3YAk=
-github.com/DataDog/go-sqllexer v0.1.6/go.mod h1:GGpo1h9/BVSN+6NJKaEcJ9Jn44Hqc63Rakeb+24Mjgo=
-github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4=
-github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0=
+github.com/DataDog/go-sqllexer v0.2.0 h1:MLhGCsegZyZzP/UhAhxKHvdlYEwB4rsSZX6J21rcAJY=
+github.com/DataDog/go-sqllexer v0.2.0/go.mod h1:3xTFXBU69vUikYpESggScvC0RKYA7ZIdVrIkLwUOWdE=
+github.com/DataDog/go-tuf v1.1.1-0.5.2 h1:YWvghV4ZvrQsPcUw8IOUMSDpqc3W5ruOIC+KJxPknv0=
+github.com/DataDog/go-tuf v1.1.1-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0=
 github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=
 github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
-github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0 h1:5US5SqqhfkZkg/E64uvn7YmeTwnudJHtlPEH/LOT99w=
-github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0/go.mod h1:VRo4D6rj92AExpVBlq3Gcuol9Nm1bber12KyxRjKGWw=
-github.com/DataDog/sketches-go v1.4.7 h1:eHs5/0i2Sdf20Zkj0udVFWuCrXGRFig2Dcfm5rtcTxc=
-github.com/DataDog/sketches-go v1.4.7/go.mod h1:eAmQ/EBmtSO+nQp7IZMZVRPT4BQTmIc5RZQ+deGlTPM=
+github.com/DataDog/sketches-go v1.4.8 h1:pFk9BNn+Rzv8IMIoPUttoOpOr3bJOqU3P6EP5wK+Lv8=
+github.com/DataDog/sketches-go v1.4.8/go.mod h1:a/wjRUqzqtGS8qRHRPDCs4EAQfmvPDZGDlMIF5mxXOE=
 github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs=
 github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU=
-github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
-github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
@@ -80,50 +88,50 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
-github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
-github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19 h1:Gxj3kAlmM+a/VVO4YNsmgHGVUZhSxs0tuVwLIxZBCtM=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19/go.mod h1:XGq5kImVqQT4HUNbbG+0Y8O74URsPNH7CGPg1s1HW5E=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.279.2 h1:MG12Z/W1zzJLkw2gCU2gKZ872rqLM0pi9LdkZ/z3FHc=
-github.com/aws/aws-sdk-go-v2/service/ec2 v1.279.2/go.mod h1:Uy+C+Sc58jozdoL1McQr8bDsEvNFx+/nBY+vpO1HVUY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.5 h1:DKibav4XF66XSeaXcrn9GlWGHos6D/vJ4r7jsK7z5CE=
-github.com/aws/aws-sdk-go-v2/service/kms v1.49.5/go.mod h1:1SdcmEGUEQE1mrU2sIgeHtcMSxHuybhPvuEPANzIDfI=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1 h1:C2dUPSnEpy4voWFIq3JNd8gN0Y5vYGDo44eUE58a/p8=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
-github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
-github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
+github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
+github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs=
+github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.6 h1:xuOfOJR0SPBrHhzAXZ5c+8i1KyJ+aUVJ2cl8DT16qH4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.6/go.mod h1:rUVOV4y5upo55JxPss99p9FaN9BvqUjFgE/N54tvLuE=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0 h1:776KnBqePBBR6zEDi0bUIHXzUBOISa2WgAKEgckUF8M=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0/go.mod h1:rB577GvkmJADVOFGY8/j9sPv/ewcsEtQNsd9Lrn7Zx0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24=
+github.com/aws/aws-sdk-go-v2/service/kms v1.50.2 h1:UOHOXigIzDRaEU03CBQcZ5uW7FNC7E+vwfhsQWXl5RQ=
+github.com/aws/aws-sdk-go-v2/service/kms v1.50.2/go.mod h1:nAa5gmcmAmjXN3tGuhPSHLXFeWv+7nzKhjZzh8F7MH0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI=
+github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
+github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE=
@@ -132,12 +140,12 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN
 github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj0JTv4mTs=
 github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
-github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf h1:WCnJxXZXx9c8gwz598wvdqmu+YTzB9wx2X1OovK3Le8=
-github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf/go.mod h1:CeKhh8xSs3WZAc50xABMxu+FlfAAd5PNumo7NfOv7EE=
+github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20221221133751-67e37ae746cd h1:C0dfBzAdNMqxokqWUysk2KTJSMmqvh9cNW1opdy5+0Q=
+github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20221221133751-67e37ae746cd/go.mod h1:CeKhh8xSs3WZAc50xABMxu+FlfAAd5PNumo7NfOv7EE=
 github.com/buildkite/bintest/v3 v3.3.0 h1:RTWcSaJRlOT6t/K311ejPf+0J3LE/QEODzVG3vlLnWo=
 github.com/buildkite/bintest/v3 v3.3.0/go.mod h1:btqpTsVODiJcb0NMdkkmtMQ6xoFc2W/nY5yy+3I0zcs=
-github.com/buildkite/go-buildkite/v4 v4.13.1 h1:3PhrShdQwlZ2+OIWy0QbxbbjP6M97NCvarlIB6Oye+k=
-github.com/buildkite/go-buildkite/v4 v4.13.1/go.mod h1:DlebrRJqpZttXDjCW+MJ1QyW9AN++ZWt/UbPtKdbSSk=
+github.com/buildkite/go-buildkite/v4 v4.16.0 h1:uRZmOg6zfZOCpak1tizzlv9pq8Syt7WmeEb0Ov7r1NE=
+github.com/buildkite/go-buildkite/v4 v4.16.0/go.mod h1:8+7GiWBKwEPAWoZnRU/kpNCt46j1iVH8kFMMbD4YDfc=
 github.com/buildkite/go-pipeline v0.16.0 h1:wEgWUMRAgSg1ZnWOoA3AovtYYdTvN0dLY1zwUWmPP+4=
 github.com/buildkite/go-pipeline v0.16.0/go.mod h1:VE37qY3X5pmAKKUMoDZvPsHOQuyakB9cmXj9Qn6QasA=
 github.com/buildkite/interpolate v0.1.5 h1:v2Ji3voik69UZlbfoqzx+qfcsOKLA61nHdU79VV+tPU=
@@ -150,8 +158,8 @@ github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
 github.com/buildkite/zstash v0.8.0 h1:z8hBGGKIaN/UX0MZh51P8s28mWaDBkgD3uMoOu6Q96g=
 github.com/buildkite/zstash v0.8.0/go.mod h1:60v4pl8PnsVA6b09MC47aTR8Z1QNJoeQqOGp2jkEww8=
-github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf h1:yxlp0s+Sge9UsKEK0Bsvjiopb9XRk+vxylmZ9eGBfm8=
-github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -161,14 +169,14 @@ github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1
 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/creack/pty v1.1.19 h1:tUN6H7LWqNx4hQVxomd0CVsDwaDr9gaRQaI4GpSmrsA=
-github.com/creack/pty v1.1.19/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
 github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
 github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@@ -181,10 +189,10 @@ github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Z
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S6Dco8FtAhEI/qkg/00H6RdEGC+MCy5GPiQ+xweNRFE=
-github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
-github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
-github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b h1:qZ21OofI7zneC9dOEqul4FmIWz/YjJJMrf6fL7jrFYQ=
+github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
+github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
+github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -207,16 +215,12 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
 github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
-github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
-github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
-github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
-github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
 github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
-github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
 github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs=
@@ -239,18 +243,18 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.9 h1:TOpi/QG8iDcZlkQlGlFUti/ZtyLkliXvHDcyUIMuFrU=
-github.com/googleapis/enterprise-certificate-proxy v0.3.9/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
-github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y=
-github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
+github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
+github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU=
 github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII=
-github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
-github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
+github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
+github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -261,18 +265,18 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
-github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lestrrat-go/blackmagic v1.0.3 h1:94HXkVLxkZO9vJI/w2u1T0DAoprShFd13xtnSINtDWs=
-github.com/lestrrat-go/blackmagic v1.0.3/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw=
+github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA=
+github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw=
 github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
 github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
 github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k=
@@ -283,8 +287,10 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf
 github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU=
 github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
 github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
-github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc=
-github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
+github.com/linkdata/deadlock v0.5.5 h1:d6O+rzEqasSfamGDA8u7bjtaq7hOX8Ha4Zn36Wxrkvo=
+github.com/linkdata/deadlock v0.5.5/go.mod h1:tXb28stzAD3trzEEK0UJWC+rZKuobCoPktPYzebb1u0=
+github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM=
+github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -292,6 +298,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/minio/simdjson-go v0.4.5 h1:r4IQwjRGmWCQ2VeMc7fGiilu1z5du0gJ/I/FsKwgo5A=
+github.com/minio/simdjson-go v0.4.5/go.mod h1:eoNz0DcLQRyEDeaPr4Ru6JpjlZPzbA0IodxVJk8lO8E=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -304,10 +312,10 @@ github.com/oleiade/reflections v1.1.0 h1:D+I/UsXQB4esMathlt0kkZRJZdUDmhv5zGi/HOw
 github.com/oleiade/reflections v1.1.0/go.mod h1:mCxx0QseeVCHs5Um5HhJeCKVC7AwS8kO67tky4rdisA=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.125.0 h1:0dOJCEtabevxxDQmxed69oMzSw+gb3ErCnFwFYZFu0M=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.125.0/go.mod h1:QwzQhtxPThXMUDW1XRXNQ+l0GrI2BRsvNhX6ZuKyAds=
-github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.125.0 h1:F68/Nbpcvo3JZpaWlRUDJtG7xs8FHBZ7A8GOMauDkyc=
-github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.125.0/go.mod h1:haO4cJtAk05Y0p7NO9ME660xxtSh54ifCIIT7+PO9C0=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.144.0 h1:/gAMDyjZL9yQ5JqcGlwDttcQ9K+GZyBQamTRZXe2wew=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.144.0/go.mod h1:F8ASQLmExrE2TGLLWGKsmqEHFsVTqUhL2tH1bjsBi1w=
+github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.144.0 h1:lddh3Dg/abSA+YtEAUgF+aMxaRGRW68B94ttzfHIObA=
+github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.144.0/go.mod h1:Ud6F626uuq3ogpaum8wxHKQhSSXbgtEox1T85vdepRw=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0=
@@ -316,8 +324,11 @@ github.com/pact-foundation/pact-go/v2 v2.4.1 h1:eaLC58qzeCTbwdlCY8UvWz1HmDW+qrjT
 github.com/pact-foundation/pact-go/v2 v2.4.1/go.mod h1:OwnXXRliPZvKDMJn/IsAwQ95tQprmp5gPTzPYz54mTg=
 github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
 github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY=
-github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
+github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
+github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE=
+github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
+github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
+github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -333,10 +344,10 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h
 github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
+github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
+github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
+github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
 github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU=
 github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU=
 github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg=
@@ -356,24 +367,24 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 h1:8zyjtFyKi5NJySVOJRiHmSN1vl6qugQ5n9C4X7WyY3U=
 github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216/go.mod h1:hnzuad9d2wdd3z8fC6UouHQK5qZxqv3F/E6MMzXc7q0=
-github.com/secure-systems-lab/go-securesystemslib v0.9.0 h1:rf1HIbL64nUpEIZnjLZ3mcNEL9NBPB0iuVjyxvq3LZc=
-github.com/secure-systems-lab/go-securesystemslib v0.9.0/go.mod h1:DVHKMcZ+V4/woA/peqr+L0joiRXbPpQ042GgJckkFgw=
-github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
-github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14=
+github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk=
+github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
+github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
-github.com/shirou/gopsutil/v4 v4.25.8 h1:NnAsw9lN7587WHxjJA9ryDnqhJpFH6A+wagYWTOH970=
-github.com/shirou/gopsutil/v4 v4.25.8/go.mod h1:q9QdMmfAOVIw7a+eF86P7ISEU6ka+NLgkUxlopV4RwI=
+github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
+github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
 github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
-github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
-github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
-github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -390,97 +401,97 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/theckman/httpforwarded v0.4.0 h1:N55vGJT+6ojTnLY3LQCNliJC4TW0P0Pkeys1G1WpX2w=
-github.com/theckman/httpforwarded v0.4.0/go.mod h1:GVkFynv6FJreNbgH/bpOU9ITDZ7a5WuzdNCtIMI1pVI=
-github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po=
-github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
-github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
-github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4=
-github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso=
-github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
+github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s=
+github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
+github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561 h1:qqa3P9AtNn6RMe90l/lxd3eJWnIRxjI4eb5Rx8xqCLA=
+github.com/trailofbits/go-mutexasserts v0.0.0-20250514102930-c1f3d2e37561/go.mod h1:GA3+Mq3kt3tYAfM0WZCu7ofy+GW9PuGysHfhr+6JX7s=
 github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ=
 github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo=
-github.com/vektah/gqlparser/v2 v2.5.25 h1:FmWtFEa+invTIzWlWK6Vk7BVEZU/97QBzeI8Z1JjGt8=
-github.com/vektah/gqlparser/v2 v2.5.25/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo=
+github.com/vektah/gqlparser/v2 v2.5.32 h1:k9QPJd4sEDTL+qB4ncPLflqTJ3MmjB9SrVzJrawpFSc=
+github.com/vektah/gqlparser/v2 v2.5.32/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts=
 github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI=
 github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
 github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
 github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/wolfeidau/quickzip v1.0.2 h1:QPc4CVE8ECYng87o63C4t6+6ihZk1howyrZWwklKjq8=
 github.com/wolfeidau/quickzip v1.0.2/go.mod h1:ZDvxJMzI2iVp6CavOqxFq1tORhRyuxCDFw4HO25RbuA=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/collector/component v1.31.0 h1:9LzU8X1RhV3h8/QsAoTX23aFUfoJ3EUc9O/vK+hFpSI=
-go.opentelemetry.io/collector/component v1.31.0/go.mod h1:JbZl/KywXJxpUXPbt96qlEXJSym1zQ2hauMxYMuvlxM=
-go.opentelemetry.io/collector/component/componentstatus v0.125.0 h1:zlxGQZYd9kknRZSjRpOYW5SBjl0a5zYFYRPbreobXoU=
-go.opentelemetry.io/collector/component/componentstatus v0.125.0/go.mod h1:bHXc2W8bqqo9adOvCgvhcO7pYzJOSpyV4cuQ1wiIl04=
-go.opentelemetry.io/collector/component/componenttest v0.125.0 h1:E2mpnMQbkMpYoZ3Q8pHx4kod7kedjwRs1xqDpzCe/84=
-go.opentelemetry.io/collector/component/componenttest v0.125.0/go.mod h1:pQtsE1u/SPZdTphP5BZP64XbjXSq6wc+mDut5Ws/JDI=
-go.opentelemetry.io/collector/consumer v1.31.0 h1:L+y66ywxLHnAxnUxv0JDwUf5bFj53kMxCCyEfRKlM7s=
-go.opentelemetry.io/collector/consumer v1.31.0/go.mod h1:rPsqy5ni+c6xNMUkOChleZYO/nInVY6eaBNZ1FmWJVk=
-go.opentelemetry.io/collector/consumer/consumertest v0.125.0 h1:TUkxomGS4DAtjBvcWQd2UY4FDLLEKMQD6iOIDUr/5dM=
-go.opentelemetry.io/collector/consumer/consumertest v0.125.0/go.mod h1:vkHf3y85cFLDHARO/cTREVjLjOPAV+cQg7lkC44DWOY=
-go.opentelemetry.io/collector/consumer/xconsumer v0.125.0 h1:oTreUlk1KpMSWwuHFnstW+orrjGTyvs2xd3o/Dpy+hI=
-go.opentelemetry.io/collector/consumer/xconsumer v0.125.0/go.mod h1:FX0G37r0W+wXRgxxFtwEJ4rlsCB+p0cIaxtU3C4hskw=
-go.opentelemetry.io/collector/featuregate v1.31.0 h1:20q7plPQZwmAiaYAa6l1m/i2qDITZuWlhjr4EkmeQls=
-go.opentelemetry.io/collector/featuregate v1.31.0/go.mod h1:Y/KsHbvREENKvvN9RlpiWk/IGBK+CATBYzIIpU7nccc=
-go.opentelemetry.io/collector/internal/telemetry v0.125.0 h1:6lcGOxw3dAg7LfXTKdN8ZjR+l7KvzLdEiPMhhLwG4r4=
-go.opentelemetry.io/collector/internal/telemetry v0.125.0/go.mod h1:5GyFslLqjZgq1DZTtFiluxYhhXrCofHgOOOybodDPGE=
-go.opentelemetry.io/collector/pdata v1.31.0 h1:P5WuLr1l2JcIvr6Dw2hl01ltp2ZafPnC4Isv+BLTBqU=
-go.opentelemetry.io/collector/pdata v1.31.0/go.mod h1:m41io9nWpy7aCm/uD1L9QcKiZwOP0ldj83JEA34dmlk=
-go.opentelemetry.io/collector/pdata/pprofile v0.125.0 h1:Qqlx8w1HpiYZ9RQqjmMQIysI0cHNO1nh3E/fCTeFysA=
-go.opentelemetry.io/collector/pdata/pprofile v0.125.0/go.mod h1:p/yK023VxAp8hm27/1G5DPTcMIpnJy3cHGAFUQZGyaQ=
-go.opentelemetry.io/collector/pdata/testdata v0.125.0 h1:due1Hl0EEVRVwfCkiamRy5E8lS6yalv0lo8Zl/SJtGw=
-go.opentelemetry.io/collector/pdata/testdata v0.125.0/go.mod h1:1GpEWlgdMrd+fWsBk37ZC2YmOP5YU3gFQ4rWuCu9g24=
-go.opentelemetry.io/collector/pipeline v0.125.0 h1:oitBgcAFqntDB4ihQJUHJSQ8IHqKFpPkaTVbTYdIUzM=
-go.opentelemetry.io/collector/pipeline v0.125.0/go.mod h1:TO02zju/K6E+oFIOdi372Wk0MXd+Szy72zcTsFQwXl4=
-go.opentelemetry.io/collector/processor v1.31.0 h1:+u7sBUpnCBsHYoALp4hfr9VEjLHHYa4uKENGITe0K9Q=
-go.opentelemetry.io/collector/processor v1.31.0/go.mod h1:5hDYJ7/hTdfd2tF2Rj5Hs6+mfyFz2O7CaPzVvW1qHQc=
-go.opentelemetry.io/collector/processor/processorhelper v0.125.0 h1:QRpX7oFW88DAZhy+Q93npklRoaQr8ue0GKpeup7C/Fk=
-go.opentelemetry.io/collector/processor/processorhelper v0.125.0/go.mod h1:oXRvslUuN62wErcoJrcEJYoTXu5wHyNyJsE+/a9Cc9s=
-go.opentelemetry.io/collector/processor/processortest v0.125.0 h1:ZVAN4iZPDcWhpzKqnuok2NIuS5hwGVVQUOWkJFR12tA=
-go.opentelemetry.io/collector/processor/processortest v0.125.0/go.mod h1:VAw0IRG35cWTBjBtreXeXJEgqkRegfjrH/EuLhNX2+I=
-go.opentelemetry.io/collector/processor/xprocessor v0.125.0 h1:VWYPMW1VmDq6xB7M5SYjBpQCCIq3MhQ3W++wU47QpZM=
-go.opentelemetry.io/collector/processor/xprocessor v0.125.0/go.mod h1:bCxUyFVlksANg8wjYZqWVsRB33lkLQ294rTrju/IZiM=
-go.opentelemetry.io/collector/semconv v0.125.0 h1:SyRP617YGvNSWRSKMy7Lbk9RaJSR+qFAAfyxJOeZe4s=
-go.opentelemetry.io/collector/semconv v0.125.0/go.mod h1:te6VQ4zZJO5Lp8dM2XIhDxDiL45mwX0YAQQWRQ0Qr9U=
-go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU=
-go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY=
-go.opentelemetry.io/contrib/propagators/aws v1.40.0 h1:4VIrh75jW4RTimUNx1DSk+6H9/nDr1FvmKoOVDh3K04=
-go.opentelemetry.io/contrib/propagators/aws v1.40.0/go.mod h1:B0dCov9KNQGlut3T8wZZjDnLXEXdBroM7bFsHh/gRos=
-go.opentelemetry.io/contrib/propagators/b3 v1.40.0 h1:xariChe8OOVF3rNlfzGFgQc61npQmXhzZj/i82mxMfg=
-go.opentelemetry.io/contrib/propagators/b3 v1.40.0/go.mod h1:72WvbdxbOfXaELEQfonFfOL6osvcVjI7uJEE8C2nkrs=
-go.opentelemetry.io/contrib/propagators/jaeger v1.40.0 h1:aXl9uobjJs5vquMLt9ZkI/3zIuz8XQ3TqOKSWx0/xdU=
-go.opentelemetry.io/contrib/propagators/jaeger v1.40.0/go.mod h1:ioMePqe6k6c/ovXSkmkMr1mbN5qRBGJxNTVop7/2XO0=
-go.opentelemetry.io/contrib/propagators/ot v1.40.0 h1:Lon8J5SPmWaL1Ko2TIlCNHJ42/J1b5XbJlgJaE/9m7I=
-go.opentelemetry.io/contrib/propagators/ot v1.40.0/go.mod h1:dKWtJTlp1Yj+8Cneye5idO46eRPIbi23qVuJYKjNnvY=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40=
-go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y=
-go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
-go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
-go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/collector/component v1.53.0 h1:A+GU9n4eKnFVmrr7NPpbVvJ1kp985jXtachb9gy12mk=
+go.opentelemetry.io/collector/component v1.53.0/go.mod h1:yqyFwDuP4JKwOFaxdqoWj25aVthtavGkSDp2K42x+YY=
+go.opentelemetry.io/collector/component/componentstatus v0.144.0 h1:ahrQ66clOcPJuCxoEe1Lm0agIC/3Css4sMHouYFWV34=
+go.opentelemetry.io/collector/component/componentstatus v0.144.0/go.mod h1:PwtvA7cYiIb4e4ZbOmovMpLn1No5jRB4rgmnyoZikEw=
+go.opentelemetry.io/collector/component/componenttest v0.144.0 h1:Ah7E3OVdc3QKu8gyxpxkm4a5TAypUIAICNgY/6GW0sY=
+go.opentelemetry.io/collector/component/componenttest v0.144.0/go.mod h1:4YV3d9+4nhxrtOdFHcX80/YQHK4bFTxyxCgonJgXNGs=
+go.opentelemetry.io/collector/consumer v1.50.0 h1:Sxbue3zNH3IJla+vUyMXEiomfRJaS6wemZd4qv5na48=
+go.opentelemetry.io/collector/consumer v1.50.0/go.mod h1:GB6gfWsZyeTBWn+Cb3ITkJaH4aA5NW0r2Dm+VLFnD/M=
+go.opentelemetry.io/collector/consumer/consumertest v0.144.0 h1:R2iR10e2rK+9xCCyl/OH0A/SyYzAauFGePovNQlOz90=
+go.opentelemetry.io/collector/consumer/consumertest v0.144.0/go.mod h1:4Mpk+JdFQOjPPxeyRORCgQFWJiCE9Rq0P/6vP3OaNEs=
+go.opentelemetry.io/collector/consumer/xconsumer v0.144.0 h1:7J6FCC2qAR2ZHKYX9hH1zvH0+G8E0mc1FZ1V8y/ZAkg=
+go.opentelemetry.io/collector/consumer/xconsumer v0.144.0/go.mod h1:FagtMUc1f8sPryGwyZNCTix20kmO51LKqaZ7FYLj2y0=
+go.opentelemetry.io/collector/featuregate v1.53.0 h1:cgjXdtl7jezWxq6V0eohe/JqjY4PBotZGb5+bTR2OJw=
+go.opentelemetry.io/collector/featuregate v1.53.0/go.mod h1:PS7zY/zaCb28EqciePVwRHVhc3oKortTFXsi3I6ee4g=
+go.opentelemetry.io/collector/internal/componentalias v0.144.0 h1:LO9QWYbce01aP38i5RI6UQsCSa5FSv6fs55qobpvMGQ=
+go.opentelemetry.io/collector/internal/componentalias v0.144.0/go.mod h1:oAZoM7bcqeeQ2mpXaThkhGeTzxceZ6/LnIlUZ7GiC40=
+go.opentelemetry.io/collector/internal/testutil v0.147.0 h1:DFlRxBRp23/sZnpTITK25yqe0d56yNvK+63IaWc6OsU=
+go.opentelemetry.io/collector/internal/testutil v0.147.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE=
+go.opentelemetry.io/collector/pdata v1.53.0 h1:DlYDbRwammEZaxDZHINx5v0n8SEOVNniPbi6FRTlVkA=
+go.opentelemetry.io/collector/pdata v1.53.0/go.mod h1:LRSYGNjKXaUrZEwZv3Yl+8/zV2HmRGKXW62zB2bysms=
+go.opentelemetry.io/collector/pdata/pprofile v0.147.0 h1:yQS3RBvcvRcy9N7AnJvsxmse0AxJcRqBZfwMA22xBA8=
+go.opentelemetry.io/collector/pdata/pprofile v0.147.0/go.mod h1:pm9mUqHNpT1SaCkxILu4FW1BvMAelh7EKhpSKe2KJIQ=
+go.opentelemetry.io/collector/pdata/testdata v0.144.0 h1:zg1XWm/S/fBrFy5lr56DLrI5PVFB2sZxU0q5Yf/71Ko=
+go.opentelemetry.io/collector/pdata/testdata v0.144.0/go.mod h1:uOhCQeFRoBsrCoE4wlxvWnVYYfwdcgtnp5tTJuV/g5g=
+go.opentelemetry.io/collector/pipeline v1.50.0 h1:yOOSvkzpX3yOfO4qvLsUhQflFZ9MI4FmcL+gsAx/WgQ=
+go.opentelemetry.io/collector/pipeline v1.50.0/go.mod h1:xUrAqiebzYbrgxyoXSkk6/Y3oi5Sy3im2iCA51LwUAI=
+go.opentelemetry.io/collector/processor v1.50.0 h1:RP7kKIZBu1LjVd9dEYUxvYdbQRKg1V+g5NvkYY2nA7U=
+go.opentelemetry.io/collector/processor v1.50.0/go.mod h1:pEs55PVHE67Ov327Q7ikkNsy8E0dGmhBqWwJDuyBxMw=
+go.opentelemetry.io/collector/processor/processorhelper v0.144.0 h1:DZef7rGngEcy3ZuJ3zb4BdOAxK7xrYBm1pQu/zoWGA4=
+go.opentelemetry.io/collector/processor/processorhelper v0.144.0/go.mod h1:B6lbjKY3t4UMjinR/sZWa6I9pwkObXOojqujVS79CeU=
+go.opentelemetry.io/collector/processor/processortest v0.144.0 h1:1OqDusu0YLHlpOCTI4Qi+QxaoqTEkuN3BvzvWjpZC6c=
+go.opentelemetry.io/collector/processor/processortest v0.144.0/go.mod h1:kxHoHyfKOvWZu3AmiRrrMxafTODlvIEcyUxeJSqm8+s=
+go.opentelemetry.io/collector/processor/xprocessor v0.144.0 h1:KgOK28goG/wtmPHxG/P+hWSS3lnR+ylr8f20Xo5wEiU=
+go.opentelemetry.io/collector/processor/xprocessor v0.144.0/go.mod h1:b/qLCOr5NIy64cP7a8aD0BgYCa9xpWzj/XF1SUx8Ky0=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
+go.opentelemetry.io/contrib/propagators/aws v1.42.0 h1:Kbr3xDxs6kcxp5ThXTKWK2OtwLhNoXBVtqguNYcsZL0=
+go.opentelemetry.io/contrib/propagators/aws v1.42.0/go.mod h1:Jzw9hZHtxdpCN7x8S17UH59X/EiFivp6VXLs9bdM1OQ=
+go.opentelemetry.io/contrib/propagators/b3 v1.42.0 h1:B2Pew5ufEtgkjLF+tSkXjgYZXQr9m7aCm1wLKB0URbU=
+go.opentelemetry.io/contrib/propagators/b3 v1.42.0/go.mod h1:iPgUcSEF5DORW6+yNbdw/YevUy+QqJ508ncjhrRSCjc=
+go.opentelemetry.io/contrib/propagators/jaeger v1.42.0 h1:jP8unWI6q5kcb3gpGLjKDGaUa+JW+nHKWvpS/q+YuWA=
+go.opentelemetry.io/contrib/propagators/jaeger v1.42.0/go.mod h1:xd89e/pUyPatUP1C4z1UknD9jHptESO99tWyvd4mWD4=
+go.opentelemetry.io/contrib/propagators/ot v1.42.0 h1:uQjD1NNqX1+DfcAoWParPt1egNg9vC9gH4xarJ9Khxo=
+go.opentelemetry.io/contrib/propagators/ot v1.42.0/go.mod h1:yw/c2TCmQLIv109HBOCn6NlJ8Dp7MNfjMcqQZRnAMmg=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 h1:uLXP+3mghfMf7XmV4PkGfFhFKuNWoCvvx5wP/wOXo0o=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0/go.mod h1:v0Tj04armyT59mnURNUJf7RCKcKzq+lgJs6QSjHjaTc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
+go.opentelemetry.io/proto/slim/otlp v1.9.0 h1:fPVMv8tP3TrsqlkH1HWYUpbCY9cAIemx184VGkS6vlE=
+go.opentelemetry.io/proto/slim/otlp v1.9.0/go.mod h1:xXdeJJ90Gqyll+orzUkY4bOd2HECo5JofeoLpymVqdI=
+go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.2.0 h1:o13nadWDNkH/quoDomDUClnQBpdQQ2Qqv0lQBjIXjE8=
+go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.2.0/go.mod h1:Gyb6Xe7FTi/6xBHwMmngGoHqL0w29Y4eW8TGFzpefGA=
+go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.2.0 h1:EiUYvtwu6PMrMHVjcPfnsG3v+ajPkbUeH+IL93+QYyk=
+go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.2.0/go.mod h1:mUUHKFiN2SST3AhJ8XhJxEoeVW12oqfXog0Bo8W3Ec4=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
@@ -488,42 +499,36 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
-go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
+go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
 golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
-golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
-golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
-golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
-golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
-golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
-golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -532,8 +537,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
 golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
@@ -541,35 +546,32 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
 golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.260.0 h1:XbNi5E6bOVEj/uLXQRlt6TKuEzMD7zvW/6tNwltE4P4=
-google.golang.org/api v0.260.0/go.mod h1:Shj1j0Phr/9sloYrKomICzdYgsSDImpTxME8rGLaZ/o=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/api v0.270.0 h1:4rJZbIuWSTohczG9mG2ukSDdt9qKx4sSSHIydTN26L4=
+google.golang.org/api v0.270.0/go.mod h1:5+H3/8DlXpQWrSz4RjGGwz5HfJAQSEI8Bc6JqQNH77U=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
-google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934=
-google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
-google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
-google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
-google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
+google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4=
+google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU=
+google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/DataDog/dd-trace-go.v1 v1.74.8 h1:h96ji92t9eXbPvSWhJ+lrPWetHiQNYlt48JKRO09NFA=
@@ -577,8 +579,8 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.74.8/go.mod h1:LpHbtHsCZBlm1HWrlVOUQcEXwMWZnU
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
-gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k=
+gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -588,7 +590,7 @@ gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo=
 gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q=
 gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
 gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
-k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
-k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
+k8s.io/apimachinery v0.35.0-alpha.0 h1:FrJ3gqYFPIldvKa2KHzmT0lL0gqcRr1GiS6thHvdSGM=
+k8s.io/apimachinery v0.35.0-alpha.0/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw=
 mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4=
 mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s=
diff --git a/internal/experiments/BUILD.bazel b/internal/experiments/BUILD.bazel
index 7152249f93..d99bb994e8 100644
--- a/internal/experiments/BUILD.bazel
+++ b/internal/experiments/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@rules_go//go:def.bzl", "go_library")
+load("@rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "experiments",
@@ -7,3 +7,9 @@ go_library(
     visibility = ["//:__subpackages__"],
     deps = ["//logger"],
 )
+
+go_test(
+    name = "experiments_test",
+    srcs = ["experiments_test.go"],
+    embed = [":experiments"],
+)
diff --git a/internal/job/BUILD.bazel b/internal/job/BUILD.bazel
index a49d81ac21..b613606907 100644
--- a/internal/job/BUILD.bazel
+++ b/internal/job/BUILD.bazel
@@ -35,7 +35,6 @@ go_library(
         "//internal/socket",
         "//internal/tempfile",
         "//jobapi",
-        "//kubernetes",
         "//logger",
         "//process",
         "//tracetools",
diff --git a/internal/secrets/BUILD.bazel b/internal/secrets/BUILD.bazel
index 9329d7acc4..60a0f34675 100644
--- a/internal/secrets/BUILD.bazel
+++ b/internal/secrets/BUILD.bazel
@@ -10,6 +10,8 @@ go_library(
     visibility = ["//:__subpackages__"],
     deps = [
         "//api",
+        "//logger",
+        "@com_github_buildkite_roko//:roko",
         "@org_golang_x_sync//semaphore",
     ],
 )
diff --git a/kubernetes/BUILD.bazel b/kubernetes/BUILD.bazel
index 9aed5ca97a..9dfe99a347 100644
--- a/kubernetes/BUILD.bazel
+++ b/kubernetes/BUILD.bazel
@@ -4,6 +4,7 @@ go_library(
     name = "kubernetes",
     srcs = [
         "client.go",
+        "doc.go",
         "runner.go",
         "umask.go",
         "umask_windows.go",

From 361ee1fcd9131b1138b088ae6d952da42813456e Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 10 Mar 2026 15:22:30 +1100
Subject: [PATCH 234/242] fix: Use targetPath helper and tempfile for Azure
 Blob download

---
 internal/artifact/azure_blob_downloader.go | 49 ++++++++++++++++++----
 1 file changed, 42 insertions(+), 7 deletions(-)

diff --git a/internal/artifact/azure_blob_downloader.go b/internal/artifact/azure_blob_downloader.go
index 712b21fcab..49004a88d0 100644
--- a/internal/artifact/azure_blob_downloader.go
+++ b/internal/artifact/azure_blob_downloader.go
@@ -2,11 +2,14 @@ package artifact
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/buildkite/agent/v3/logger"
+	"github.com/dustin/go-humanize"
 )
 
 // AzureBlobUploaderConfig configures AzureBlobDownloader.
@@ -47,17 +50,27 @@ func (d *AzureBlobDownloader) Start(ctx context.Context) error {
 		return err
 	}
 
-	f, err := os.Create(d.conf.Path)
+	targetPath := targetPath(ctx, d.conf.Path, d.conf.Destination)
+	targetDirectory, targetFile := filepath.Split(targetPath)
+
+	// Now make the folder for our file
+	// Actual file permissions will be reduced by umask, and won't be 0o777 unless the user has manually changed the umask to 000
+	if err := os.MkdirAll(targetDirectory, 0o777); err != nil {
+		return fmt.Errorf("creating directory for %s (%T: %w)", targetPath, err, err)
+	}
+
+	// Create a temporary file to write to.
+	temp, err := os.CreateTemp(targetDirectory, targetFile)
 	if err != nil {
-		return err
+		return fmt.Errorf("creating temp file (%T: %w)", err, err)
 	}
-	// Best-effort close for cleanup - Close error returned for checking below.
-	defer f.Close() //nolint:errcheck
+	defer os.Remove(temp.Name()) //nolint:errcheck // Best-effort cleanup
+	defer temp.Close()           //nolint:errcheck // Best-effort cleanup - primary Close checked below.
 
 	fullPath := path.Join(loc.BlobPath, d.conf.Path)
 
 	// Show a nice message that we're starting to download the file
-	d.logger.Debug("Downloading %s to %s", loc.URL(d.conf.Path), d.conf.Path)
+	d.logger.Debug("Downloading %s to %s", loc.URL(d.conf.Path), targetPath)
 
 	opts := &azblob.DownloadFileOptions{
 		RetryReaderOptionsPerBlock: azblob.RetryReaderOptions{
@@ -65,9 +78,31 @@ func (d *AzureBlobDownloader) Start(ctx context.Context) error {
 		},
 	}
 	bc := client.NewContainerClient(loc.ContainerName).NewBlobClient(fullPath)
-	if _, err := bc.DownloadFile(ctx, f, opts); err != nil {
+	bytes, err := bc.DownloadFile(ctx, temp, opts)
+	if err != nil {
 		return err
 	}
 
-	return f.Close()
+	// os.CreateTemp uses 0o600 permissions, but in the past we used os.Create
+	// which uses 0x666. Since these are set at open time, they are restricted
+	// by umask.
+	if err := temp.Chmod(0o666 &^ umask); err != nil {
+		return fmt.Errorf("setting file permissions (%T: %w)", err, err)
+	}
+
+	// close must succeed for the file to be considered properly written.
+	if err := temp.Close(); err != nil {
+		return fmt.Errorf("closing temp file (%T: %w)", err, err)
+	}
+
+	// Rename the temp file to its intended name within the same directory.
+	// On Unix-like platforms this is generally an "atomic replace".
+	// Caveats: https://pkg.go.dev/os#Rename
+	if err := os.Rename(temp.Name(), targetPath); err != nil {
+		return fmt.Errorf("renaming temp file to target (%T: %w)", err, err)
+	}
+
+	d.logger.Info("Successfully downloaded %q %s", d.conf.Path, humanize.IBytes(uint64(bytes)))
+
+	return nil
 }

From 192cfd0d6f927490e7df2c95306021b4352df128 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 10 Mar 2026 17:00:08 +1100
Subject: [PATCH 235/242] fix: Make submodule clone config an agent config

---
 agent/agent_configuration.go |  1 +
 agent/job_runner.go          |  2 ++
 clicommand/agent_start.go    | 30 +++++++++++++++++++-----------
 clicommand/bootstrap.go      |  2 +-
 4 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go
index 664c373753..051210e273 100644
--- a/agent/agent_configuration.go
+++ b/agent/agent_configuration.go
@@ -24,6 +24,7 @@ type AgentConfiguration struct {
 	GitCleanFlags               string
 	GitFetchFlags               string
 	GitSubmodules               bool
+	GitSubmoduleCloneConfig     []string
 	SkipCheckout                bool
 	GitSkipFetchExistingCommits bool
 	AllowedRepositories         []*regexp.Regexp
diff --git a/agent/job_runner.go b/agent/job_runner.go
index b3282987b1..1edd1f88f9 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -450,6 +450,7 @@ BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT
 BUILDKITE_GIT_MIRRORS_PATH
 BUILDKITE_GIT_MIRRORS_SKIP_UPDATE
 BUILDKITE_GIT_SUBMODULES
+BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG
 BUILDKITE_CANCEL_GRACE_PERIOD
 BUILDKITE_COMMAND_EVAL
 BUILDKITE_LOCAL_HOOKS_ENABLED
@@ -586,6 +587,7 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setEnv("BUILDKITE_GIT_CLONE_MIRROR_FLAGS", r.conf.AgentConfiguration.GitCloneMirrorFlags)
 	setEnv("BUILDKITE_GIT_CLEAN_FLAGS", r.conf.AgentConfiguration.GitCleanFlags)
 	setEnv("BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", strconv.Itoa(r.conf.AgentConfiguration.GitMirrorsLockTimeout))
+	setEnv("BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG", strings.Join(r.conf.AgentConfiguration.GitSubmoduleCloneConfig, ","))
 
 	setEnv("BUILDKITE_SHELL", r.conf.AgentConfiguration.Shell)
 	setEnv("BUILDKITE_AGENT_EXPERIMENT", strings.Join(experiments.Enabled(ctx), ","))
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index baf490d186..acb18ebc25 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -150,17 +150,18 @@ type AgentStartConfig struct {
 	WaitForECSMetaDataTimeout string   `cli:"wait-for-ecs-meta-data-timeout"`
 	WaitForGCPLabelsTimeout   string   `cli:"wait-for-gcp-labels-timeout"`
 
-	GitCheckoutFlags            string `cli:"git-checkout-flags"`
-	GitCloneFlags               string `cli:"git-clone-flags"`
-	GitCloneMirrorFlags         string `cli:"git-clone-mirror-flags"`
-	GitCleanFlags               string `cli:"git-clean-flags"`
-	GitFetchFlags               string `cli:"git-fetch-flags"`
-	GitMirrorsPath              string `cli:"git-mirrors-path" normalize:"filepath"`
-	GitMirrorsLockTimeout       int    `cli:"git-mirrors-lock-timeout"`
-	GitMirrorsSkipUpdate        bool   `cli:"git-mirrors-skip-update"`
-	NoGitSubmodules             bool   `cli:"no-git-submodules"`
-	SkipCheckout                bool   `cli:"skip-checkout"`
-	GitSkipFetchExistingCommits bool   `cli:"git-skip-fetch-existing-commits"`
+	GitCheckoutFlags            string   `cli:"git-checkout-flags"`
+	GitCloneFlags               string   `cli:"git-clone-flags"`
+	GitCloneMirrorFlags         string   `cli:"git-clone-mirror-flags"`
+	GitCleanFlags               string   `cli:"git-clean-flags"`
+	GitFetchFlags               string   `cli:"git-fetch-flags"`
+	GitMirrorsPath              string   `cli:"git-mirrors-path" normalize:"filepath"`
+	GitMirrorsLockTimeout       int      `cli:"git-mirrors-lock-timeout"`
+	GitMirrorsSkipUpdate        bool     `cli:"git-mirrors-skip-update"`
+	NoGitSubmodules             bool     `cli:"no-git-submodules"`
+	GitSubmoduleCloneConfig     []string `cli:"git-submodule-clone-config"`
+	SkipCheckout                bool     `cli:"skip-checkout"`
+	GitSkipFetchExistingCommits bool     `cli:"git-skip-fetch-existing-commits"`
 
 	NoSSHKeyscan            bool     `cli:"no-ssh-keyscan"`
 	NoCommandEval           bool     `cli:"no-command-eval"`
@@ -551,6 +552,12 @@ var AgentStartCommand = cli.Command{
 			Usage:  "Skip updating the Git mirror (default: false)",
 			EnvVar: "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE",
 		},
+		cli.StringSliceFlag{
+			Name:   "git-submodule-clone-config",
+			Value:  &cli.StringSlice{},
+			Usage:  "Comma separated key=value git config pairs applied before git submodule clone commands such as ′update --init′. If the config is needed to be applied to all git commands, supply it in a global git config file for the system that the agent runs in instead",
+			EnvVar: "BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG",
+		},
 		cli.StringFlag{
 			Name:   "bootstrap-script",
 			Value:  "",
@@ -1080,6 +1087,7 @@ var AgentStartCommand = cli.Command{
 			GitCleanFlags:                cfg.GitCleanFlags,
 			GitFetchFlags:                cfg.GitFetchFlags,
 			GitSubmodules:                !cfg.NoGitSubmodules,
+			GitSubmoduleCloneConfig:      cfg.GitSubmoduleCloneConfig,
 			SkipCheckout:                 cfg.SkipCheckout,
 			GitSkipFetchExistingCommits:  cfg.GitSkipFetchExistingCommits,
 			SSHKeyscan:                   !cfg.NoSSHKeyscan,
diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go
index eec06e2050..16532aee30 100644
--- a/clicommand/bootstrap.go
+++ b/clicommand/bootstrap.go
@@ -78,7 +78,7 @@ type BootstrapConfig struct {
 	GitMirrorsPath               string   `cli:"git-mirrors-path" normalize:"filepath"`
 	GitMirrorsLockTimeout        int      `cli:"git-mirrors-lock-timeout"`
 	GitMirrorsSkipUpdate         bool     `cli:"git-mirrors-skip-update"`
-	GitSubmoduleCloneConfig      []string `cli:"git-submodule-clone-config"`
+	GitSubmoduleCloneConfig      []string `cli:"git-submodule-clone-config" normalize:"list"`
 	BinPath                      string   `cli:"bin-path" normalize:"filepath"`
 	BuildPath                    string   `cli:"build-path" normalize:"filepath"`
 	HooksPath                    string   `cli:"hooks-path" normalize:"filepath"`

From 7cd6f6c93921c9cc5af941894950c814c5ff9f0c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 10 Mar 2026 23:25:50 +0000
Subject: [PATCH 236/242] build(deps): bump the container-images group across 5
 directories with 1 update

Bumps the container-images group with 1 update in the /packaging/docker/alpine directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/alpine-k8s directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-20.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-22.04 directory: buildkite/agent-base.
Bumps the container-images group with 1 update in the /packaging/docker/ubuntu-24.04 directory: buildkite/agent-base.


Updates `buildkite/agent-base` from `e4ae53a` to `69aece0`

Updates `buildkite/agent-base` from `2ffa7f7` to `637ed54`

Updates `buildkite/agent-base` from `d1a0b17` to `639d0da`

Updates `buildkite/agent-base` from `861ff33` to `2275eff`

Updates `buildkite/agent-base` from `e1bf16d` to `3566e5d`

---
updated-dependencies:
- dependency-name: buildkite/agent-base
  dependency-version: alpine
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: alpine-k8s
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-focal
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-jammy
  dependency-type: direct:production
  dependency-group: container-images
- dependency-name: buildkite/agent-base
  dependency-version: ubuntu-noble
  dependency-type: direct:production
  dependency-group: container-images
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 packaging/docker/alpine-k8s/Dockerfile   | 2 +-
 packaging/docker/alpine/Dockerfile       | 2 +-
 packaging/docker/ubuntu-20.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-22.04/Dockerfile | 2 +-
 packaging/docker/ubuntu-24.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packaging/docker/alpine-k8s/Dockerfile b/packaging/docker/alpine-k8s/Dockerfile
index 4ab14e277b..9f4605af70 100644
--- a/packaging/docker/alpine-k8s/Dockerfile
+++ b/packaging/docker/alpine-k8s/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:2ffa7f782edbf46a9f283b0c326b5ec0f44c7d690db4fd50040f73013e13f29f
+FROM public.ecr.aws/buildkite/agent-base:alpine-k8s@sha256:238641388435829ef06a5ac88a45defb1e6b4c4385fbaf7cb9547ff7fe03e1fc
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/alpine/Dockerfile b/packaging/docker/alpine/Dockerfile
index aea2de9fb8..5d73f5aec0 100644
--- a/packaging/docker/alpine/Dockerfile
+++ b/packaging/docker/alpine/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:e4ae53ac87c5b870537c3509e8dfe1ea95cba33ead9b2958fd2a8c7ae301ec1f
+FROM public.ecr.aws/buildkite/agent-base:alpine@sha256:c9ff93a4f2e77978c7fcd7e098376080bd66d8860e40464a8c68247444a70873
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-20.04/Dockerfile b/packaging/docker/ubuntu-20.04/Dockerfile
index fa134042a3..4571f91816 100644
--- a/packaging/docker/ubuntu-20.04/Dockerfile
+++ b/packaging/docker/ubuntu-20.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:d1a0b17c3fef91918940272dcb1f67222d8d1afa7a1dfcd7a3834bd96ddd23ff
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-focal@sha256:88e4ef6e752e3966c081e38d6f280350f05aaa04bcaf563f2dba368437350d15
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-22.04/Dockerfile b/packaging/docker/ubuntu-22.04/Dockerfile
index d89dd512c2..0015ec8c1e 100644
--- a/packaging/docker/ubuntu-22.04/Dockerfile
+++ b/packaging/docker/ubuntu-22.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:861ff33773b82f0d3677d735e95876c867a97593705ff903eecbad30d07f94e4
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-jammy@sha256:3821ffab88bbdc9234f97c25d430fec955ce88a3928980db676cfd13b79ec9b3
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/packaging/docker/ubuntu-24.04/Dockerfile b/packaging/docker/ubuntu-24.04/Dockerfile
index 9e34e94745..22de6666dd 100644
--- a/packaging/docker/ubuntu-24.04/Dockerfile
+++ b/packaging/docker/ubuntu-24.04/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:e1bf16dc4c2c7b1fa382322c52e3edee632b4491166af33fb83be58dad5d7886
+FROM public.ecr.aws/buildkite/agent-base:ubuntu-noble@sha256:94a082dbb792a71048f0e7ae0683d9cb02a7bfd4d381f6cb60e49de1596c54a6
 ARG TARGETOS
 ARG TARGETARCH
 

From 17d38d00a678a65fb31f3f92c79eb2671da83948 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Tue, 10 Mar 2026 10:55:52 +1100
Subject: [PATCH 237/242] fix: potential deadlock in baton

---
 agent/agent_worker.go           |  6 ++-
 agent/agent_worker_debouncer.go |  3 +-
 agent/agent_worker_ping.go      |  3 +-
 agent/baton.go                  | 86 ++++++++++++++-------------------
 agent/baton_test.go             | 43 +++++++++++++++++
 5 files changed, 87 insertions(+), 54 deletions(-)
 create mode 100644 agent/baton_test.go

diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index 5625351056..e670bc350b 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -339,7 +339,8 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	switch a.agentConfiguration.PingMode {
 	case "", PingModeAuto: // note: "" can happen in some tests
 		loops = []func(){pingLoop, streamingLoop, debouncerLoop}
-		bat.Acquire(actorDebouncer)
+		<-bat.Acquire()
+		bat.Acquired(actorDebouncer)
 
 	case PingModePollOnly:
 		loops = []func(){pingLoop}
@@ -348,7 +349,8 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	case PingModeStreamOnly:
 		loops = []func(){streamingLoop, debouncerLoop}
 		fromPingLoopCh = nil // prevent action loop listening to ping side
-		bat.Acquire(actorDebouncer)
+		<-bat.Acquire()
+		bat.Acquired(actorDebouncer)
 
 	default:
 		return fmt.Errorf("unknown ping mode %q", a.agentConfiguration.PingMode)
diff --git a/agent/agent_worker_debouncer.go b/agent/agent_worker_debouncer.go
index f09cc0b000..c38e1616da 100644
--- a/agent/agent_worker_debouncer.go
+++ b/agent/agent_worker_debouncer.go
@@ -67,7 +67,8 @@ func (a *AgentWorker) runDebouncer(ctx context.Context, bat *baton, outCh chan<-
 			a.logger.Debug("[runDebouncer] Stopping due to context cancel")
 			return ctx.Err()
 
-		case <-iif(healthy, bat.Acquire(actorDebouncer)): // if the stream is healthy, take the baton if available
+		case <-iif(healthy, bat.Acquire()): // if the stream is healthy, take the baton if available
+			bat.Acquired(actorDebouncer)
 			a.logger.Debug("[runDebouncer] Took the baton")
 			// We now have the baton!
 			// continue below to send any pending message, if able
diff --git a/agent/agent_worker_ping.go b/agent/agent_worker_ping.go
index 7e4957ad03..2dd1cbac45 100644
--- a/agent/agent_worker_ping.go
+++ b/agent/agent_worker_ping.go
@@ -112,7 +112,8 @@ func (a *pingLoopState) pingLoopInner(ctx context.Context) error {
 	// preferred.
 	a.logger.Debug("[runPingLoop] Waiting for baton")
 	select {
-	case <-a.bat.Acquire(actorPingLoop): // the baton is ours!
+	case <-a.bat.Acquire(): // the baton is ours!
+		a.bat.Acquired(actorPingLoop)
 		a.logger.Debug("[runPingLoop] Acquired the baton")
 		defer func() { // <- this is why the ping loop body is in a func
 			a.logger.Debug("[runPingLoop] Releasing the baton")
diff --git a/agent/baton.go b/agent/baton.go
index 0603935a75..a8e5eea151 100644
--- a/agent/baton.go
+++ b/agent/baton.go
@@ -5,17 +5,19 @@ import "sync"
 // baton is a channel-based mutex. This allows for using it as part of a select
 // statement.
 type baton struct {
-	mu      sync.Mutex
-	holder  string
-	acquire map[string]chan struct{}
+	mu     sync.Mutex
+	holder string
+	ch     chan struct{}
 }
 
 // newBaton creates a new baton for sharing among actors, each identified
-// by a non-empty string.
+// by a non-empty string. The baton is initially not held by anything.
 func newBaton() *baton {
-	return &baton{
-		acquire: make(map[string]chan struct{}),
+	b := &baton{
+		ch: make(chan struct{}, 1),
 	}
+	b.ch <- struct{}{}
+	return b
 }
 
 // HeldBy reports if the actor specified by the argument holds the baton.
@@ -26,61 +28,45 @@ func (b *baton) HeldBy(by string) bool {
 }
 
 // Acquire returns a channel that receives when the baton is acquired by the
-// acquirer.
-func (b *baton) Acquire(by string) <-chan struct{} {
+// caller.
+// Be sure to call [baton.Acquired] after receiving from the channel, e.g.
+//
+//		select {
+//		case <-bat.Acquire():
+//			bat.Acquired("me")
+//			defer bat.Release("me")
+//	 	...
+//	 	}
+func (b *baton) Acquire() <-chan struct{} {
+	// b.ch should never change, so no need to lock around it
+	return b.ch
+}
+
+// Acquired must be called by the actor that successfully acquired the baton
+// immediately after acquiring it.
+// It panics if the baton is already marked as held.
+// It is necessary to separate this from [baton.Acquire] because it is practically
+// impossible to reliably and atomically pass the baton and record the new holder
+// at the same time without deadlocks.
+func (b *baton) Acquired(by string) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-
-	// If there's an existing channel for this actor, reuse it.
-	ch := b.acquire[by]
-	if ch == nil {
-		ch = make(chan struct{})
-	}
-
-	// If nothing holds the baton currently, assign it to the caller.
-	// The caller won't be receiving on the channel until after we
-	// return it, so make the channel receivable by closing it.
-	if b.holder == "" {
-		b.holder = by
-		close(ch)
-		delete(b.acquire, by) // in case it is in the map
-		return ch
+	if b.holder != "" {
+		// panic is not ideal for a few reasons (a fatal log might be better),
+		// but keeps baton focused on being a concurrency primitive. As long as
+		// the panic reaches the Go runtime, Go will give us a traceback and exit.
+		panic("baton already held by " + b.holder)
 	}
-
-	// Something holds the baton, so record that this actor is
-	// waiting for the baton.
-	b.acquire[by] = ch
-	return ch
+	b.holder = by
 }
 
 // Release releases the baton, if it is held by the argument.
 func (b *baton) Release(by string) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-
-	// Only release if its the same actor, to prevent bugs due
-	// to double-releasing.
 	if b.holder != by {
 		return
 	}
-
-	// Attempt to pass the baton to anything still waiting for it.
-	for a, ch := range b.acquire {
-		delete(b.acquire, a)
-		select {
-		case ch <- struct{}{}:
-			// We were able to send a value to the channel,
-			// so this actor was still waiting to receive.
-			// Therefore this actor has acquired the baton.
-			b.holder = a
-			return
-		default:
-			// This actor has stopped waiting to receive,
-			// so try another.
-		}
-	}
-
-	// Nothing was still waiting on its channel,
-	// so now nothing holds the baton.
 	b.holder = ""
+	b.ch <- struct{}{}
 }
diff --git a/agent/baton_test.go b/agent/baton_test.go
new file mode 100644
index 0000000000..415edd8445
--- /dev/null
+++ b/agent/baton_test.go
@@ -0,0 +1,43 @@
+package agent
+
+import (
+	"math/rand/v2"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestBaton_NoDroppedBatonDeadlock(t *testing.T) {
+	t.Parallel()
+
+	bat := newBaton()
+
+	actor := func(n string) func() {
+		return func() {
+			time.Sleep(rand.N(1 * time.Microsecond))
+			<-bat.Acquire()
+			bat.Acquired(n)
+			time.Sleep(rand.N(1 * time.Microsecond))
+			bat.Release(n)
+		}
+	}
+
+	done := make(chan struct{})
+
+	go func() {
+		for range 10000 {
+			var wg sync.WaitGroup
+			wg.Go(actor("a"))
+			wg.Go(actor("b"))
+			wg.Wait()
+		}
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		// It probably doesn't deadlock that way
+	case <-time.After(10 * time.Second):
+		t.Error("Repeated baton.Acquire/Release failed to progress, possible deadlock")
+	}
+}

From 5abe2287506a19d48d8520f14b25be103c23125a Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 11 Mar 2026 17:44:47 +1100
Subject: [PATCH 238/242] chore: use WaitGroup.Go where possible

---
 agent/agent_pool.go                            |  6 +-----
 agent/agent_worker.go                          | 13 +++----------
 .../integration/job_runner_integration_test.go | 13 +++++--------
 agent/job_runner.go                            |  9 ++-------
 agent/log_streamer.go                          |  4 +---
 agent/run_job.go                               |  8 +++-----
 clicommand/agent_start.go                      |  6 ++----
 internal/artifact/downloader.go                |  7 ++-----
 internal/artifact/uploader.go                  | 18 +++++-------------
 internal/cache/cache.go                        | 14 ++++----------
 .../job/integration/hooks_integration_test.go  |  7 ++-----
 lock/lock_test.go                              | 12 ++++--------
 process/process.go                             |  8 ++------
 process/process_test.go                        |  8 ++------
 14 files changed, 38 insertions(+), 95 deletions(-)

diff --git a/agent/agent_pool.go b/agent/agent_pool.go
index cb0f75393e..56ac3c0eb5 100644
--- a/agent/agent_pool.go
+++ b/agent/agent_pool.go
@@ -110,16 +110,12 @@ func (r *AgentPool) StopGracefully() {
 // cancellation to finish.
 func (r *AgentPool) StopUngracefully() {
 	var wg sync.WaitGroup
-	wg.Add(len(r.workers))
 	for _, worker := range r.workers {
 		// Because StopUngracefully calls the job runner's Cancel, which blocks,
 		// concurrently stop all the workers.
 		// The number of concurrent Stops is bounded by the spawn count, and
 		// there already exists a handful of goroutines per worker.
-		go func() {
-			worker.StopUngracefully()
-			wg.Done()
-		}()
+		wg.Go(worker.StopUngracefully)
 	}
 	wg.Wait()
 }
diff --git a/agent/agent_worker.go b/agent/agent_worker.go
index e670bc350b..388f007ce6 100644
--- a/agent/agent_worker.go
+++ b/agent/agent_worker.go
@@ -298,17 +298,12 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 	fromStreamingLoopCh := make(chan actionMessage) // streaming loop to debouncer
 	fromDebouncerCh := make(chan actionMessage)     // debouncer to action handler
 
-	// Start the loops and block until they have all stopped.
 	// Based on configuration, we have our choice of ping loop,
 	// streaming loop+debouncer loop, or both.
-	var wg sync.WaitGroup
-
 	pingLoop := func() {
-		defer wg.Done()
 		errCh <- a.runPingLoop(ctx, bat, fromPingLoopCh)
 	}
 	streamingLoop := func() {
-		defer wg.Done()
 		err := a.runStreamingPingLoop(ctx, fromStreamingLoopCh)
 		if err != nil {
 			switch a.agentConfiguration.PingMode {
@@ -331,7 +326,6 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 		errCh <- err
 	}
 	debouncerLoop := func() {
-		defer wg.Done()
 		errCh <- a.runDebouncer(ctx, bat, fromDebouncerCh, fromStreamingLoopCh)
 	}
 
@@ -358,15 +352,14 @@ func (a *AgentWorker) Start(ctx context.Context, idleMon *idleMonitor) (startErr
 
 	// There's always an action handler.
 	actionLoop := func() {
-		defer wg.Done()
 		errCh <- a.runActionLoop(ctx, idleMon, fromPingLoopCh, fromDebouncerCh)
 	}
 	loops = append(loops, actionLoop)
 
-	// Go loops!
-	wg.Add(len(loops))
+	// Start the loops and block until they have all stopped.
+	var wg sync.WaitGroup
 	for _, l := range loops {
-		go l()
+		wg.Go(l)
 	}
 	wg.Wait()
 
diff --git a/agent/integration/job_runner_integration_test.go b/agent/integration/job_runner_integration_test.go
index de64712efd..ba0ae1c648 100644
--- a/agent/integration/job_runner_integration_test.go
+++ b/agent/integration/job_runner_integration_test.go
@@ -421,19 +421,16 @@ func TestChunksIntervalSeconds_ControlsUploadTiming(t *testing.T) {
 	t.Run("2s interval should upload fewer chunks than 1s interval", func(t *testing.T) {
 		var count1s, count2s int
 
-		wg := &sync.WaitGroup{}
-		wg.Add(2)
+		var wg sync.WaitGroup
 
 		// these run for 4 seconds, so we run them in parallel to not quite so much wall-clock time
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			count1s = runTestWithInterval(t, 1)
-		}()
+		})
 
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			count2s = runTestWithInterval(t, 2)
-		}()
+		})
 
 		wg.Wait()
 
diff --git a/agent/job_runner.go b/agent/job_runner.go
index b3282987b1..3ae81fc968 100644
--- a/agent/job_runner.go
+++ b/agent/job_runner.go
@@ -767,17 +767,12 @@ func (r *JobRunner) executePreBootstrapHook(ctx context.Context, hook string) (b
 // jobCancellationChecker waits for the processes to start, then continuously
 // polls GetJobState to see if the job has been cancelled server-side. If so,
 // it calls r.Cancel.
-func (r *JobRunner) jobCancellationChecker(ctx context.Context, wg *sync.WaitGroup) {
+func (r *JobRunner) jobCancellationChecker(ctx context.Context) {
 	ctx, setStat, done := status.AddSimpleItem(ctx, "Job Cancellation Checker")
 	defer done()
 	setStat("Starting...")
 
-	defer func() {
-		// Mark this routine as done in the wait group
-		wg.Done()
-
-		r.agentLogger.Debug("[JobRunner] Routine that refreshes the job has finished")
-	}()
+	defer r.agentLogger.Debug("[JobRunner] Routine that refreshes the job has finished")
 
 	select {
 	case <-r.process.Started():
diff --git a/agent/log_streamer.go b/agent/log_streamer.go
index 8db1d239c2..07c6fd8806 100644
--- a/agent/log_streamer.go
+++ b/agent/log_streamer.go
@@ -92,9 +92,8 @@ func (ls *LogStreamer) Start(ctx context.Context) error {
 		ls.conf.MaxSizeBytes = defaultLogMaxSize
 	}
 
-	ls.workerWG.Add(ls.conf.Concurrency)
 	for i := range ls.conf.Concurrency {
-		go ls.worker(ctx, i)
+		ls.workerWG.Go(func() { ls.worker(ctx, i) })
 	}
 
 	return nil
@@ -180,7 +179,6 @@ func (ls *LogStreamer) worker(ctx context.Context, id int) {
 	ls.logger.Debug("[LogStreamer/Worker#%d] Worker is starting...", id)
 
 	defer ls.logger.Debug("[LogStreamer/Worker#%d] Worker has shutdown", id)
-	defer ls.workerWG.Done()
 
 	ctx, setStat, done := status.AddSimpleItem(ctx, fmt.Sprintf("Log Streamer Worker %d", id))
 	defer done()
diff --git a/agent/run_job.go b/agent/run_job.go
index 28eff695c5..f314888d98 100644
--- a/agent/run_job.go
+++ b/agent/run_job.go
@@ -208,9 +208,8 @@ func (r *JobRunner) Run(ctx context.Context, ignoreAgentInDispatches *bool) (err
 	}
 
 	// Kick off log streaming and job status checking when the process starts.
-	wg.Add(2)
-	go r.streamJobLogsAfterProcessStart(cctx, &wg)
-	go r.jobCancellationChecker(cctx, &wg)
+	wg.Go(func() { r.streamJobLogsAfterProcessStart(cctx) })
+	wg.Go(func() { r.jobCancellationChecker(cctx) })
 
 	exit = r.runJob(cctx)
 	// The defer mutates the error return in some cases.
@@ -443,13 +442,12 @@ func (r *JobRunner) cleanup(ctx context.Context, wg *sync.WaitGroup, exit core.P
 
 // streamJobLogsAfterProcessStart waits for the process to start, then grabs the job output
 // every few seconds and sends it back to Buildkite.
-func (r *JobRunner) streamJobLogsAfterProcessStart(ctx context.Context, wg *sync.WaitGroup) {
+func (r *JobRunner) streamJobLogsAfterProcessStart(ctx context.Context) {
 	ctx, setStat, done := status.AddSimpleItem(ctx, "Job Log Streamer")
 	defer done()
 	setStat("🏃 Starting...")
 
 	defer func() {
-		wg.Done()
 		r.agentLogger.Debug("[JobRunner] Routine that processes the log has finished")
 	}()
 
diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 6a655f9187..310fd90536 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -1565,15 +1565,13 @@ func agentLifecycleHook(hookName string, log logger.Logger, cfg AgentStartConfig
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
+	wg.Go(func() {
 		scan := bufio.NewScanner(r) // log each line separately
 		log = log.WithFields(logger.StringField("hook", hookName))
 		for scan.Scan() {
 			log.Info(scan.Text())
 		}
-	}()
+	})
 	defer func() {
 		_ = w.Close() // closing the writer ends scan.Scan and lets wg.Wait return
 		wg.Wait()
diff --git a/internal/artifact/downloader.go b/internal/artifact/downloader.go
index 53193690f5..9278a7cf0f 100644
--- a/internal/artifact/downloader.go
+++ b/internal/artifact/downloader.go
@@ -109,10 +109,7 @@ func (a *Downloader) Download(ctx context.Context) error {
 	var wg sync.WaitGroup
 	artifactsCh := make(chan *api.Artifact)
 	for range min(10*runtime.GOMAXPROCS(0), len(artifacts)) {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
+		wg.Go(func() {
 			for {
 				var artifact *api.Artifact
 				var open bool
@@ -146,7 +143,7 @@ func (a *Downloader) Download(ctx context.Context) error {
 					}
 				}
 			}
-		}()
+		})
 	}
 
 	// Send the artifacts to the workers then signal completion by closing the
diff --git a/internal/artifact/uploader.go b/internal/artifact/uploader.go
index 484bdcf9f1..0a8266cf86 100644
--- a/internal/artifact/uploader.go
+++ b/internal/artifact/uploader.go
@@ -165,14 +165,11 @@ func (a *Uploader) collect(ctx context.Context) ([]*api.Artifact, error) {
 	defer cancel(nil)
 	var wg sync.WaitGroup
 	for range runtime.GOMAXPROCS(0) {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
+		wg.Go(func() {
 			if err := ac.worker(wctx, filesCh); err != nil {
 				cancel(err)
 			}
-		}()
+		})
 	}
 
 	fileFinder := a.glob
@@ -476,9 +473,6 @@ type workUnitResult struct {
 type artifactUploadWorker struct {
 	*Uploader
 
-	// Counts the worker goroutines.
-	wg sync.WaitGroup
-
 	// A tracker for every artifact.
 	// The map is written at the start of upload, and other goroutines only read
 	// afterwards.
@@ -562,9 +556,9 @@ func (a *Uploader) upload(ctx context.Context, artifacts []*api.Artifact, upload
 	go worker.stateUpdater(ctx, resultsCh, errCh)
 
 	// Worker goroutines that work on work units.
+	var wg sync.WaitGroup
 	for range runtime.GOMAXPROCS(0) {
-		worker.wg.Add(1)
-		go worker.doWorkUnits(ctx, unitsCh, resultsCh)
+		wg.Go(func() { worker.doWorkUnits(ctx, unitsCh, resultsCh) })
 	}
 
 	// Send the work units for each artifact to the workers.
@@ -585,7 +579,7 @@ func (a *Uploader) upload(ctx context.Context, artifacts []*api.Artifact, upload
 	a.logger.Debug("Waiting for uploads to complete...")
 
 	// Wait for the workers to finish
-	worker.wg.Wait()
+	wg.Wait()
 
 	// Since the workers are done, all work unit states have been sent to the
 	// state updater.
@@ -604,8 +598,6 @@ func (a *Uploader) upload(ctx context.Context, artifacts []*api.Artifact, upload
 }
 
 func (a *artifactUploadWorker) doWorkUnits(ctx context.Context, unitsCh <-chan workUnit, resultsCh chan<- workUnitResult) {
-	defer a.wg.Done()
-
 	for {
 		select {
 		case <-ctx.Done():
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index 8a26700ed5..a78c459fbd 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -174,10 +174,7 @@ func restoreWithClient(ctx context.Context, l logger.Logger, client CacheClient,
 	var wg sync.WaitGroup
 
 	for range workerCount {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
+		wg.Go(func() {
 			for {
 				select {
 				case cacheID, open := <-cacheIDsCh:
@@ -217,7 +214,7 @@ func restoreWithClient(ctx context.Context, l logger.Logger, client CacheClient,
 					return
 				}
 			}
-		}()
+		})
 	}
 
 sendLoop:
@@ -253,10 +250,7 @@ func saveWithClient(ctx context.Context, l logger.Logger, client CacheClient, ca
 	var wg sync.WaitGroup
 
 	for range workerCount {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
+		wg.Go(func() {
 			for {
 				select {
 				case cacheID, open := <-cacheIDsCh:
@@ -295,7 +289,7 @@ func saveWithClient(ctx context.Context, l logger.Logger, client CacheClient, ca
 					return
 				}
 			}
-		}()
+		})
 	}
 
 sendLoop:
diff --git a/internal/job/integration/hooks_integration_test.go b/internal/job/integration/hooks_integration_test.go
index 0f75eaebde..6e8a3d6bc4 100644
--- a/internal/job/integration/hooks_integration_test.go
+++ b/internal/job/integration/hooks_integration_test.go
@@ -542,15 +542,12 @@ func TestPreExitHooksFireAfterCancel(t *testing.T) {
 	tester.ExpectLocalHook("pre-exit").Once()
 
 	var wg sync.WaitGroup
-	wg.Add(1)
-
-	go func() {
-		defer wg.Done()
+	wg.Go(func() {
 		if err := tester.Run(t, "BUILDKITE_COMMAND=sleep 5"); err == nil {
 			t.Errorf(`tester.Run(t, "BUILDKITE_COMMAND=sleep 5") = %v, want non-nil error`, err)
 		}
 		t.Logf("Command finished")
-	}()
+	})
 
 	time.Sleep(time.Millisecond * 500)
 	tester.Cancel()
diff --git a/lock/lock_test.go b/lock/lock_test.go
index e7cea322ec..aa8b28e9ee 100644
--- a/lock/lock_test.go
+++ b/lock/lock_test.go
@@ -99,13 +99,11 @@ func TestLocker(t *testing.T) {
 	var wg sync.WaitGroup
 	var locks int
 	for range 10 {
-		wg.Add(1)
-		go func() {
+		wg.Go(func() {
 			l.Lock()
 			locks++
 			l.Unlock()
-			wg.Done()
-		}()
+		})
 	}
 
 	wg.Wait()
@@ -126,15 +124,13 @@ func TestDoOnce(t *testing.T) {
 	var wg sync.WaitGroup
 	var calls atomic.Int32
 	for range 10 {
-		wg.Add(1)
-		go func() {
+		wg.Go(func() {
 			if err := cli.DoOnce(ctx, "once", func() {
 				calls.Add(1)
 			}); err != nil {
 				t.Errorf("Client.DoOnce(ctx, once, inc) = %v", err)
 			}
-			wg.Done()
-		}()
+		})
 	}
 
 	wg.Wait()
diff --git a/process/process.go b/process/process.go
index 38a4248ff8..f301e39ad6 100644
--- a/process/process.go
+++ b/process/process.go
@@ -198,9 +198,7 @@ func (p *Process) Run(ctx context.Context) error {
 		// Signal waiting consumers in Started() by closing the started channel
 		close(p.started)
 
-		waitGroup.Add(1)
-
-		go func() {
+		waitGroup.Go(func() {
 			p.logger.Debug("[Process] Starting to copy PTY to the buffer")
 
 			// Copy the pty to our writer. This will block until it EOFs or something breaks.
@@ -222,9 +220,7 @@ func (p *Process) Run(ctx context.Context) error {
 			default:
 				p.logger.Error("[Process] PTY output copy failed with error: %T: %v", err, err)
 			}
-
-			waitGroup.Done()
-		}()
+		})
 	} else {
 		p.logger.Debug("[Process] Running without a PTY")
 
diff --git a/process/process_test.go b/process/process_test.go
index ad48510700..e911376241 100644
--- a/process/process_test.go
+++ b/process/process_test.go
@@ -150,16 +150,12 @@ func TestProcessRunsAndSignalsStartedAndStopped(t *testing.T) {
 	})
 
 	var wg sync.WaitGroup
-	wg.Add(1)
-
-	go func() {
-		defer wg.Done()
-
+	wg.Go(func() {
 		<-p.Started()
 		atomic.AddInt32(&started, 1)
 		<-p.Done()
 		atomic.AddInt32(&done, 1)
-	}()
+	})
 
 	// wait for the process to finish
 	if err := p.Run(context.Background()); err != nil {

From 40d733ad9bd4a61c7f0c3fa34b2657e1853acb4a Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 12 Mar 2026 10:08:02 +1100
Subject: [PATCH 239/242] chore: Apply other go fixes

---
 internal/job/config.go               |  2 +-
 internal/osutil/path_windows_test.go |  1 -
 internal/shell/lookpath.go           |  1 -
 kubernetes/umask.go                  |  1 -
 kubernetes/umask_windows.go          |  1 -
 logger/init_windows.go               |  1 -
 logger/log.go                        | 12 ++++++------
 process/pty.go                       |  1 -
 process/signal.go                    |  1 -
 process/signal_windows.go            |  1 -
 10 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/internal/job/config.go b/internal/job/config.go
index 55cee9889a..319b8d3698 100644
--- a/internal/job/config.go
+++ b/internal/job/config.go
@@ -203,7 +203,7 @@ func (c *ExecutorConfig) ReadFromEnvironment(environ *env.Environment) map[strin
 	changed := map[string]string{}
 
 	// Use reflection for the type and values
-	fields := reflect.TypeOf(*c)
+	fields := reflect.TypeFor[ExecutorConfig]()
 	values := reflect.ValueOf(c).Elem()
 
 	// Iterate over all available fields and read the tag value
diff --git a/internal/osutil/path_windows_test.go b/internal/osutil/path_windows_test.go
index 3cdf315701..f62ee6273b 100644
--- a/internal/osutil/path_windows_test.go
+++ b/internal/osutil/path_windows_test.go
@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package osutil
 
diff --git a/internal/shell/lookpath.go b/internal/shell/lookpath.go
index 93b3a3ce2a..a6634df010 100644
--- a/internal/shell/lookpath.go
+++ b/internal/shell/lookpath.go
@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 // This file (along with its Windows counterpart) have been taken from:
 //
diff --git a/kubernetes/umask.go b/kubernetes/umask.go
index 3cae818d19..792f7c826f 100644
--- a/kubernetes/umask.go
+++ b/kubernetes/umask.go
@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package kubernetes
 
diff --git a/kubernetes/umask_windows.go b/kubernetes/umask_windows.go
index 188a07e1f4..57de2023d7 100644
--- a/kubernetes/umask_windows.go
+++ b/kubernetes/umask_windows.go
@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package kubernetes
 
diff --git a/logger/init_windows.go b/logger/init_windows.go
index 168f2c2c4f..5796321b0e 100644
--- a/logger/init_windows.go
+++ b/logger/init_windows.go
@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package logger
 
diff --git a/logger/log.go b/logger/log.go
index 98b5dcc3d7..d27e4194be 100644
--- a/logger/log.go
+++ b/logger/log.go
@@ -144,7 +144,7 @@ func (l *TextPrinter) Print(level Level, msg string, fields Fields) {
 	now := time.Now().Format(DateFormat)
 
 	var line string
-	var prefix string
+	var prefix strings.Builder
 	var fieldStrs []string
 
 	if l.IsPrefixFn != nil {
@@ -155,7 +155,7 @@ func (l *TextPrinter) Print(level Level, msg string, fields Fields) {
 			}
 			// Allow some fields to be shown as prefixes
 			if l.IsPrefixFn(f) {
-				prefix += f.String()
+				prefix.WriteString(f.String())
 			}
 		}
 	}
@@ -180,9 +180,9 @@ func (l *TextPrinter) Print(level Level, msg string, fields Fields) {
 			messageColor = red
 		}
 
-		if prefix != "" {
+		if prefix.String() != "" {
 			line = fmt.Sprintf("\x1b[%sm%s %-6s\x1b[0m \x1b[%sm%s\x1b[0m \x1b[%sm%s\x1b[0m",
-				levelColor, now, level, lightgray, prefix, messageColor, msg)
+				levelColor, now, level, lightgray, prefix.String(), messageColor, msg)
 		} else {
 			line = fmt.Sprintf("\x1b[%sm%s %-6s\x1b[0m \x1b[%sm%s\x1b[0m",
 				levelColor, now, level, messageColor, msg)
@@ -199,8 +199,8 @@ func (l *TextPrinter) Print(level Level, msg string, fields Fields) {
 				fieldColor, field.Key(), messageColor, field.String()))
 		}
 	} else {
-		if prefix != "" {
-			line = fmt.Sprintf("%s %-6s %s %s", now, level, prefix, msg)
+		if prefix.String() != "" {
+			line = fmt.Sprintf("%s %-6s %s %s", now, level, prefix.String(), msg)
 		} else {
 			line = fmt.Sprintf("%s %-6s %s", now, level, msg)
 		}
diff --git a/process/pty.go b/process/pty.go
index 960f8a3094..d5f6f98b6d 100644
--- a/process/pty.go
+++ b/process/pty.go
@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package process
 
diff --git a/process/signal.go b/process/signal.go
index 975f009155..6c09b658c0 100644
--- a/process/signal.go
+++ b/process/signal.go
@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package process
 
diff --git a/process/signal_windows.go b/process/signal_windows.go
index c1e7bdc20d..1a14cff559 100644
--- a/process/signal_windows.go
+++ b/process/signal_windows.go
@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package process
 

From 641edee628dcaa00e7d9b791117b46112d11a452 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 12 Mar 2026 13:33:49 +1100
Subject: [PATCH 240/242] Use t.Context within test

---
 agent/header_times_streamer_test.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/agent/header_times_streamer_test.go b/agent/header_times_streamer_test.go
index 7dbcf09f64..b3f9f05018 100644
--- a/agent/header_times_streamer_test.go
+++ b/agent/header_times_streamer_test.go
@@ -13,12 +13,9 @@ func TestHeaderTimesStreamerScanAfterStopDoesNotPanic(t *testing.T) {
 
 	h := newHeaderTimesStreamer(logger.Discard, func(context.Context, int, int, map[string]string) {})
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	runDone := make(chan struct{})
 	go func() {
-		h.Run(ctx)
+		h.Run(t.Context())
 		close(runDone)
 	}()
 
@@ -49,7 +46,6 @@ func TestHeaderTimesStreamerScanAfterStopDoesNotPanic(t *testing.T) {
 	select {
 	case <-stopDone:
 	case <-time.After(500 * time.Millisecond):
-		cancel()
 		t.Fatal("timed out waiting for header times streamer to stop")
 	}
 
@@ -66,7 +62,6 @@ func TestHeaderTimesStreamerScanAfterStopDoesNotPanic(t *testing.T) {
 	select {
 	case <-runDone:
 	case <-time.After(500 * time.Millisecond):
-		cancel()
 		t.Fatal("timed out waiting for header times streamer run loop to exit")
 	}
 }

From 5c3f300fe472f96ecd2382f8932c94b6f31453d3 Mon Sep 17 00:00:00 2001
From: Ben Moskovitz <ben.m@buildkite.com>
Date: Thu, 12 Mar 2026 14:17:40 +1100
Subject: [PATCH 241/242] Add feature detection for streaming pings

---
 clicommand/agent_start.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go
index 8e887e98ef..9018de8fd8 100644
--- a/clicommand/agent_start.go
+++ b/clicommand/agent_start.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"maps"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -220,12 +221,18 @@ func (asc AgentStartConfig) Features(ctx context.Context) []string {
 		return []string{}
 	}
 
-	features := make([]string, 0, 8)
+	features := make([]string, 0, 9)
 
 	if asc.GitMirrorsPath != "" {
 		features = append(features, "git-mirrors")
 	}
 
+	if endpointURL, err := url.Parse(asc.Endpoint); err == nil {
+		if endpointURL.Host == "agent-edge.buildkite.com" && asc.PingMode != agent.PingModePollOnly {
+			features = append(features, "streaming-pings")
+		}
+	}
+
 	if asc.AcquireJob != "" {
 		features = append(features, "acquire-job")
 	}

From d6c149fe3bbfde6e625d85beb7bdb0df8b0ddd63 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 12 Mar 2026 15:17:08 +1100
Subject: [PATCH 242/242] Bump version and changelog for v3.120.0

---
 CHANGELOG.md    | 33 +++++++++++++++++++++++++++++++++
 version/VERSION |  2 +-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cbbf5349f3..e933973884 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v3.120.0](https://github.com/buildkite/agent/tree/v3.120.0) (2026-03-13)
+[Full Changelog](https://github.com/buildkite/agent/compare/v3.119.2...v3.120.0)
+
+> [!TIP] 
+> **Streaming job dispatch (Public Preview):** This release adds opt-in support for a new streaming connection between agents and Buildkite, significantly reducing job acceptance latency for self-hosted agents. To try it, start your agent with `--endpoint https://agent-edge.buildkite.com/v3`, for example:
+>
+>     buildkite-agent start --endpoint https://agent-edge.buildkite.com/v3
+>
+> You may alternatively use the environment variable `BUILDKITE_AGENT_ENDPOINT` or edit your `buildkite-agent.cfg` to contain `endpoint=https://agent-edge.buildkite.com/v3`.
+>
+> This capability is in public preview and will become the default in a future release. If you have any feedback or run into issues, please reach out to support@buildkite.com.
+
+> [!NOTE]
+> The minimum version of Go used to build the agent is now Go 1.25.
+
+### Fixed
+- fix: Make submodule clone config an agent config [#3752](https://github.com/buildkite/agent/pull/3752) (@DrJosh9000)
+- fix: prevent header times scan panic after stop [#3740](https://github.com/buildkite/agent/pull/3740) (@lox)
+- fix: handle multiple lifecycle hooks without closed pipe reuse [#3741](https://github.com/buildkite/agent/pull/3741) (@lox)
+- fix: potential deadlock in baton [#3754](https://github.com/buildkite/agent/pull/3754) (@DrJosh9000)
+- fix: Use targetPath helper and tempfile for Azure Blob download [#3751](https://github.com/buildkite/agent/pull/3751) (@DrJosh9000)
+
+### Internal
+
+- Add feature detection for streaming pings [#3757](https://github.com/buildkite/agent/pull/3757) (@moskyb)
+- chore: Apply other go fixes [#3756](https://github.com/buildkite/agent/pull/3756) (@DrJosh9000)
+- chore: use WaitGroup.Go where possible [#3755](https://github.com/buildkite/agent/pull/3755) (@DrJosh9000)
+
+### Dependency updates
+
+- build(deps): bump the container-images group across 5 directories with 1 update [#3749](https://github.com/buildkite/agent/pull/3749) (@dependabot[bot])
+- Upgrade to Go 1.25 and update all dependencies [#3750](https://github.com/buildkite/agent/pull/3750) (@DrJosh9000)
+
 ## [v3.119.2](https://github.com/buildkite/agent/tree/v3.119.2) (2026-03-09)
 
 [Full Changelog](https://github.com/buildkite/agent/compare/v3.119.1...v3.119.2)
diff --git a/version/VERSION b/version/VERSION
index 03938b9efb..ae03d830f8 100644
--- a/version/VERSION
+++ b/version/VERSION
@@ -1 +1 @@
-3.119.2
+3.120.0