diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env index 843959969..a064b2b37 100644 --- a/benchmark/agent-versions.env +++ b/benchmark/agent-versions.env @@ -5,3 +5,11 @@ CLAUDE_CODE_VERSION=2.1.211 CODEX_VERSION=0.144.5 OPENCODE_VERSION=1.18.3 NODE_VERSION=20.11.1 +# Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time. +# Must be a full 40-character commit SHA; anything else is rejected at build time. +# A tag would be enough to read but not to trust — it can be deleted or repointed, +# so two builds could record the same string while installing different code, and +# the run manifest records this value as a pin. The installer script is fetched +# from this same commit, so the installer cannot drift either. +# 3c27eb62 is the commit tagged v2026.8.3 (2026-08-03). +HERMES_VERSION=3c27eb6234bf91b8ceee9e9071591b31e9b148cb diff --git a/benchmark/patches/harbor-agent-patches.diff b/benchmark/patches/harbor-agent-patches.diff index bc62838d1..54454cd68 100644 --- a/benchmark/patches/harbor-agent-patches.diff +++ b/benchmark/patches/harbor-agent-patches.diff @@ -426,3 +426,105 @@ +++ b/harbor/switchyard_patch_id.txt @@ -0,0 +1 @@ +switchyard-harbor-patches-2026-05-22-v2 + +--- a/harbor/agents/installed/hermes.py ++++ b/harbor/agents/installed/hermes.py +@@ -59,6 +59,16 @@ + return 'export PATH="$HOME/.local/bin:$PATH"; hermes version' + + async def install(self, environment: BaseEnvironment) -> None: ++ # Skip if hermes is already installed (e.g. from the dataset ++ # agent-bake layer). hermes installs to $HOME/.local/bin, so export ++ # that path before probing. Required for closed-book runs where the ++ # task-time curl install cannot reach github/pypi. ++ probe = await self.exec_as_agent( ++ environment, ++ command='export PATH="$HOME/.local/bin:$PATH"; command -v hermes >/dev/null 2>&1 && echo present || true', ++ ) ++ if "present" in (getattr(probe, "stdout", "") or ""): ++ return + await self.exec_as_root( + environment, + command="apt-get update && apt-get install -y curl git ripgrep xz-utils", +@@ -92,11 +92,31 @@ + # ------------------------------------------------------------------ + + @staticmethod +- def _build_config_yaml(model: str) -> str: +- """Generate a hermes config.yaml with full capabilities enabled.""" ++ def _build_config_yaml( ++ model: str, base_url: str | None = None, api_key: str | None = None ++ ) -> str: ++ """Generate a hermes config.yaml with full capabilities enabled. ++ ++ When ``base_url`` is given, route the model through a custom ++ OpenAI-compatible endpoint (e.g. a Switchyard gateway): hermes only ++ honors a custom endpoint via its config.yaml ``model.provider=custom`` + ++ ``model.base_url`` (the ``OPENAI_BASE_URL`` env var is ignored for the ++ chat model), so we nest the provider/base_url/api_key under ``model``. ++ """ ++ model_field: Any ++ if base_url: ++ model_field = { ++ "default": model, ++ "provider": "custom", ++ "base_url": base_url, ++ } ++ if api_key: ++ model_field["api_key"] = api_key ++ else: ++ model_field = model + config: dict[str, Any] = { +- "model": model, +- "provider": "auto", ++ "model": model_field, ++ **({} if base_url else {"provider": "auto"}), + "toolsets": ["hermes-cli"], + "agent": {"max_turns": 90}, + "memory": { +@@ -357,6 +377,9 @@ + # Try native provider key first, fall back to OpenRouter. + hermes_provider_flag: str | None = None + use_native = False ++ # Custom OpenAI-compatible endpoint (e.g. Switchyard) for the chat model. ++ custom_base_url: str | None = None ++ custom_api_key: str | None = None + + if provider in _NATIVE_PROVIDERS: + native_flag, key_names = _NATIVE_PROVIDERS[provider] +@@ -367,11 +390,15 @@ + hermes_provider_flag = native_flag + use_native = True + break +- # Forward OPENAI_BASE_URL when using native OpenAI key ++ # Forward OPENAI_BASE_URL when using native OpenAI key. Hermes ignores ++ # this env var for the chat model, so also thread it into config.yaml ++ # as a custom provider (see _build_config_yaml). + if use_native and provider == "openai": + base_url = os.environ.get("OPENAI_BASE_URL") + if base_url: + env["OPENAI_BASE_URL"] = base_url ++ custom_base_url = base_url ++ custom_api_key = os.environ.get("OPENAI_API_KEY") + + if not use_native: + openrouter_key = os.environ.get("OPENROUTER_API_KEY") +@@ -386,9 +413,15 @@ + env["OPENROUTER_API_KEY"] = openrouter_key + + # Native providers with --provider flag use just the model name; +- # everything else (OpenRouter, openai direct) uses provider/model. +- cli_model = model if hermes_provider_flag else self.model_name +- config_yaml = self._build_config_yaml(cli_model) ++ # a custom OpenAI-compatible endpoint also uses the bare model (the ++ # base_url already targets the gateway); everything else uses provider/model. ++ if custom_base_url: ++ cli_model = model ++ else: ++ cli_model = model if hermes_provider_flag else self.model_name ++ config_yaml = self._build_config_yaml( ++ cli_model, base_url=custom_base_url, api_key=custom_api_key ++ ) + + # Pass instruction via env var (safe from shell escaping issues) + env["HARBOR_INSTRUCTION"] = instruction diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py index fb30a8ce3..44c08f146 100644 --- a/benchmark/prepare_harbor_dataset.py +++ b/benchmark/prepare_harbor_dataset.py @@ -29,6 +29,7 @@ AGENT_VERSIONS_FILE = SCRIPT_DIR / "agent-versions.env" PROXY_ASSET_DIR = SCRIPT_DIR / "closed_book_proxy" / "proxy" AGENT_ENTRYPOINT = "switchyard-agent-entrypoint.sh" +COMMIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}") TERMINAL_BENCH_2_SOURCE_DATASET = "terminal-bench/terminal-bench-2" TERMINAL_BENCH_2_1_SOURCE_DATASET = "terminal-bench/terminal-bench-2-1" # Shared across the TB2 family (2.0 + the 2.1 verified iteration): 2.1 tweaks @@ -187,10 +188,35 @@ def _install_layer(pins: dict[str, str]) -> str: claude_version = pins["CLAUDE_CODE_VERSION"] codex_version = pins["CODEX_VERSION"] opencode_version = pins["OPENCODE_VERSION"] + # Hermes (NousResearch hermes-agent) is a per-user uv app installed from + # GitHub, not an npm package. Baking it here (build-time, with host network) + # means the runtime install() skip-guard short-circuits, so tasks need no + # egress for it — enabling closed-book Hermes runs. + # + # HERMES_VERSION must be a full commit SHA. A tag is not enough: tags can be + # deleted or repointed, so two builds could record the same version string while + # installing different code — the manifest would assert a reproducibility it does + # not have. Requiring one shape is complete by construction, where a deny-list of + # moving names ("main", "master", ...) can never be, since any branch name passes. + # The installer script is fetched from the same commit for the same reason: pinning + # the agent but running whatever installer main has today reintroduces the drift. + # + # The pin is applied with the installer's --commit, not --branch: --branch reaches + # `git clone --branch`, which accepts only branch and tag names and rejects a SHA. + # --force-commit is required with it. Without it the installer skips the pin when + # the commit is an ancestor of the freshly cloned HEAD, logging a warning and + # silently leaving the image on the tip of main — the drift this pin exists to + # prevent, arriving as a warning rather than a build failure. + hermes_version = pins["HERMES_VERSION"] + if not COMMIT_SHA_PATTERN.fullmatch(hermes_version): + raise SystemExit( + f"HERMES_VERSION={hermes_version!r} is not a full 40-character commit SHA; " + "a tag or branch can be repointed and cannot be recorded as a reproducible pin" + ) return f""" # Switchyard benchmark prebaked coding agents. -ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version}" +ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}" RUN set -eux; \\ if command -v apt-get >/dev/null 2>&1; then \\ apt-get update; \\ @@ -231,6 +257,19 @@ def _install_layer(pins: dict[str, str]) -> str: claude --version; \\ codex --version; \\ opencode --version +RUN set -eux; \\ + export HOME=/root; \\ + export PATH="/root/.local/bin:$PATH"; \\ + if command -v apt-get >/dev/null 2>&1; then \\ + apt-get update; \\ + apt-get install -y --no-install-recommends git ripgrep xz-utils; \\ + rm -rf /var/lib/apt/lists/*; \\ + elif command -v apk >/dev/null 2>&1; then \\ + apk add --no-cache bash git ripgrep xz; \\ + fi; \\ + curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\ + | bash -s -- --skip-setup --commit {hermes_version} --force-commit; \\ + hermes version """ @@ -463,7 +502,13 @@ def prepare_dataset( overwrite: bool, ) -> Path: pins = _read_env_file(AGENT_VERSIONS_FILE) - required = {"CLAUDE_CODE_VERSION", "CODEX_VERSION", "OPENCODE_VERSION", "NODE_VERSION"} + required = { + "CLAUDE_CODE_VERSION", + "CODEX_VERSION", + "HERMES_VERSION", + "NODE_VERSION", + "OPENCODE_VERSION", + } missing = sorted(required - pins.keys()) if missing: raise ValueError(f"missing pins in {AGENT_VERSIONS_FILE}: {', '.join(missing)}") diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py index 1c69052ff..74dd620bf 100644 --- a/tests/test_prepare_harbor_dataset.py +++ b/tests/test_prepare_harbor_dataset.py @@ -8,6 +8,7 @@ from pathlib import Path from types import ModuleType +import pytest import yaml REPO = Path(__file__).resolve().parents[1] @@ -284,6 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat assert manifest["agent_versions"] == { "CLAUDE_CODE_VERSION": "2.1.211", "CODEX_VERSION": "0.144.5", + "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb", "NODE_VERSION": "20.11.1", "OPENCODE_VERSION": "1.18.3", } @@ -306,3 +308,110 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None: proxy_env = "\n".join(compose["services"]["proxy"]["environment"]) assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env + +def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None: + """Only a full commit SHA can be recorded as a pin. + + The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two + builds recording the same string while installing different Hermes code is worse + than recording nothing, so the build fails rather than asserting a pin it does not + have. Tags are rejected alongside branches: a tag can be deleted or repointed, so + it reads as immutable without being so. + """ + base = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + } + rejected = ( + "main", + "master", + "HEAD", + "v2026.8.3", + "release/2026.8", + "3c27eb6", + "3C27EB6234BF91B8CEEE9E9071591B31E9B148CB", + "3c27eb6234bf91b8ceee9e9071591b31e9b148cbb", + ) + for ref in rejected: + with pytest.raises(SystemExit, match="commit SHA"): + _load_generator_module()._install_layer({**base, "HERMES_VERSION": ref}) + + +def test_the_hermes_installer_is_fetched_at_the_pinned_commit() -> None: + """Pinning the agent but running main's installer reintroduces the same drift.""" + sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb" + pins = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + "HERMES_VERSION": sha, + } + + layer = _load_generator_module()._install_layer(pins) + + assert f"hermes-agent/{sha}/scripts/install.sh" in layer + assert "hermes-agent/main/" not in layer + + +def test_the_hermes_pin_is_applied_by_commit_and_forced() -> None: + """`--branch` reaches `git clone --branch`, which rejects a SHA outright. + + `--force-commit` is what makes the pin take effect. Without it the installer skips + the checkout whenever the commit is an ancestor of the freshly cloned HEAD, warns, + and leaves the image on the tip of main — the drift the pin exists to prevent, + arriving as a warning rather than a build failure. + """ + sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb" + pins = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + "HERMES_VERSION": sha, + } + + layer = _load_generator_module()._install_layer(pins) + + assert f"--commit {sha}" in layer + assert "--force-commit" in layer + assert "--branch" not in layer + + +def test_the_alpine_branch_installs_the_shell_the_installer_needs() -> None: + """The installer is piped into bash, which Alpine does not ship by default.""" + pins = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb", + } + + layer = _load_generator_module()._install_layer(pins) + + assert "apk add --no-cache bash git ripgrep xz" in layer + + +def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> None: + """Absent, it must fail the shared pin check rather than crash reading the layer.""" + module = _load_generator_module() + versions = tmp_path / "agent-versions.env" + versions.write_text( + "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nOPENCODE_VERSION=3\nNODE_VERSION=4\n" + ) + module.AGENT_VERSIONS_FILE = versions + source = tmp_path / "source" + _write_task(source, "task-a", "[environment]\n", "FROM ubuntu:22.04\n") + + with pytest.raises(ValueError, match="missing pins.*HERMES_VERSION"): + module.prepare_dataset( + source_dataset="openthoughts-tblite@2.0", + source_dir=source, + output_dir=tmp_path / "prepared", + harbor_command="harbor", + overwrite=False, + ) +