feat(benchmark): pre-bake hermes agent into dataset images for closed-book runs

benchmark/agent-versions.env

@@ -5,3 +5,11 @@ CLAUDE_CODE_VERSION=2.1.211
 CODEX_VERSION=0.144.5
 OPENCODE_VERSION=1.18.3
 NODE_VERSION=20.11.1
+# Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time.
+# Must be a full 40-character commit SHA; anything else is rejected at build time.
+# A tag would be enough to read but not to trust — it can be deleted or repointed,
+# so two builds could record the same string while installing different code, and
+# the run manifest records this value as a pin. The installer script is fetched
+# from this same commit, so the installer cannot drift either.
+# 3c27eb62 is the commit tagged v2026.8.3 (2026-08-03).
+HERMES_VERSION=3c27eb6234bf91b8ceee9e9071591b31e9b148cb

benchmark/patches/harbor-agent-patches.diff

@@ -426,3 +426,105 @@
 +++ b/harbor/switchyard_patch_id.txt
 @@ -0,0 +1 @@
 +switchyard-harbor-patches-2026-05-22-v2
+
+--- a/harbor/agents/installed/hermes.py
++++ b/harbor/agents/installed/hermes.py
+@@ -59,6 +59,16 @@
+         return 'export PATH="$HOME/.local/bin:$PATH"; hermes version'
+
+     async def install(self, environment: BaseEnvironment) -> None:
++        # Skip if hermes is already installed (e.g. from the dataset
++        # agent-bake layer). hermes installs to $HOME/.local/bin, so export
++        # that path before probing. Required for closed-book runs where the
++        # task-time curl install cannot reach github/pypi.
++        probe = await self.exec_as_agent(
++            environment,
++            command='export PATH="$HOME/.local/bin:$PATH"; command -v hermes >/dev/null 2>&1 && echo present || true',
++        )
++        if "present" in (getattr(probe, "stdout", "") or ""):
++            return
+         await self.exec_as_root(
+             environment,
+             command="apt-get update && apt-get install -y curl git ripgrep xz-utils",
+@@ -92,11 +92,31 @@
+     # ------------------------------------------------------------------
+
+     @staticmethod
+-    def _build_config_yaml(model: str) -> str:
+-        """Generate a hermes config.yaml with full capabilities enabled."""
++    def _build_config_yaml(
++        model: str, base_url: str | None = None, api_key: str | None = None
++    ) -> str:
++        """Generate a hermes config.yaml with full capabilities enabled.
++
++        When ``base_url`` is given, route the model through a custom
++        OpenAI-compatible endpoint (e.g. a Switchyard gateway): hermes only
++        honors a custom endpoint via its config.yaml ``model.provider=custom`` +
++        ``model.base_url`` (the ``OPENAI_BASE_URL`` env var is ignored for the
++        chat model), so we nest the provider/base_url/api_key under ``model``.
++        """
++        model_field: Any
++        if base_url:
++            model_field = {
++                "default": model,
++                "provider": "custom",
++                "base_url": base_url,
++            }
++            if api_key:
++                model_field["api_key"] = api_key
++        else:
++            model_field = model
+         config: dict[str, Any] = {
+-            "model": model,
+-            "provider": "auto",
++            "model": model_field,
++            **({} if base_url else {"provider": "auto"}),
+             "toolsets": ["hermes-cli"],
+             "agent": {"max_turns": 90},
+             "memory": {
+@@ -357,6 +377,9 @@
+         # Try native provider key first, fall back to OpenRouter.
+         hermes_provider_flag: str | None = None
+         use_native = False
++        # Custom OpenAI-compatible endpoint (e.g. Switchyard) for the chat model.
++        custom_base_url: str | None = None
++        custom_api_key: str | None = None
+
+         if provider in _NATIVE_PROVIDERS:
+             native_flag, key_names = _NATIVE_PROVIDERS[provider]
+@@ -367,11 +390,15 @@
+                     hermes_provider_flag = native_flag
+                     use_native = True
+                     break
+-            # Forward OPENAI_BASE_URL when using native OpenAI key
++            # Forward OPENAI_BASE_URL when using native OpenAI key. Hermes ignores
++            # this env var for the chat model, so also thread it into config.yaml
++            # as a custom provider (see _build_config_yaml).
+             if use_native and provider == "openai":
+                 base_url = os.environ.get("OPENAI_BASE_URL")
+                 if base_url:
+                     env["OPENAI_BASE_URL"] = base_url
++                    custom_base_url = base_url
++                    custom_api_key = os.environ.get("OPENAI_API_KEY")
+
+         if not use_native:
+             openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+@@ -386,9 +413,15 @@
+             env["OPENROUTER_API_KEY"] = openrouter_key
+
+         # Native providers with --provider flag use just the model name;
+-        # everything else (OpenRouter, openai direct) uses provider/model.
+-        cli_model = model if hermes_provider_flag else self.model_name
+-        config_yaml = self._build_config_yaml(cli_model)
++        # a custom OpenAI-compatible endpoint also uses the bare model (the
++        # base_url already targets the gateway); everything else uses provider/model.
++        if custom_base_url:
++            cli_model = model
++        else:
++            cli_model = model if hermes_provider_flag else self.model_name
++        config_yaml = self._build_config_yaml(
++            cli_model, base_url=custom_base_url, api_key=custom_api_key
++        )
+
+         # Pass instruction via env var (safe from shell escaping issues)
+         env["HARBOR_INSTRUCTION"] = instruction

benchmark/prepare_harbor_dataset.py

@@ -29,6 +29,7 @@
 AGENT_VERSIONS_FILE = SCRIPT_DIR / "agent-versions.env"
 PROXY_ASSET_DIR = SCRIPT_DIR / "closed_book_proxy" / "proxy"
 AGENT_ENTRYPOINT = "switchyard-agent-entrypoint.sh"
+COMMIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}")
 TERMINAL_BENCH_2_SOURCE_DATASET = "terminal-bench/terminal-bench-2"
 TERMINAL_BENCH_2_1_SOURCE_DATASET = "terminal-bench/terminal-bench-2-1"
 # Shared across the TB2 family (2.0 + the 2.1 verified iteration): 2.1 tweaks
@@ -187,10 +188,35 @@ def _install_layer(pins: dict[str, str]) -> str:
     claude_version = pins["CLAUDE_CODE_VERSION"]
     codex_version = pins["CODEX_VERSION"]
     opencode_version = pins["OPENCODE_VERSION"]
+    # Hermes (NousResearch hermes-agent) is a per-user uv app installed from
+    # GitHub, not an npm package. Baking it here (build-time, with host network)
+    # means the runtime install() skip-guard short-circuits, so tasks need no
+    # egress for it — enabling closed-book Hermes runs.
+    #
+    # HERMES_VERSION must be a full commit SHA. A tag is not enough: tags can be
+    # deleted or repointed, so two builds could record the same version string while
+    # installing different code — the manifest would assert a reproducibility it does
+    # not have. Requiring one shape is complete by construction, where a deny-list of
+    # moving names ("main", "master", ...) can never be, since any branch name passes.
+    # The installer script is fetched from the same commit for the same reason: pinning
+    # the agent but running whatever installer main has today reintroduces the drift.
+    #
+    # The pin is applied with the installer's --commit, not --branch: --branch reaches
+    # `git clone --branch`, which accepts only branch and tag names and rejects a SHA.
+    # --force-commit is required with it. Without it the installer skips the pin when
+    # the commit is an ancestor of the freshly cloned HEAD, logging a warning and
+    # silently leaving the image on the tip of main — the drift this pin exists to
+    # prevent, arriving as a warning rather than a build failure.
+    hermes_version = pins["HERMES_VERSION"]
+    if not COMMIT_SHA_PATTERN.fullmatch(hermes_version):
+        raise SystemExit(
+            f"HERMES_VERSION={hermes_version!r} is not a full 40-character commit SHA; "
+            "a tag or branch can be repointed and cannot be recorded as a reproducible pin"
+        )
     return f"""
 
 # Switchyard benchmark prebaked coding agents.
-ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version}"
+ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}"
 RUN set -eux; \\
     if command -v apt-get >/dev/null 2>&1; then \\
         apt-get update; \\
@@ -231,6 +257,19 @@ def _install_layer(pins: dict[str, str]) -> str:
     claude --version; \\
     codex --version; \\
     opencode --version
+RUN set -eux; \\
+    export HOME=/root; \\
+    export PATH="/root/.local/bin:$PATH"; \\
+    if command -v apt-get >/dev/null 2>&1; then \\
+        apt-get update; \\
+        apt-get install -y --no-install-recommends git ripgrep xz-utils; \\
+        rm -rf /var/lib/apt/lists/*; \\
+    elif command -v apk >/dev/null 2>&1; then \\
+        apk add --no-cache bash git ripgrep xz; \\
+    fi; \\
+    curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\
+        | bash -s -- --skip-setup --commit {hermes_version} --force-commit; \\
+    hermes version
 """
 
 
@@ -463,7 +502,13 @@ def prepare_dataset(
     overwrite: bool,
 ) -> Path:
     pins = _read_env_file(AGENT_VERSIONS_FILE)
-    required = {"CLAUDE_CODE_VERSION", "CODEX_VERSION", "OPENCODE_VERSION", "NODE_VERSION"}
+    required = {
+        "CLAUDE_CODE_VERSION",
+        "CODEX_VERSION",
+        "HERMES_VERSION",
+        "NODE_VERSION",
+        "OPENCODE_VERSION",
+    }
     missing = sorted(required - pins.keys())
     if missing:
         raise ValueError(f"missing pins in {AGENT_VERSIONS_FILE}: {', '.join(missing)}")

tests/test_prepare_harbor_dataset.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 from types import ModuleType
 
+import pytest
 import yaml
 
 REPO = Path(__file__).resolve().parents[1]
@@ -284,6 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat
     assert manifest["agent_versions"] == {
         "CLAUDE_CODE_VERSION": "2.1.211",
         "CODEX_VERSION": "0.144.5",
+        "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb",
         "NODE_VERSION": "20.11.1",
         "OPENCODE_VERSION": "1.18.3",
     }
@@ -306,3 +308,110 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None:
     proxy_env = "\n".join(compose["services"]["proxy"]["environment"])
     assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env
     assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env
+
+def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None:
+    """Only a full commit SHA can be recorded as a pin.
+
+    The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two
+    builds recording the same string while installing different Hermes code is worse
+    than recording nothing, so the build fails rather than asserting a pin it does not
+    have. Tags are rejected alongside branches: a tag can be deleted or repointed, so
+    it reads as immutable without being so.
+    """
+    base = {
+        "CLAUDE_CODE_VERSION": "1",
+        "CODEX_VERSION": "2",
+        "OPENCODE_VERSION": "3",
+        "NODE_VERSION": "4",
+    }
+    rejected = (
+        "main",
+        "master",
+        "HEAD",
+        "v2026.8.3",
+        "release/2026.8",
+        "3c27eb6",
+        "3C27EB6234BF91B8CEEE9E9071591B31E9B148CB",
+        "3c27eb6234bf91b8ceee9e9071591b31e9b148cbb",
+    )
+    for ref in rejected:
+        with pytest.raises(SystemExit, match="commit SHA"):
+            _load_generator_module()._install_layer({**base, "HERMES_VERSION": ref})
+
+
+def test_the_hermes_installer_is_fetched_at_the_pinned_commit() -> None:
+    """Pinning the agent but running main's installer reintroduces the same drift."""
+    sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"
+    pins = {
+        "CLAUDE_CODE_VERSION": "1",
+        "CODEX_VERSION": "2",
+        "OPENCODE_VERSION": "3",
+        "NODE_VERSION": "4",
+        "HERMES_VERSION": sha,
+    }
+
+    layer = _load_generator_module()._install_layer(pins)
+
+    assert f"hermes-agent/{sha}/scripts/install.sh" in layer
+    assert "hermes-agent/main/" not in layer
+
+
+def test_the_hermes_pin_is_applied_by_commit_and_forced() -> None:
+    """`--branch` reaches `git clone --branch`, which rejects a SHA outright.
+
+    `--force-commit` is what makes the pin take effect. Without it the installer skips
+    the checkout whenever the commit is an ancestor of the freshly cloned HEAD, warns,
+    and leaves the image on the tip of main — the drift the pin exists to prevent,
+    arriving as a warning rather than a build failure.
+    """
+    sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"
+    pins = {
+        "CLAUDE_CODE_VERSION": "1",
+        "CODEX_VERSION": "2",
+        "OPENCODE_VERSION": "3",
+        "NODE_VERSION": "4",
+        "HERMES_VERSION": sha,
+    }
+
+    layer = _load_generator_module()._install_layer(pins)
+
+    assert f"--commit {sha}" in layer
+    assert "--force-commit" in layer
+    assert "--branch" not in layer
+
+
+def test_the_alpine_branch_installs_the_shell_the_installer_needs() -> None:
+    """The installer is piped into bash, which Alpine does not ship by default."""
+    pins = {
+        "CLAUDE_CODE_VERSION": "1",
+        "CODEX_VERSION": "2",
+        "OPENCODE_VERSION": "3",
+        "NODE_VERSION": "4",
+        "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb",
+    }
+
+    layer = _load_generator_module()._install_layer(pins)
+
+    assert "apk add --no-cache bash git ripgrep xz" in layer
+
+
+def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> None:
+    """Absent, it must fail the shared pin check rather than crash reading the layer."""
+    module = _load_generator_module()
+    versions = tmp_path / "agent-versions.env"
+    versions.write_text(
+        "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nOPENCODE_VERSION=3\nNODE_VERSION=4\n"
+    )
+    module.AGENT_VERSIONS_FILE = versions
+    source = tmp_path / "source"
+    _write_task(source, "task-a", "[environment]\n", "FROM ubuntu:22.04\n")
+
+    with pytest.raises(ValueError, match="missing pins.*HERMES_VERSION"):
+        module.prepare_dataset(
+            source_dataset="openthoughts-tblite@2.0",
+            source_dir=source,
+            output_dir=tmp_path / "prepared",
+            harbor_command="harbor",
+            overwrite=False,
+        )
+