Merge issue-143-engineer: Harden scheduled regression evidence capture (#143)

Author: alexeygrigorev

.github/workflows/scheduled-full-regression.yml

@@ -56,6 +56,7 @@ jobs:
             --summary "$GITHUB_STEP_SUMMARY"
           uv run --frozen python -m ci.verification environment \
             --plan .tmp/scheduled/verification-plan.json --component selector \
+            --allow-hosted-runner-drift \
             --output .tmp/scheduled/selector-environment.json
           uv run --frozen python -m ci.verification record \
             --plan .tmp/scheduled/verification-plan.json --component selector \
@@ -67,6 +68,7 @@ jobs:
             --artifact-id "ci-selection-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
             --artifact-root .tmp/scheduled \
             --execution-environment .tmp/scheduled/selector-environment.json \
+            --allow-hosted-runner-drift \
             --machine-output .tmp/scheduled/verification-plan.json \
             --output .tmp/scheduled/selector-evidence.json
       - name: Preserve the scheduled decision and shared plan
@@ -85,6 +87,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"
@@ -114,6 +117,7 @@ jobs:
             environment=.tmp/evidence/$component-environment.json
             uv run --frozen python -m ci.verification environment \
               --plan .tmp/input/verification-plan.json --component "$component" \
+              --allow-hosted-runner-drift \
               --output "$environment"
             uv run --frozen python -m ci.verification record \
               --plan .tmp/input/verification-plan.json --component "$component" \
@@ -124,6 +128,7 @@ jobs:
               --job-id quality \
               --artifact-id "verification-component-quality-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
               --artifact-root .tmp/evidence --execution-environment "$environment" \
+              --allow-hosted-runner-drift \
               --machine-output .tmp/evidence/quality-output.log \
               --output ".tmp/evidence/$component-evidence.json"
           done
@@ -143,6 +148,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"
@@ -159,6 +165,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"
@@ -177,6 +184,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"
@@ -202,6 +210,7 @@ jobs:
           mkdir -p .tmp/evidence
           uv run --frozen python -m ci.verification environment \
             --plan .tmp/input/verification-plan.json --component django \
+            --allow-hosted-runner-drift \
             --output .tmp/evidence/django-environment.json
           uv run --frozen python -m ci.verification record \
             --plan .tmp/input/verification-plan.json --component django \
@@ -213,6 +222,7 @@ jobs:
             --artifact-id "verification-component-django-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
             --artifact-root .tmp/evidence \
             --execution-environment .tmp/evidence/django-environment.json \
+            --allow-hosted-runner-drift \
             --machine-output .tmp/evidence/django-output.log \
             --output .tmp/evidence/django-evidence.json
       - uses: actions/upload-artifact@v4
@@ -234,6 +244,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"
@@ -249,7 +260,7 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p .tmp/evidence
-          make test-playwright 2>&1 | tee .tmp/evidence/playwright-output.log
+          make test-playwright-core 2>&1 | tee .tmp/evidence/playwright-output.log
       - name: Record shared Playwright envelope
         if: always()
         env:
@@ -259,17 +270,19 @@ jobs:
           mkdir -p .tmp/evidence
           uv run --frozen python -m ci.verification environment \
             --plan .tmp/input/verification-plan.json --component playwright \
+            --allow-hosted-runner-drift \
             --output .tmp/evidence/playwright-environment.json
           uv run --frozen python -m ci.verification record \
             --plan .tmp/input/verification-plan.json --component playwright \
-            --result "$JOB_STATUS" --command "make test-playwright" \
+            --result "$JOB_STATUS" --command "make test-playwright-core" \
             --origin-kind github_actions --repository-id "$GITHUB_REPOSITORY" \
             --workflow .github/workflows/scheduled-full-regression.yml --ref "$GITHUB_REF" \
             --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \
             --job-id playwright \
             --artifact-id "verification-component-playwright-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
             --artifact-root .tmp/evidence \
             --execution-environment .tmp/evidence/playwright-environment.json \
+            --allow-hosted-runner-drift \
             --machine-output .tmp/evidence/playwright-output.log \
             --output .tmp/evidence/playwright-evidence.json
       - uses: actions/upload-artifact@v4
@@ -288,6 +301,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"
@@ -420,6 +434,7 @@ jobs:
             }' < .tmp/evidence/container-assertions.txt > .tmp/evidence/container-check.json
           uv run --frozen python -m ci.verification environment \
             --plan .tmp/input/verification-plan.json --component container \
+            --allow-hosted-runner-drift \
             --output .tmp/evidence/container-environment.json
           uv run --frozen python -m ci.verification record \
             --plan .tmp/input/verification-plan.json --component container \
@@ -431,6 +446,7 @@ jobs:
             --artifact-id "verification-component-container-$GITHUB_RUN_ID-attempt-$GITHUB_RUN_ATTEMPT" \
             --artifact-root .tmp/evidence \
             --execution-environment .tmp/evidence/container-environment.json \
+            --allow-hosted-runner-drift \
             --machine-output .tmp/evidence/container-check.json \
             --output .tmp/evidence/container-evidence.json
       - uses: actions/upload-artifact@v4
@@ -453,6 +469,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"
@@ -522,6 +539,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           version: "0.10.11"

ci/tests/test_workflows.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import json
+import re
 from pathlib import Path
 from typing import Any
 
@@ -21,6 +23,27 @@ def runs(job: dict[str, Any]) -> str:
     )
 
 
+def logical_shell_commands(job: dict[str, Any]) -> list[str]:
+    commands: list[str] = []
+    continuation: list[str] = []
+    for raw_line in runs(job).splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        continued = line.endswith("\\")
+        continuation.append(line.removesuffix("\\").rstrip())
+        if not continued:
+            commands.append(" ".join(continuation))
+            continuation = []
+    assert not continuation
+    return commands
+
+
+def verification_invocations(job: dict[str, Any], subcommand: str) -> list[str]:
+    marker = f"python -m ci.verification {subcommand}"
+    return [command for command in logical_shell_commands(job) if marker in command]
+
+
 def test_normal_workflow_keeps_release_concurrency_and_exact_base_contract() -> None:
     data = workflow("ci.yml")
     concurrency = data["concurrency"]
@@ -294,7 +317,59 @@ def test_scheduled_workflow_has_no_mutation_or_aws_jobs_and_checks_exact_sha() -
             step for step in job["steps"] if step.get("uses") == "actions/checkout@v4"
         ]
         assert checkout_steps
-        assert all(step["with"]["ref"] == "${{ github.sha }}" for step in checkout_steps)
+        assert all(
+            step["with"]
+            == {
+                "ref": "${{ github.sha }}",
+                "fetch-depth": "0",
+            }
+            for step in checkout_steps
+        )
+
+
+def test_scheduled_evidence_capture_and_record_symmetrically_allow_runner_drift() -> None:
+    jobs = workflow("scheduled-full-regression.yml")["jobs"]
+    evidence_jobs = {"selector", "quality", "django", "playwright", "container"}
+    assert {
+        name
+        for name, job in jobs.items()
+        if verification_invocations(job, "environment") or verification_invocations(job, "record")
+    } == evidence_jobs
+
+    for name in evidence_jobs:
+        environment_calls = verification_invocations(jobs[name], "environment")
+        record_calls = verification_invocations(jobs[name], "record")
+        assert len(environment_calls) == 1, name
+        assert len(record_calls) == 1, name
+        assert environment_calls[0].split().count("--allow-hosted-runner-drift") == 1, name
+        assert record_calls[0].split().count("--allow-hosted-runner-drift") == 1, name
+
+    assert "for component in quality evidence_validation; do" in runs(jobs["quality"])
+    for name in ("selector", "django", "playwright", "container"):
+        assert f"--component {name}" in verification_invocations(jobs[name], "environment")[0]
+
+
+def test_scheduled_playwright_executes_and_records_the_planner_core_command() -> None:
+    playwright = workflow("scheduled-full-regression.yml")["jobs"]["playwright"]
+    planner_command = json.loads((ROOT / "ci" / "ownership.json").read_text(encoding="utf-8"))[
+        "components"
+    ]["playwright"]["command"]
+    assert planner_command == "make test-playwright-core"
+    command_pattern = re.compile(r"\bmake test-playwright(?:-core)?\b")
+
+    execution = next(
+        step
+        for step in playwright["steps"]
+        if step.get("name") == "Run and retain the complete Playwright output"
+    )
+    recording = next(
+        step
+        for step in playwright["steps"]
+        if step.get("name") == "Record shared Playwright envelope"
+    )
+    assert command_pattern.findall(execution["run"]) == [planner_command]
+    assert command_pattern.findall(recording["run"]) == [planner_command]
+    assert f'--command "{planner_command}"' in recording["run"]
 
 
 def test_scheduled_full_marker_and_gate_cover_every_component_or_exact_skip() -> None:
@@ -316,7 +391,7 @@ def test_scheduled_full_marker_and_gate_cover_every_component_or_exact_skip() ->
     assert "make test-factories" in runs(jobs["factories"])
     assert "make test-migrations" in runs(jobs["migrations"])
     assert "make test" in runs(jobs["django"])
-    assert "make test-playwright" in runs(jobs["playwright"])
+    assert "make test-playwright-core" in runs(jobs["playwright"])
     assert "ci.quality_contract" in runs(jobs["quality"])
     assert "make verification-quality" not in runs(jobs["quality"])
     container = runs(jobs["container"])