txn2 · cjimti · Jul 16, 2026 · Jul 16, 2026
@@ -964,14 +964,17 @@ bench-supersede-report:
 	@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
 	$(BUILD_DIR)/benchrun -supersede -summarize build/bench-results/supersede-a3.json
 
-## bench-cold-start: Run the cold-start knowledge-growth curriculum (issue #963; needs an empty-seeded a3: bench-up BENCH_ARM=a3 BENCH_SEED_PAGES=0 + bench-seed-datahub-empty; LLM=anthropic|scripted|claude-cli, K=, MODEL=)
+## bench-cold-start: Run the cold-start knowledge-growth curriculum into a fresh per-run dir under build/bench-results/ (issue #963; needs an empty-seeded a3: bench-up BENCH_ARM=a3 BENCH_SEED_PAGES=0 + bench-seed-datahub-empty on a FRESH DataHub quickstart; LLM=anthropic|scripted|claude-cli, K=, MODEL=, SETTLE=)
 bench-cold-start:
 	@mkdir -p build/bench-results
 	@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
 	@echo "Resetting cold-start state so the baseline is truly empty (search gate, prior insights/changesets, and any promoted knowledge pages persist in Postgres across runs)..."
 	@echo "  (CASCADE also clears portal_threads, which FK-references knowledge pages; the bench stack is disposable scratch state.)"
 	@$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \
 		-c "TRUNCATE search_gate_discovery, memory_records, knowledge_changesets, portal_knowledge_pages CASCADE"
+	@out_dir="build/bench-results/cold-start-a3-$$(date +%Y%m%d-%H%M%S)"; \
+	mkdir -p "$$out_dir"; \
+	echo "Cold-start results dir: $$out_dir (each run gets its own dir; nothing is ever overwritten)"; \
 	$(BUILD_DIR)/benchrun \
 		-cold-start \
 		-arm a3 \
@@ -980,20 +983,25 @@ bench-cold-start:
 		-curriculum bench/curriculum \
 		-tasks bench/tasks \
 		-git-commit $$(git rev-parse HEAD) \
-		-out build/bench-results/cold-start-a3.json \
+		-out "$$out_dir/results.json" \
 		$(if $(LLM),-llm $(LLM),) \
 		$(if $(SCRIPT),-script $(SCRIPT),) \
 		$(if $(K),-k $(K),) \
-		$(if $(MODEL),-model $(MODEL),)
+		$(if $(MODEL),-model $(MODEL),) \
+		$(if $(SETTLE),-settle $(SETTLE),)
 
-## bench-cold-start-smoke: Run the scripted (no-API-key) cold-start smoke against the running a3 platform
+## bench-cold-start-smoke: Run the scripted (no-API-key) cold-start smoke against the running a3 platform (no cache-settle pause)
 bench-cold-start-smoke:
-	@$(MAKE) bench-cold-start LLM=scripted SCRIPT=bench/curriculum/scripted-cold-start-smoke.json K=1
+	@$(MAKE) bench-cold-start LLM=scripted SCRIPT=bench/curriculum/scripted-cold-start-smoke.json K=1 SETTLE=0s
 
-## bench-cold-start-report: Print the human summary (learning curve) of the last cold-start run
+## bench-cold-start-report: Print the human summary (learning curve) of a cold-start run (RESULTS=<run dir>/results.json)
 bench-cold-start-report:
+	@if [ -z "$(RESULTS)" ]; then \
+		echo "ERROR: set RESULTS=<path to a cold-start results.json>. Available run dirs:"; \
+		ls -d build/bench-results/cold-start-a3-*/ 2>/dev/null || echo "  (none under build/bench-results/)"; \
+		exit 1; fi
 	@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
-	$(BUILD_DIR)/benchrun -cold-start -summarize build/bench-results/cold-start-a3.json
+	$(BUILD_DIR)/benchrun -cold-start -summarize $(RESULTS)
 
 ## bench-report: Print the human summary of the last run for BENCH_ARM
 bench-report:

@@ -406,6 +406,36 @@ suite reuses one identity pool (a distinct teacher per lesson, fresh evaluators
 per checkpoint), and a run refuses to start when the lessons plus per-checkpoint
 evaluators exceed the pool.
 
+**A cold-start run requires a FRESH DataHub quickstart**, not just re-ingesting
+the empty seed. `apply_knowledge` description promotions write the
+`editableDatasetProperties` aspect, and a prior a2 seed leaves
+`editableSchemaMetadata` column docs, tags, and deprecation; the empty seed
+(`bench_mces_empty.json`) upserts only `datasetProperties`, so re-ingesting it
+cannot clear any of that, and the read path prefers the editable description
+when non-empty. A **baseline-integrity preflight** enforces this: before any
+episode is spent, the run reads every lesson entity through the platform,
+lists insights for every (teacher, lesson URN) pair, and scans knowledge pages
+for the curriculum slugs, refusing to start if anything is already there.
+Postgres state (search gate, memory records, changesets, knowledge pages) is
+reset by the `bench-cold-start` target's TRUNCATE; DataHub state requires
+`datahub docker nuke`, a re-quickstart, then `make bench-seed-datahub-empty`.
+
+Between a successful datahub-sink promote and the next eval checkpoint the
+runner pauses for the `-settle` window (default `5m`, matching the a3 semantic
+cache TTL, `SETTLE=` on the make target) so a table-context cache entry
+populated by the previous checkpoint's evaluators can never serve the stale
+pre-promotion description to the next ones. Page-sink promotes skip the pause
+(page hits are served live from the portal store, nothing is cached), as do
+lessons that did not promote. The window is recorded on the results manifest,
+and the scripted smoke runs with `SETTLE=0s`.
+
+**Pass criteria.** A zero exit code is NOT a pass signal: capture misses and
+promote refusals are measured outcomes by design, so a run can exit 0 with a
+flat curve. A valid full run's summary must read `lessons 6 (captured 6,
+promoted 6)` with `harness failures 0`, enrichment coverage climbing from the
+baseline, and no evaluator memory-write warning (evaluators are forbidden from
+saving memories; the summary warns if the audit-side count is non-zero).
+
 ## Running
 
 From the repository root:
@@ -439,19 +469,38 @@ the insight/changeset APIs, supersede, grading, and the metrics against the live
 platform with no API key and no model variance.
 
 The cold-start suite (#963) boots the same `a3` arm but with the empty baseline
-(no knowledge pages, undocumented DataHub), then teaches the curriculum:
+(no knowledge pages, undocumented DataHub, on a FRESH quickstart; the preflight
+refuses leftovers from a prior run or an a2 seed, see the cold-start section
+above), then teaches the curriculum:
 
 ```bash
-# Boot a3 with an empty enrichment layer: no knowledge pages, empty DataHub.
+# Fresh DataHub quickstart first (datahub docker nuke + re-quickstart if reused),
+# then boot a3 with an empty enrichment layer: no knowledge pages, empty DataHub.
 make bench-up BENCH_ARM=a3 BENCH_SEED_PAGES=0
 make bench-seed-datahub-empty            # entities present, undocumented
 
-make bench-cold-start-smoke              # scripted no-API-key loop validation
+make bench-cold-start-smoke              # scripted no-API-key loop validation (SETTLE=0s)
 make bench-cold-start K=1                # real learning-curve run (needs a model)
 make bench-cold-start LLM=claude-cli MODEL=sonnet K=1   # subscription run
-make bench-cold-start-report             # print the learning curve
+make bench-cold-start-report RESULTS=build/bench-results/cold-start-a3-<stamp>/results.json
 ```
 
+Every `bench-cold-start` invocation writes into its own timestamped directory
+(`build/bench-results/cold-start-a3-<stamp>/results.json` plus a
+`results.json.transcripts/` directory beside it) and `benchrun` refuses an
+`-out` that already exists, so a re-run can never overwrite a prior run's
+paid-for results; `bench-cold-start-report` therefore takes the run to
+summarize via `RESULTS=` (it lists the available run dirs when unset).
+
+Two model paths, two cost profiles. `LLM=claude-cli` runs each episode through
+a real `claude -p` client and is subscription-funded (the runner strips
+`ANTHROPIC_API_KEY` from the child environment): a k=1 full run is 181 episodes
+(6 teaches + 7 checkpoints x 25 eval tasks) at roughly 50s each, so plan for
+2.5-3 hours of wall clock. The `anthropic` adapter bills the API: k=1 is
+estimated around USD 20 at claude-sonnet-5 pricing, extrapolated from the
+phase-2 per-attempt token data. Either way a full run exceeds an interactive
+session: launch it in the background and read the summary from the run dir.
+
 The **scripted cold-start smoke** (`-llm scripted`) plays
 `curriculum/scripted-cold-start-smoke.json` (generated): each lesson captures its
 fact and the harness drives the real promotion; each eval task answers with its

@@ -57,6 +57,7 @@ type config struct {
 	merge         string
 	coldStart     bool
 	curriculumDir string
+	settle        time.Duration
 	supersede     bool
 	teachBudget   int
 }
@@ -102,6 +103,7 @@ func parseFlags() config {
 	flag.StringVar(&cfg.merge, "merge", "", "comma-separated per-pass lifecycle result JSONs (with -lifecycle): merge independent k=1 passes into one k=N result and exit")
 	flag.BoolVar(&cfg.coldStart, "cold-start", false, "run the cold-start knowledge-growth curriculum (issue #963) instead of the task suites")
 	flag.StringVar(&cfg.curriculumDir, "curriculum", "curriculum", "curriculum YAML directory (with -cold-start)")
+	flag.DurationVar(&cfg.settle, "settle", 5*time.Minute, "pause between a successful promote and the next eval checkpoint (with -cold-start), matching the a3 semantic-cache TTL so a stale pre-promotion cache entry cannot attenuate the lift; 0 disables (scripted smoke)")
 	flag.BoolVar(&cfg.supersede, "supersede", false, "run the isolated supersede sub-benchmark (issue #964: the supersede protocols only) instead of the full S5 lifecycle")
 	flag.IntVar(&cfg.teachBudget, "teach-budget", 0, "override the per-episode tool-call budget for the capture-bearing stages (teach, update); 0 = protocol budget (issue #964 capture-budget lever)")
 	flag.Parse()
@@ -325,6 +327,9 @@ func runColdStart(cfg config) error {
 	if cfg.baseline != "" {
 		return errors.New("-baseline is not supported with -cold-start (the regression gate scores S1-S3 task suites, not the learning curve)")
 	}
+	if err := refuseExistingRunArtifacts(cfg.out); err != nil {
+		return err
+	}
 	log := slog.New(slog.NewTextHandler(os.Stderr, nil))
 	opts := coldstart.Options{
 		Target:        target.Target{BaseURL: cfg.url, Credential: cfg.credential},
@@ -338,6 +343,7 @@ func runColdStart(cfg config) error {
 		GitCommit:     cfg.gitCommit,
 		AuditTimeout:  cfg.auditTimeout,
 		IdentityKeys:  cfg.identityKeys,
+		Settle:        cfg.settle,
 		OnCheckpoint: func(r *coldstart.Results) {
 			if err := r.WriteJSON(cfg.out); err != nil {
 				log.Warn("checkpoint write", "error", err)
@@ -706,6 +712,30 @@ func transcriptDir(out string) string {
 	return filepath.Join(filepath.Dir(out), filepath.Base(out)+".transcripts")
 }
 
+// refuseExistingRunArtifacts errors when out or its sibling transcript
+// directory already exists. A cold-start run's results and transcripts are
+// paid-for evidence that can never be regenerated; an existing -out would be
+// silently clobbered by the per-checkpoint flush, and the transcript dir by
+// the deterministic per-episode filenames — a run interrupted before its first
+// flush leaves only transcripts, which a rerun to the same -out would destroy.
+// There is deliberately no override: the operator picks a new path, every
+// run's data is kept.
+func refuseExistingRunArtifacts(out string) error {
+	if _, err := os.Stat(out); err == nil {
+		return fmt.Errorf("-out %s already exists; refusing to overwrite a prior run's results; choose a new -out path (all run data is kept)", out)
+	}
+	if dir := transcriptDir(out); dirExists(dir) {
+		return fmt.Errorf("transcript directory %s already exists (an earlier run's episode transcripts); refusing to overwrite; choose a new -out path (all run data is kept)", dir)
+	}
+	return nil
+}
+
+// dirExists reports whether path exists and is a directory.
+func dirExists(path string) bool {
+	info, err := os.Stat(path)
+	return err == nil && info.IsDir()
+}
+
 // claudeCLIProvider is the -llm value selecting the real Claude Code client
 // path (see package claudecli).
 const claudeCLIProvider = "claude-cli"

@@ -3,6 +3,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/txn2/mcp-data-platform/bench/internal/lifecycle"
@@ -245,6 +246,71 @@ func TestSupersedeRejectsBaseline(t *testing.T) {
 	}
 }
 
+// TestColdStartRequiresArm proves the cold-start run refuses to start without
+// an arm rather than launching an unattributable run.
+func TestColdStartRequiresArm(t *testing.T) {
+	if err := runColdStart(config{coldStart: true}); err == nil {
+		t.Fatal("cold-start run accepted an empty arm")
+	}
+}
+
+// TestColdStartRejectsBaseline proves -baseline is refused for cold-start runs:
+// the regression gate scores the S1-S3 task shape, not the learning curve, so a
+// silently-ignored -baseline would give a false sense of gating.
+func TestColdStartRejectsBaseline(t *testing.T) {
+	if err := runColdStart(config{coldStart: true, arm: "a3", baseline: "b.json"}); err == nil {
+		t.Fatal("cold-start run accepted -baseline")
+	}
+}
+
+// TestColdStartRefusesToOverwriteResults proves a cold-start run whose -out
+// already exists is refused before anything is spent: a prior run's results and
+// transcripts are paid-for evidence that must never be overwritten, and there
+// is deliberately no override flag.
+func TestColdStartRefusesToOverwriteResults(t *testing.T) {
+	out := filepath.Join(t.TempDir(), "results.json")
+	if err := os.WriteFile(out, []byte("{}"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	err := runColdStart(config{coldStart: true, arm: "a3", out: out})
+	if err == nil {
+		t.Fatal("cold-start run accepted an -out that already exists")
+	}
+	if got := err.Error(); !strings.Contains(got, "refusing to overwrite") {
+		t.Errorf("error should explain the overwrite refusal, got: %s", got)
+	}
+}
+
+// TestColdStartRefusesToOverwriteTranscripts proves the refusal also covers the
+// sibling transcript directory: a run interrupted before its first checkpoint
+// flush leaves transcripts but no results file, and a rerun to the same -out
+// would overwrite them episode by episode (deterministic filenames).
+func TestColdStartRefusesToOverwriteTranscripts(t *testing.T) {
+	out := filepath.Join(t.TempDir(), "results.json")
+	if err := os.MkdirAll(transcriptDir(out), 0o750); err != nil {
+		t.Fatal(err)
+	}
+	err := runColdStart(config{coldStart: true, arm: "a3", out: out})
+	if err == nil {
+		t.Fatal("cold-start run accepted an -out whose transcript directory already exists")
+	}
+	if got := err.Error(); !strings.Contains(got, "transcript directory") {
+		t.Errorf("error should name the transcript directory, got: %s", got)
+	}
+}
+
+// TestBuildColdStartFactoryErrors proves the factory wiring fails fast on a
+// scripted run without a script and on an unknown provider, mirroring the
+// lifecycle factory contract.
+func TestBuildColdStartFactoryErrors(t *testing.T) {
+	if _, err := buildColdStartFactory(config{llmProvider: "scripted"}); err == nil {
+		t.Error("scripted provider without -script was accepted")
+	}
+	if _, err := buildColdStartFactory(config{llmProvider: "nope"}); err == nil {
+		t.Error("unknown provider was accepted")
+	}
+}
+
 // baselineFile writes a results JSON to a temp path and returns it, standing in
 // for a committed baseline that gateOnBaseline loads from disk.
 func baselineFile(t *testing.T, r *report.Results) string {

@@ -22,6 +22,7 @@ lessons:
       page:
         slug: revenue-reporting-policy
         title: Revenue Reporting Policy
+        summary: Revenue = amount - discount over completed orders only; amounts are stored in US cents. The authoritative definition for all bench warehouse reporting.
         body: |-
             # Revenue Reporting Policy
 
@@ -51,6 +52,7 @@ lessons:
       page:
         slug: fiscal-calendar-policy
         title: Fiscal Calendar Policy
+        summary: 'The fiscal year starts February 1: fiscal year 2025 runs 2025-02-01 through 2026-01-31. Fiscal figures must not be computed over the calendar year.'
         body: |-
             # Fiscal Calendar Policy
 
@@ -91,6 +93,7 @@ lessons:
       page:
         slug: customer-tier-definitions
         title: Customer Tier Definitions
+        summary: A 'key account' is any customer on the plus or enterprise tier, a derived grouping not stored in any column and broader than the enterprise tier alone.
         body: |-
             # Customer Tier Definitions