From d9af6a338da99198c3d1b6c445eb67c700cff769 Mon Sep 17 00:00:00 2001 From: cjimti Date: Tue, 14 Jul 2026 22:17:58 -0700 Subject: [PATCH] feat(bench): cold-start knowledge-growth suite (#963) Add a cold-start knowledge-growth benchmark that measures the platform getting smarter as knowledge accumulates. It boots the a3 arm against an empty enrichment layer (undocumented DataHub, no knowledge pages), teaches a six-lesson curriculum (one fact per S3 trap class, promoted to the same DataHub descriptions and knowledge pages the A2 seed pre-loads), and re-runs the fixed S3 trap suite with a fresh, never-taught evaluator identity after each promotion. The result is a learning curve of accuracy, per-trap-class resistance, and enrichment coverage as promoted knowledge accumulates toward the A2 fact-bearing ceiling, exercising the coupling between the lifecycle and the enrichment layer end to end. - internal/curriculum: curriculum schema, loader, and set hash - internal/coldstart: curriculum runner, fresh-evaluator eval loop, learning-curve metrics, and results model - internal/promote: shared reviewer-promotion path (approve + apply_knowledge + verify), extracted so S5 lifecycle and cold-start share one implementation - gen: empty-DataHub emitter (bench_mces_empty.json) plus the curriculum and scripted-smoke generators; committed artifacts are drift-checked - auditapi: enrichment-coverage ratio (EnrichedCalls/AuditedCalls) - benchrun -cold-start mode; Makefile bench-cold-start / -smoke / -report, bench-seed-datahub-empty, and BENCH_SEED_PAGES=0 for the empty baseline - docs: bench/README, benchmarks.md, llms.txt, llms-full.txt A real learning-curve run is gated by model budget; the whole loop is validated with no API key by make bench-cold-start-smoke. --- Makefile | 50 +- bench/README.md | 79 ++- bench/benchrun/main.go | 142 ++++- bench/curriculum/cs-traps.yaml | 119 ++++ .../curriculum/scripted-cold-start-smoke.json | 590 +++++++++++++++++ bench/internal/auditapi/auditapi.go | 13 + bench/internal/auditapi/auditapi_test.go | 13 + bench/internal/coldstart/episode.go | 288 +++++++++ bench/internal/coldstart/report.go | 380 +++++++++++ bench/internal/coldstart/report_test.go | 134 ++++ bench/internal/coldstart/runner.go | 379 +++++++++++ bench/internal/coldstart/runner_test.go | 596 ++++++++++++++++++ bench/internal/curriculum/curriculum.go | 194 ++++++ bench/internal/curriculum/curriculum_test.go | 149 +++++ bench/internal/gen/emit_curriculum.go | 124 ++++ bench/internal/gen/emit_datahub.go | 23 +- bench/internal/gen/gen_test.go | 106 ++++ bench/internal/lifecycle/promote.go | 135 +--- bench/internal/lifecycle/runner.go | 28 +- bench/internal/promote/promote.go | 183 ++++++ bench/internal/promote/promote_test.go | 275 ++++++++ bench/seed/datahub/bench_mces_empty.json | 54 ++ bench/seedgen/main.go | 58 +- docs/llms-full.txt | 2 +- docs/llms.txt | 2 +- docs/reference/benchmarks.md | 15 + 26 files changed, 3965 insertions(+), 166 deletions(-) create mode 100644 bench/curriculum/cs-traps.yaml create mode 100644 bench/curriculum/scripted-cold-start-smoke.json create mode 100644 bench/internal/coldstart/episode.go create mode 100644 bench/internal/coldstart/report.go create mode 100644 bench/internal/coldstart/report_test.go create mode 100644 bench/internal/coldstart/runner.go create mode 100644 bench/internal/coldstart/runner_test.go create mode 100644 bench/internal/curriculum/curriculum.go create mode 100644 bench/internal/curriculum/curriculum_test.go create mode 100644 bench/internal/gen/emit_curriculum.go create mode 100644 bench/internal/promote/promote.go create mode 100644 bench/internal/promote/promote_test.go create mode 100644 bench/seed/datahub/bench_mces_empty.json diff --git a/Makefile b/Makefile index 4fe923e4a..cb1382458 100644 --- a/Makefile +++ b/Makefile @@ -822,7 +822,7 @@ BENCH_COMPOSE := DOCKER_DEFAULT_PLATFORM= docker compose -f docker-compose.e2e.y ## bench-gen: Regenerate seed artifacts, the task set, and the S5 protocols from the fixed seed bench-gen: - @cd bench && $(GO) run ./seedgen -seed-dir seed -tasks-dir tasks -protocols-dir protocols + @cd bench && $(GO) run ./seedgen -seed-dir seed -tasks-dir tasks -protocols-dir protocols -curriculum-dir curriculum ## bench-up: Start the compose stack, seed the bench warehouse, and run the platform (BENCH_ARM=a0|a1|a2|a3) bench-up: e2e-up @@ -856,9 +856,13 @@ bench-up: e2e-up if ! kill -0 $$(cat $(BENCH_PID)) 2>/dev/null; then \ echo "ERROR: bench platform exited after start (another server answered readiness?); see $(BENCH_LOG)"; \ tail -20 $(BENCH_LOG); exit 1; fi - @echo "Seeding knowledge pages (requires platform migrations, just applied on boot)..." - @$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \ - < bench/seed/postgres/knowledge_pages.sql + @if [ "$(BENCH_SEED_PAGES)" = "0" ]; then \ + echo "Skipping knowledge-page seeding (BENCH_SEED_PAGES=0, cold-start empty baseline)."; \ + else \ + echo "Seeding knowledge pages (requires platform migrations, just applied on boot)..."; \ + $(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \ + < bench/seed/postgres/knowledge_pages.sql; \ + fi @echo "Platform ready (pid $$(cat $(BENCH_PID)), arm $(BENCH_ARM))." ## bench-seed-datahub: Push bench metadata into a running DataHub quickstart (a2 arm) @@ -869,6 +873,13 @@ bench-seed-datahub: @printf 'source:\n type: file\n config:\n path: %s/bench/seed/datahub/bench_mces.json\nsink:\n type: datahub-rest\n config:\n server: %s\n' "$$(pwd)" "$(BENCH_DATAHUB_GMS)" > $(BUILD_DIR)/bench-datahub-recipe.yml datahub ingest -c $(BUILD_DIR)/bench-datahub-recipe.yml +## bench-seed-datahub-empty: Push the cold-start empty baseline into DataHub (entities present, undocumented; issue #963) +bench-seed-datahub-empty: + @command -v datahub >/dev/null 2>&1 || { echo "ERROR: datahub CLI not found (pip install acryl-datahub)"; exit 1; } + @mkdir -p $(BUILD_DIR) + @printf 'source:\n type: file\n config:\n path: %s/bench/seed/datahub/bench_mces_empty.json\nsink:\n type: datahub-rest\n config:\n server: %s\n' "$$(pwd)" "$(BENCH_DATAHUB_GMS)" > $(BUILD_DIR)/bench-datahub-empty-recipe.yml + datahub ingest -c $(BUILD_DIR)/bench-datahub-empty-recipe.yml + ## bench-run: Run the benchmark (ARM must match bench-up; LLM=anthropic|scripted|claude-cli, SUITE=, K=, MODEL=) bench-run: @mkdir -p build/bench-results @@ -923,6 +934,37 @@ bench-lifecycle-report: @cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun $(BUILD_DIR)/benchrun -lifecycle -summarize build/bench-results/lifecycle-a3.json +## bench-cold-start: Run the cold-start knowledge-growth curriculum (issue #963; needs an empty-seeded a3: bench-up BENCH_ARM=a3 BENCH_SEED_PAGES=0 + bench-seed-datahub-empty; LLM=anthropic|scripted|claude-cli, K=, MODEL=) +bench-cold-start: + @mkdir -p build/bench-results + @cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun + @echo "Resetting cold-start state so the baseline is truly empty (search gate, prior insights/changesets, and any promoted knowledge pages persist in Postgres across runs)..." + @echo " (CASCADE also clears portal_threads, which FK-references knowledge pages; the bench stack is disposable scratch state.)" + @$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \ + -c "TRUNCATE search_gate_discovery, memory_records, knowledge_changesets, portal_knowledge_pages CASCADE" + $(BUILD_DIR)/benchrun \ + -cold-start \ + -arm a3 \ + -url $(BENCH_URL) \ + -credential $(BENCH_KEY) \ + -curriculum bench/curriculum \ + -tasks bench/tasks \ + -git-commit $$(git rev-parse HEAD) \ + -out build/bench-results/cold-start-a3.json \ + $(if $(LLM),-llm $(LLM),) \ + $(if $(SCRIPT),-script $(SCRIPT),) \ + $(if $(K),-k $(K),) \ + $(if $(MODEL),-model $(MODEL),) + +## bench-cold-start-smoke: Run the scripted (no-API-key) cold-start smoke against the running a3 platform +bench-cold-start-smoke: + @$(MAKE) bench-cold-start LLM=scripted SCRIPT=bench/curriculum/scripted-cold-start-smoke.json K=1 + +## bench-cold-start-report: Print the human summary (learning curve) of the last cold-start run +bench-cold-start-report: + @cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun + $(BUILD_DIR)/benchrun -cold-start -summarize build/bench-results/cold-start-a3.json + ## bench-report: Print the human summary of the last run for BENCH_ARM bench-report: @cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun diff --git a/bench/README.md b/bench/README.md index a56381133..85ee1f31e 100644 --- a/bench/README.md +++ b/bench/README.md @@ -219,6 +219,53 @@ applicable lifecycle). Harness-level failures (connect, adapter, API read-back) are excluded from the metrics and reported separately, mirroring the S1–S3 pipeline. +## Cold-start knowledge growth (#963) + +The S1-S3 and S5 suites ablate the platform with a **pre-seeded** knowledge base. +The cold-start suite (`curriculum/`, generated) instead starts from an **empty +enrichment layer** and measures the platform getting smarter as knowledge +accumulates — a learning curve whose independent variable is the amount of +**promoted (shared)** knowledge, holding the model, prompt, task set, and dataset +constant. + +It runs on the `a3` arm against an **empty baseline**: an undocumented DataHub +(`seed/datahub/bench_mces_empty.json` — entities present, but no descriptions, +column docs, tags, or glossary) and **no knowledge pages** (`bench-up` with +`BENCH_SEED_PAGES=0`). Over an ordered **curriculum** of six lessons — one per S3 +trap class — the harness: + +1. **Teaches** each fact (a teacher identity states it and captures it via + `memory_capture`), then **promotes** it to its sink through `apply_knowledge`: + a DataHub entity description (units, freshness, deprecation) or a portal + knowledge page (net-revenue policy, fiscal calendar, tier definitions). Each + lesson teaches the same S3 trap fact the A2 seed pre-loads, so the trap + suite reaches its A2 accuracy ceiling once all six are promoted (the + fact-bearing description and page channels are restored; A2's auxiliary + aspects — tags, the structured deprecation flag, column docs — are not, but + the S3 traps read the fact text, not those). Capture and promotion are verified through + the admin insights and changesets APIs, reusing the same reviewer-promotion + path (`internal/promote`) the S5 lifecycle uses. +2. **Evaluates** at every checkpoint (the empty baseline and after each lesson) + by re-running the fixed S3 trap suite with a **fresh, never-taught evaluator + identity**. Its only knowledge source is what the platform surfaces — + cross-enrichment for the DataHub-sink facts, `search` for the page-sink facts + — so accuracy climbs only because promotion pushed the fact into shared + knowledge. This isolates the delivery of *promoted* knowledge (the coupling + between the lifecycle and the enrichment layer), not an evaluator's own memory. + +The report is a **learning curve**: per checkpoint, the eval set's accuracy, a +per-trap-class breakdown (which lesson unlocked which class), and the +delivery-side **enrichment coverage** (the fraction of tool calls whose response +carried cross-enrichment, from the audit trail). Lesson order is the x-axis, run +foundational-first (units before net-revenue, then the calendar/freshness/tier/ +deprecation facts) so a multi-fact trap flips to correct only once every fact it +needs has landed. + +Grading is the deterministic S3 grading (numeric tolerance, entity alias); the +suite reuses one identity pool (a distinct teacher per lesson, fresh evaluators +per checkpoint), and a run refuses to start when the lessons plus per-checkpoint +evaluators exceed the pool. + ## Running From the repository root: @@ -251,6 +298,30 @@ drives the reviewer-side promotion, and abstains — validating handle threading the insight/changeset APIs, supersede, grading, and the metrics against the live platform with no API key and no model variance. +The cold-start suite (#963) boots the same `a3` arm but with the empty baseline +(no knowledge pages, undocumented DataHub), then teaches the curriculum: + +```bash +# Boot a3 with an empty enrichment layer: no knowledge pages, empty DataHub. +make bench-up BENCH_ARM=a3 BENCH_SEED_PAGES=0 +make bench-seed-datahub-empty # entities present, undocumented + +make bench-cold-start-smoke # scripted no-API-key loop validation +make bench-cold-start K=1 # real learning-curve run (needs a model) +make bench-cold-start LLM=claude-cli MODEL=sonnet K=1 # subscription run +make bench-cold-start-report # print the learning curve +``` + +The **scripted cold-start smoke** (`-llm scripted`) plays +`curriculum/scripted-cold-start-smoke.json` (generated): each lesson captures its +fact and the harness drives the real promotion; each eval task answers with its +computed ground truth. One run validates the whole teach → capture → promote → +eval loop, the insight/changeset APIs, deterministic grading, and the +learning-curve metrics against the live platform with no model. Its eval answers +are always correct (the smoke measures plumbing, not model behavior), so its +curve is flat-high; the climbing curve is a property of a real model run against +the empty baseline. + For the DataHub arms (`a1`, `a2`, `a3`), start a DataHub quickstart first (same external convention as e2e and load), then `make bench-seed-datahub` and `make bench-up BENCH_ARM=a2` (or `a1`/`a3`). Run each arm, then @@ -334,21 +405,25 @@ bench/ ├── seed/ generated seed artifacts (committed; bench-gen) ├── tasks/ generated task YAML + smoke script (committed) ├── protocols/ generated S5 lifecycle protocol YAML + smoke (committed) +├── curriculum/ generated cold-start curriculum YAML + smoke (committed) ├── judge/ versioned rubric + human-labeled calibration set └── internal/ - ├── gen/ dataset model, emitters, ground-truth computation, protocols + ├── gen/ dataset model, emitters, ground-truth computation, protocols, curriculum ├── task/ task schema, loader, task-set hash ├── protocol/ S5 lifecycle protocol schema, loader, protocol-set hash + ├── curriculum/ cold-start curriculum schema, loader, curriculum-set hash ├── llm/ adapter interface + anthropic + scripted ├── claudecli/ real Claude Code client path (claude -p) + stream parse ├── agent/ model-driven tool loop with budget ├── mcpc/ MCP session, handle mint, session_id threading - ├── auditapi/ admin audit API read-back + metrics + ├── auditapi/ admin audit API read-back + metrics (+ enrichment coverage) ├── lifecycleapi/ admin insights + changesets read-back, approve + apply drivers + ├── promote/ shared reviewer-promotion path (approve + apply_knowledge + verify), used by S5 and cold-start ├── grade/ deterministic graders (numeric, entity, execution-result) ├── judge/ LLM judge + calibration harness ├── pipeline/ task x k orchestration ├── lifecycle/ S5 protocol runner, stage graders, metrics, results model + ├── coldstart/ cold-start curriculum runner, learning-curve metrics, results model ├── report/ results model, aggregates, cross-arm comparison └── target/ endpoint + Bearer auth ``` diff --git a/bench/benchrun/main.go b/bench/benchrun/main.go index ea9fb916c..d1210968c 100644 --- a/bench/benchrun/main.go +++ b/bench/benchrun/main.go @@ -15,6 +15,7 @@ import ( "time" "github.com/txn2/mcp-data-platform/bench/internal/claudecli" + "github.com/txn2/mcp-data-platform/bench/internal/coldstart" "github.com/txn2/mcp-data-platform/bench/internal/judge" "github.com/txn2/mcp-data-platform/bench/internal/lifecycle" "github.com/txn2/mcp-data-platform/bench/internal/llm" @@ -54,6 +55,8 @@ type config struct { protocolsDir string baseline string merge string + coldStart bool + curriculumDir string } func main() { @@ -95,6 +98,8 @@ func parseFlags() config { flag.StringVar(&cfg.protocolsDir, "protocols", "protocols", "protocol YAML directory (with -lifecycle)") flag.StringVar(&cfg.baseline, "baseline", "", "committed baseline results JSON: after the run, gate on per-suite regression and exit nonzero if the candidate falls below it") flag.StringVar(&cfg.merge, "merge", "", "comma-separated per-pass lifecycle result JSONs (with -lifecycle): merge independent k=1 passes into one k=N result and exit") + flag.BoolVar(&cfg.coldStart, "cold-start", false, "run the cold-start knowledge-growth curriculum (issue #963) instead of the task suites") + flag.StringVar(&cfg.curriculumDir, "curriculum", "curriculum", "curriculum YAML directory (with -cold-start)") flag.Parse() return cfg } @@ -105,6 +110,9 @@ func run(cfg config) error { if handled, err := runReadOnly(cfg); handled { return err } + if cfg.coldStart { + return runColdStart(cfg) + } if cfg.lifecycle { return runLifecycle(cfg) } @@ -116,20 +124,8 @@ func run(cfg config) error { // live benchmark run. func runReadOnly(cfg config) (bool, error) { switch { - case cfg.summarize != "" && cfg.lifecycle: - res, err := lifecycle.LoadJSON(cfg.summarize) - if err != nil { - return true, err - } - fmt.Print(res.HumanSummary()) - return true, nil case cfg.summarize != "": - res, err := report.LoadJSON(cfg.summarize) - if err != nil { - return true, err - } - fmt.Print(res.HumanSummary()) - return true, nil + return true, runSummarize(cfg) case cfg.merge != "": // -merge only makes sense for lifecycle results; refuse rather than fall // through to a live (paid) benchmark run when -lifecycle is forgotten. @@ -145,6 +141,32 @@ func runReadOnly(cfg config) (bool, error) { return false, nil } +// runSummarize prints the human summary of an existing results JSON, choosing +// the result shape from the run-mode flags. +func runSummarize(cfg config) error { + switch { + case cfg.coldStart: + res, err := coldstart.LoadJSON(cfg.summarize) + if err != nil { + return err + } + fmt.Print(res.HumanSummary()) + case cfg.lifecycle: + res, err := lifecycle.LoadJSON(cfg.summarize) + if err != nil { + return err + } + fmt.Print(res.HumanSummary()) + default: + res, err := report.LoadJSON(cfg.summarize) + if err != nil { + return err + } + fmt.Print(res.HumanSummary()) + } + return nil +} + // runLifecycle executes the S5 lifecycle protocols and writes outputs. Like the // task benchmark, the results JSON is written even on failure so partial // evidence is never discarded. @@ -202,9 +224,99 @@ func runLifecycle(cfg config) error { return runErr } +// runColdStart executes the cold-start knowledge-growth curriculum and writes +// outputs. Like the other runs, the results JSON is flushed per checkpoint so an +// interruption never discards paid-for work. The -baseline gate scores the +// S1-S3 report shape, so it is refused here (cold-start produces a curve, not +// per-suite accuracy). +func runColdStart(cfg config) error { + if cfg.arm == "" { + return errors.New("-arm is required") + } + if cfg.baseline != "" { + return errors.New("-baseline is not supported with -cold-start (the regression gate scores S1-S3 task suites, not the learning curve)") + } + log := slog.New(slog.NewTextHandler(os.Stderr, nil)) + opts := coldstart.Options{ + Target: target.Target{BaseURL: cfg.url, Credential: cfg.credential}, + HTTPTimeout: cfg.httpTimeout, + Arm: cfg.arm, + K: cfg.k, + CurriculumDir: cfg.curriculumDir, + TasksDir: cfg.tasksDir, + TranscriptDir: transcriptDir(cfg.out), + LLMProvider: cfg.llmProvider, + GitCommit: cfg.gitCommit, + AuditTimeout: cfg.auditTimeout, + IdentityKeys: cfg.identityKeys, + OnCheckpoint: func(r *coldstart.Results) { + if err := r.WriteJSON(cfg.out); err != nil { + log.Warn("checkpoint write", "error", err) + } + }, + Log: log, + } + if cfg.llmProvider == claudeCLIProvider { + runner, version, err := buildClaudeRunner(cfg) + if err != nil { + return err + } + opts.ClaudeCLI, opts.ClientVersion = runner, version + } else { + factory, err := buildColdStartFactory(cfg) + if err != nil { + return err + } + opts.Factory = factory + } + res, runErr := coldstart.Run(context.Background(), opts) + if res != nil { + if err := writeAndSummarize(res, cfg.out); err != nil { + return err + } + } + return runErr +} + +// buildColdStartFactory constructs the per-episode adapter factory: a shared +// stateless model adapter, or a fresh scripted adapter per episode keyed by unit +// (lesson or task id) and stage. The scripted map has the same shape as the +// lifecycle smoke (unit -> stage -> steps), so it reuses the same loader. +func buildColdStartFactory(cfg config) (coldstart.AdapterFactory, error) { + switch cfg.llmProvider { + case "anthropic": + adapter, err := llm.NewAnthropic(cfg.model, cfg.maxTokens, cfg.llmTimeout) + if err != nil { + return nil, err + } + return func(string, string) (llm.Adapter, error) { return adapter, nil }, nil + case "scripted": + if cfg.script == "" { + return nil, errors.New("-script is required for -llm scripted") + } + script, err := llm.LoadLifecycleScript(cfg.script) + if err != nil { + return nil, err + } + return func(unitID, stage string) (llm.Adapter, error) { + stages, ok := script[unitID] + if !ok { + return nil, fmt.Errorf("cold-start script has no unit %s", unitID) + } + steps, ok := stages[stage] + if !ok { + return nil, fmt.Errorf("cold-start script has no %s/%s stage", unitID, stage) + } + return llm.NewScripted(steps), nil + }, nil + default: + return nil, fmt.Errorf("unknown -llm provider %q", cfg.llmProvider) + } +} + // summarizable is any run result that can persist itself and render a summary, -// satisfied by both report.Results (S1-S3) and lifecycle.Results (S5), so the -// two run paths share one write-and-print block. +// satisfied by report.Results (S1-S3), lifecycle.Results (S5), and +// coldstart.Results (#963), so the run paths share one write-and-print block. type summarizable interface { WriteJSON(path string) error HumanSummary() string diff --git a/bench/curriculum/cs-traps.yaml b/bench/curriculum/cs-traps.yaml new file mode 100644 index 000000000..97b0b2dae --- /dev/null +++ b/bench/curriculum/cs-traps.yaml @@ -0,0 +1,119 @@ +# Generated by bench/seedgen; the eval ground truth is the committed S3 task suite. +# Do not edit; regenerate with `make bench-gen`. +id: cs-traps +title: Cold-start knowledge growth over the S3 trap suite +eval_suite: s3 +lessons: + - id: cs-units-cents + title: Monetary columns are integer cents + trap_class: units_cents + fact: In memory.bench.orders the monetary columns amount and discount are stored as INTEGERS IN US CENTS, not dollars; divide by 100 to get USD. Any total computed without dividing by 100 is off by a factor of 100. + entity_urn: urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD) + sink: datahub + budget_tool_calls: 15 + teach: + prompt: For the analysis ahead I need you to remember a specific working definition and apply it in later sessions. In memory.bench.orders the monetary columns amount and discount are stored as INTEGERS IN US CENTS, not dollars; divide by 100 to get USD. Any total computed without dividing by 100 is off by a factor of 100. Please save this definition to memory now and link it to the dataset it concerns. + - id: cs-net-revenue + title: Net-revenue reporting policy + trap_class: net_revenue + fact: 'Company revenue reporting policy: revenue = amount - discount, over COMPLETED orders only (refunded and pending orders excluded). Amounts are in US cents. A gross figure that ignores discounts or includes non-completed orders is not policy revenue.' + entity_urn: urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD) + sink: knowledge_page + page: + slug: revenue-reporting-policy + title: Revenue Reporting Policy + body: |- + # Revenue Reporting Policy + + This is the authoritative definition of "revenue" for all bench warehouse reporting. + + **Revenue = amount - discount, over COMPLETED orders only.** Refunded and pending + orders are excluded entirely. Any figure that includes refunded orders or ignores + discounts is gross volume, not revenue, and must not be reported as revenue. + + Two mechanical rules when computing revenue from [orders](urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)): + + 1. The amount and discount columns are stored as integers in **US cents**. + Divide by 100 for USD. + 2. Filter on status = 'completed' before summing. + + The pre-aggregated daily_region_revenue index is **gross of discounts** and must + not be used for policy revenue figures. + budget_tool_calls: 15 + teach: + prompt: 'For the analysis ahead I need you to remember a specific working definition and apply it in later sessions. Company revenue reporting policy: revenue = amount - discount, over COMPLETED orders only (refunded and pending orders excluded). Amounts are in US cents. A gross figure that ignores discounts or includes non-completed orders is not policy revenue. Please save this definition to memory now and link it to the dataset it concerns.' + - id: cs-fiscal-calendar + title: Fiscal calendar boundaries + trap_class: fiscal_calendar + fact: 'The company fiscal year runs February 1 through January 31: fiscal year 2025 is 2025-02-01 through 2026-01-31. Fiscal figures must not be computed over the January-December calendar year.' + entity_urn: urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD) + sink: knowledge_page + page: + slug: fiscal-calendar-policy + title: Fiscal Calendar Policy + body: |- + # Fiscal Calendar Policy + + The company fiscal year does **not** align with the calendar year. **Fiscal year + N begins on February 1 of calendar year N and ends on January 31 of calendar + year N+1.** Fiscal year 2025 therefore runs **2025-02-01 through 2026-01-31**. + + When a question asks about a "fiscal year", a "fiscal quarter", or "FY" figures, + use these boundaries, not the calendar year: + + - Fiscal Q1: February – April + - Fiscal Q2: May – July + - Fiscal Q3: August – October + - Fiscal Q4: November – January + + A figure computed over the calendar year (January – December) is a + calendar-year figure and must not be reported as a fiscal-year figure. Revenue + inside a fiscal window still follows the Revenue Reporting Policy (net = + amount - discount over completed orders, amounts in US cents). + budget_tool_calls: 15 + teach: + prompt: 'For the analysis ahead I need you to remember a specific working definition and apply it in later sessions. The company fiscal year runs February 1 through January 31: fiscal year 2025 is 2025-02-01 through 2026-01-31. Fiscal figures must not be computed over the January-December calendar year. Please save this definition to memory now and link it to the dataset it concerns.' + - id: cs-freshness-cutoff + title: Daily index freshness cutoff + trap_class: freshness_cutoff + fact: 'Pre-aggregated daily revenue by region, derived from completed orders. Values are GROSS of discounts (USD), so this index must not be used for policy net-revenue figures; use memory.bench.orders per the Revenue Reporting Policy. FRESHNESS: this index is refreshed only through 2025-11-30; it has NO rows for dates after that cutoff, so any question about a period on or after 2025-12-01 must be answered from memory.bench.orders directly, not this index.' + entity_urn: urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.daily_region_revenue,PROD) + sink: datahub + budget_tool_calls: 15 + teach: + prompt: 'For the analysis ahead I need you to remember a specific working definition and apply it in later sessions. Pre-aggregated daily revenue by region, derived from completed orders. Values are GROSS of discounts (USD), so this index must not be used for policy net-revenue figures; use memory.bench.orders per the Revenue Reporting Policy. FRESHNESS: this index is refreshed only through 2025-11-30; it has NO rows for dates after that cutoff, so any question about a period on or after 2025-12-01 must be answered from memory.bench.orders directly, not this index. Please save this definition to memory now and link it to the dataset it concerns.' + - id: cs-tier-boundary + title: Key-account tier definition + trap_class: tier_boundary + fact: A 'key account' is any customer on the plus OR enterprise tier — a derived segment broader than the enterprise tier alone and not stored in any column. Counting only enterprise customers undercounts key accounts. + entity_urn: urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.customers,PROD) + sink: knowledge_page + page: + slug: customer-tier-definitions + title: Customer Tier Definitions + body: |- + # Customer Tier Definitions + + Customers carry a tier of `basic`, `plus`, or `enterprise` in the + customers table. Reporting uses one derived grouping that is **not** stored in + any column: + + - **Key account** — any customer on the `plus` OR `enterprise` tier. "Key + accounts" is the standard segment for account-level reporting; it is broader + than the top tier alone. A figure that counts only `enterprise` customers is + an enterprise-tier figure, not a key-account figure. + + When a question refers to "key accounts", include both the plus and enterprise + tiers. When it names a specific tier, use only that tier. + budget_tool_calls: 15 + teach: + prompt: For the analysis ahead I need you to remember a specific working definition and apply it in later sessions. A 'key account' is any customer on the plus OR enterprise tier — a derived segment broader than the enterprise tier alone and not stored in any column. Counting only enterprise customers undercounts key accounts. Please save this definition to memory now and link it to the dataset it concerns. + - id: cs-deprecated-table + title: legacy_orders is deprecated + trap_class: deprecated_table + fact: DEPRECATED order extract from the retired ingestion pipeline. Partial coverage, totals in dollars. Use memory.bench.orders for all order analysis. + entity_urn: urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.legacy_orders,PROD) + sink: datahub + budget_tool_calls: 15 + teach: + prompt: For the analysis ahead I need you to remember a specific working definition and apply it in later sessions. DEPRECATED order extract from the retired ingestion pipeline. Partial coverage, totals in dollars. Use memory.bench.orders for all order analysis. Please save this definition to memory now and link it to the dataset it concerns. diff --git a/bench/curriculum/scripted-cold-start-smoke.json b/bench/curriculum/scripted-cold-start-smoke.json new file mode 100644 index 000000000..d416c2660 --- /dev/null +++ b/bench/curriculum/scripted-cold-start-smoke.json @@ -0,0 +1,590 @@ +{ + "cs-deprecated-table": { + "teach": [ + { + "tool_calls": [ + { + "id": "", + "name": "memory_capture", + "args": { + "category": "business_context", + "content": "DEPRECATED order extract from the retired ingestion pipeline. Partial coverage, totals in dollars. Use memory.bench.orders for all order analysis.", + "entity_urns": [ + "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.legacy_orders,PROD)" + ], + "type": "schema_entity" + } + } + ] + }, + { + "final_text": "saved the definition" + } + ] + }, + "cs-fiscal-calendar": { + "teach": [ + { + "tool_calls": [ + { + "id": "", + "name": "memory_capture", + "args": { + "category": "business_context", + "content": "The company fiscal year runs February 1 through January 31: fiscal year 2025 is 2025-02-01 through 2026-01-31. Fiscal figures must not be computed over the January-December calendar year.", + "entity_urns": [ + "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)" + ], + "type": "schema_entity" + } + } + ] + }, + { + "final_text": "saved the definition" + } + ] + }, + "cs-freshness-cutoff": { + "teach": [ + { + "tool_calls": [ + { + "id": "", + "name": "memory_capture", + "args": { + "category": "business_context", + "content": "Pre-aggregated daily revenue by region, derived from completed orders. Values are GROSS of discounts (USD), so this index must not be used for policy net-revenue figures; use memory.bench.orders per the Revenue Reporting Policy. FRESHNESS: this index is refreshed only through 2025-11-30; it has NO rows for dates after that cutoff, so any question about a period on or after 2025-12-01 must be answered from memory.bench.orders directly, not this index.", + "entity_urns": [ + "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.daily_region_revenue,PROD)" + ], + "type": "schema_entity" + } + } + ] + }, + { + "final_text": "saved the definition" + } + ] + }, + "cs-net-revenue": { + "teach": [ + { + "tool_calls": [ + { + "id": "", + "name": "memory_capture", + "args": { + "category": "business_context", + "content": "Company revenue reporting policy: revenue = amount - discount, over COMPLETED orders only (refunded and pending orders excluded). Amounts are in US cents. A gross figure that ignores discounts or includes non-completed orders is not policy revenue.", + "entity_urns": [ + "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)" + ], + "type": "schema_entity" + } + } + ] + }, + { + "final_text": "saved the definition" + } + ] + }, + "cs-tier-boundary": { + "teach": [ + { + "tool_calls": [ + { + "id": "", + "name": "memory_capture", + "args": { + "category": "business_context", + "content": "A 'key account' is any customer on the plus OR enterprise tier — a derived segment broader than the enterprise tier alone and not stored in any column. Counting only enterprise customers undercounts key accounts.", + "entity_urns": [ + "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.customers,PROD)" + ], + "type": "schema_entity" + } + } + ] + }, + { + "final_text": "saved the definition" + } + ] + }, + "cs-units-cents": { + "teach": [ + { + "tool_calls": [ + { + "id": "", + "name": "memory_capture", + "args": { + "category": "business_context", + "content": "In memory.bench.orders the monetary columns amount and discount are stored as INTEGERS IN US CENTS, not dollars; divide by 100 to get USD. Any total computed without dividing by 100 is off by a factor of 100.", + "entity_urns": [ + "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)" + ], + "type": "schema_entity" + } + } + ] + }, + { + "final_text": "saved the definition" + } + ] + }, + "s3-deprecated-completed-usd": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1430603.83" + } + ] + }, + "s3-deprecated-order-count": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1200.00" + } + ] + }, + "s3-fiscal-2025-count": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 873.00" + } + ] + }, + "s3-fiscal-2025-east": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 245825.11" + } + ] + }, + "s3-fiscal-2025-net": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1187140.04" + } + ] + }, + "s3-fiscal-q1-net": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 317090.52" + } + ] + }, + "s3-fiscal-q4-net": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 238592.95" + } + ] + }, + "s3-fresh-dec-gross": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 133777.39" + } + ] + }, + "s3-fresh-fullyear-gross": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1430603.83" + } + ] + }, + "s3-fresh-novdec-gross": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 266073.26" + } + ] + }, + "s3-fresh-q4-gross": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 403518.09" + } + ] + }, + "s3-net-east-2025": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 257733.53" + } + ] + }, + "s3-net-east-march": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 38952.32" + } + ] + }, + "s3-net-top-region": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: West" + } + ] + }, + "s3-net-total-2025": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1286561.19" + } + ] + }, + "s3-net-west-2025": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 466889.38" + } + ] + }, + "s3-tier-key-avg": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1515.59" + } + ] + }, + "s3-tier-key-completed": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 527.00" + } + ] + }, + "s3-tier-key-count": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 43.00" + } + ] + }, + "s3-tier-key-east": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 9.00" + } + ] + }, + "s3-tier-key-orders": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 648.00" + } + ] + }, + "s3-units-avg-enterprise": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1614.57" + } + ] + }, + "s3-units-completed-gross": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1430603.83" + } + ] + }, + "s3-units-q1-total": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 446819.28" + } + ] + }, + "s3-units-total-all": { + "eval": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "recall the saved definition" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: 1839443.21" + } + ] + } +} diff --git a/bench/internal/auditapi/auditapi.go b/bench/internal/auditapi/auditapi.go index bd0930f05..9c1db43fe 100644 --- a/bench/internal/auditapi/auditapi.go +++ b/bench/internal/auditapi/auditapi.go @@ -181,6 +181,19 @@ type Metrics struct { EnrichmentTokensDedup int `json:"enrichment_tokens_dedup"` } +// EnrichmentCoverage is the fraction of audited tool calls whose response +// carried cross-enrichment. It is the delivery-side signal for the cold-start +// suite (issue #963): as promoted knowledge fills the catalog, more tool +// responses carry enrichment context, so coverage climbs alongside accuracy. It +// is zero when the session audited no calls (an empty denominator is not +// coverage of nothing). +func (m Metrics) EnrichmentCoverage() float64 { + if m.AuditedCalls == 0 { + return 0 + } + return float64(m.EnrichedCalls) / float64(m.AuditedCalls) +} + // Summarize folds a session's events into Metrics. func Summarize(events []Event) Metrics { var m Metrics diff --git a/bench/internal/auditapi/auditapi_test.go b/bench/internal/auditapi/auditapi_test.go index 0e9fd4844..f7877895b 100644 --- a/bench/internal/auditapi/auditapi_test.go +++ b/bench/internal/auditapi/auditapi_test.go @@ -172,4 +172,17 @@ func TestSummarize(t *testing.T) { if m != want { t.Errorf("Summarize = %+v, want %+v", m, want) } + if got := m.EnrichmentCoverage(); got != 2.0/3.0 { + t.Errorf("EnrichmentCoverage = %v, want %v", got, 2.0/3.0) + } +} + +func TestEnrichmentCoverageEmpty(t *testing.T) { + if got := (Metrics{}).EnrichmentCoverage(); got != 0 { + t.Errorf("coverage of no calls = %v, want 0", got) + } + // All calls enriched -> full coverage. + if got := (Metrics{AuditedCalls: 4, EnrichedCalls: 4}).EnrichmentCoverage(); got != 1 { + t.Errorf("coverage = %v, want 1", got) + } } diff --git a/bench/internal/coldstart/episode.go b/bench/internal/coldstart/episode.go new file mode 100644 index 000000000..8d8d0ade6 --- /dev/null +++ b/bench/internal/coldstart/episode.go @@ -0,0 +1,288 @@ +package coldstart + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/txn2/mcp-data-platform/bench/internal/agent" + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" + "github.com/txn2/mcp-data-platform/bench/internal/claudecli" + "github.com/txn2/mcp-data-platform/bench/internal/grade" + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/mcpc" + "github.com/txn2/mcp-data-platform/bench/internal/pool" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +// searchToolName marks the discovery tool for the search-called signal. +const searchToolName = "search" + +// Stage names, one per episode kind. +const ( + StageTeach = "teach" + StageEval = "eval" +) + +// teachScaffold instructs the agent to capture the stated fact and link it to +// the dataset it concerns, so the runner can verify capture and entity linkage. +// Cold-start runs only on a3, so this is not an ablation variable. +const teachScaffold = `You are a data analyst agent connected to a data platform over MCP. You work across many separate sessions and can save knowledge for later sessions to use. +Rules: +- THIS SESSION IS FOR RECORDING KNOWLEDGE. Save the definition stated below using the memory tools so a future session can recall it, and link it to the dataset it concerns. Then confirm in one line what you saved.` + +// evalScaffold frames an evaluation session. The evaluator was never taught the +// facts, so its only knowledge source is what the platform surfaces — search +// results, catalog descriptions, and knowledge pages — which is exactly the +// promoted-knowledge delivery channel the curve measures. +const evalScaffold = `You are a data analyst agent connected to a data platform over MCP. +Rules: +- Ground every answer in tool results and in the knowledge the platform surfaces (search results, catalog descriptions, knowledge pages); do not answer from prior knowledge about any specific dataset. +- Use the search tool to discover context and data before querying. +- When you have the answer, end your reply with a single line: "FINAL ANSWER: ".` + +// Per-grading-kind answer format rules, matching the deterministic graders' +// parsing convention (shared with the S1-S3 pipeline and S5 lifecycle). +const ( + numericFormat = `- The FINAL ANSWER line must contain exactly one number (USD unless the question states otherwise), for example "FINAL ANSWER: 12345.67".` + entityFormat = `- The FINAL ANSWER line must name the single best answer: a fully qualified table name (catalog.schema.table) for dataset questions, or the exact name requested.` +) + +// evalSystem builds the evaluator's system prompt for a grading kind. +func evalSystem(kind string) string { + format := entityFormat + if kind == task.GradeNumeric { + format = numericFormat + } + return evalScaffold + "\n" + format +} + +// gradeEval scores an eval answer with the deterministic graders, reusing the +// task graders so a cold-start eval is graded exactly as an S1-S3 question. +// Only numeric and entity reach here (exec_sql is rejected at load time). +func gradeEval(finalAnswer string, g task.Grading) bool { + final := grade.ExtractFinal(finalAnswer) + switch g.Kind { + case task.GradeNumeric: + if g.Value == nil { + return false + } + _, _, correct := grade.Numeric(final, *g.Value, g.AbsTolerance) + return correct + case task.GradeEntity: + _, correct := grade.Entity(final, g.Aliases, g.WrongAliases) + return correct + default: + return false + } +} + +// episodeSpec is one session's parameters. +type episodeSpec struct { + stage string + unitID string // lesson id (teach) or task id (eval), keys the adapter + transcript + seq int // pool identity sequence number + prompt string + system string + budget int +} + +// episodeResult is one session's raw outcome, mapped by callers into a lesson +// EpisodeRecord or an EvalAttempt. +type episodeResult struct { + email string + sessionID string + toolCalls int + toolErrors int + searchCalled bool + wallMS int64 + usage llm.Usage + audit auditapi.Metrics + finalAnswer string + err string +} + +// runEpisode drives one fresh MCP session end to end: authenticate as the pool +// identity, mint the handle, run the agent loop against the a3 tool surface, and +// read the audit trail back (best effort — lesson state comes from the knowledge +// API and eval correctness from grading, not audit). A harness failure lands in +// the result's err; a graded outcome does not. +func (e *runEnv) runEpisode(ctx context.Context, spec episodeSpec) episodeResult { + if e.opts.ClaudeCLI != nil { + return e.runClaudeCLIEpisode(ctx, spec) + } + res := episodeResult{email: pool.Email(spec.seq)} + client := e.attemptClient(spec.seq) + + session, err := client.Connect(ctx) + if err != nil { + res.err = fmt.Sprintf("connect: %v", err) + return res + } + defer func() { _ = session.Close() }() + + info, err := mcpc.Mint(ctx, session) + if err != nil { + res.err = fmt.Sprintf("mint session handle: %v", err) + return res + } + res.sessionID = info.Handle + e.recordPlatformVersion(info.PlatformVersion) + + tools, err := mcpc.ListTools(ctx, session) + if err != nil { + res.err = fmt.Sprintf("list tools: %v", err) + return res + } + + adapter, err := e.opts.Factory(spec.unitID, spec.stage) + if err != nil { + res.err = fmt.Sprintf("build adapter: %v", err) + return res + } + e.recordModel(adapter.Model()) + + audited, indeterminate := 0, 0 + exec := func(ctx context.Context, name string, args map[string]any) llm.ToolResult { + if name == searchToolName { + res.searchCalled = true + } + r := mcpc.Call(ctx, session, name, args, info.Handle) + if r.TransportErr != nil { + indeterminate++ + return llm.ToolResult{Text: "transport error: " + r.TransportErr.Error(), IsError: true} + } + if !preAuditRefusal(r.ErrorCode) { + audited++ + } + return llm.ToolResult{Text: r.Text, IsError: r.ToolErr} + } + + start := time.Now() + result, runErr := agent.Run(ctx, adapter, agent.Config{ + System: spec.system, Prompt: spec.prompt, Tools: tools, Budget: spec.budget, + }, exec) + res.wallMS = time.Since(start).Milliseconds() + res.toolCalls = result.ToolCalls + res.toolErrors = result.ToolErrors + res.usage = result.Usage + res.finalAnswer = result.FinalAnswer + e.writeTranscript(spec, result.Transcript) + if runErr != nil { + res.err = fmt.Sprintf("agent loop: %v", runErr) + return res + } + res.audit = e.readAudit(ctx, info.Handle, audited, audited+indeterminate) + return res +} + +// runClaudeCLIEpisode drives one episode through a real `claude -p` client. +// Claude Code authenticates as the pool identity, mints and threads its own +// handle, and drives the tools; the harness reconstructs the transcript and +// reads audit back best effort by the threaded handle. +func (e *runEnv) runClaudeCLIEpisode(ctx context.Context, spec episodeSpec) episodeResult { + res := episodeResult{email: pool.Email(spec.seq)} + e.recordModel(e.opts.ClaudeCLI.Model()) + + start := time.Now() + cres, err := e.opts.ClaudeCLI.Run(ctx, claudecli.Request{ + Endpoint: e.opts.Target.BaseURL, + Credential: pool.Credential(e.opts.Target.Credential, spec.seq, e.opts.IdentityKeys), + System: spec.system, + Prompt: spec.prompt, + }) + res.wallMS = time.Since(start).Milliseconds() + if err != nil { + res.err = fmt.Sprintf("claude-cli: %v", err) + return res + } + res.sessionID = cres.Handle + res.toolCalls = cres.MCPCalls + res.toolErrors = cres.ToolErrors + res.searchCalled = cres.SearchCalled + res.usage = cres.Usage + res.finalAnswer = cres.FinalText + e.recordPlatformVersion(cres.PlatformVersion) + e.writeClaudeTranscript(spec, cres.Transcript) + + if cres.IsError { + res.err = fmt.Sprintf("claude-cli result error (subtype %q): %.300s", cres.Subtype, cres.FinalText) + return res + } + if !cres.ServerConnected { + res.err = fmt.Sprintf("bench MCP server did not connect (status %q)", cres.ServerStatus) + return res + } + if cres.Handle != "" { + res.audit = e.readAudit(ctx, cres.Handle, cres.SuccessfulMCPCalls, cres.MCPCalls) + } + return res +} + +// readAudit reads the session's audit trail back best effort. A missing row does +// not fail an episode: lesson state comes from the knowledge API and eval +// correctness from grading; audit only enriches the efficiency and coverage +// picture. A read failure yields zero metrics and is logged. +func (e *runEnv) readAudit(ctx context.Context, handle string, minAudited, maxAudited int) auditapi.Metrics { + events, err := e.audit.WaitForSession(ctx, handle, minAudited, maxAudited, e.opts.AuditTimeout) + if err != nil { + e.log.Warn("cold-start audit read-back", "handle", handle, "error", err) + return auditapi.Metrics{} + } + return auditapi.Summarize(events) +} + +// writeTranscript persists a loop episode's transcript for manual audit. +func (e *runEnv) writeTranscript(spec episodeSpec, msgs []llm.Message) { + e.persistTranscript(spec, msgs) +} + +// writeClaudeTranscript persists a claude-cli episode's reconstructed transcript. +func (e *runEnv) writeClaudeTranscript(spec episodeSpec, msgs []llm.Message) { + e.persistTranscript(spec, msgs) +} + +// persistTranscript writes one episode's transcript to the transcript dir. +func (e *runEnv) persistTranscript(spec episodeSpec, msgs []llm.Message) { + if e.opts.TranscriptDir == "" { + return + } + if err := os.MkdirAll(e.opts.TranscriptDir, 0o750); err != nil { + e.log.Warn("transcript dir", "error", err) + return + } + path := filepath.Join(e.opts.TranscriptDir, + fmt.Sprintf("%s-%s-%s.json", spec.stage, spec.unitID, pool.Email(spec.seq))) + payload := transcriptFile{Stage: spec.stage, UnitID: spec.unitID, Email: pool.Email(spec.seq), Transcript: msgs} + raw, err := json.MarshalIndent(payload, "", " ") + if err != nil { + e.log.Warn("marshal transcript", "error", err) + return + } + if err := os.WriteFile(path, raw, 0o600); err != nil { + e.log.Warn("write transcript", "error", err) + } +} + +// transcriptFile is the on-disk transcript layout. +type transcriptFile struct { + Stage string `json:"stage"` + UnitID string `json:"unit_id"` + Email string `json:"email"` + Transcript []llm.Message `json:"transcript"` +} + +// preAuditRefusal reports whether a structured error code marks a platform +// refusal issued outer to the audit middleware (so it leaves no audit row). +// Mirrors the S1-S3 pipeline and S5 lifecycle classification. +func preAuditRefusal(code string) bool { + switch code { + case "unauthenticated", "unauthorized", "session_required", "session_expired", + "search_required", "setup_required", "rate_limited": + return true + } + return false +} diff --git a/bench/internal/coldstart/report.go b/bench/internal/coldstart/report.go new file mode 100644 index 000000000..34460d31b --- /dev/null +++ b/bench/internal/coldstart/report.go @@ -0,0 +1,380 @@ +// Package coldstart runs the cold-start knowledge-growth suite (issue #963) and +// scores it. This file defines the results model: a manifest pinning the run, a +// record per lesson (teach + promote outcome), and a learning curve — one +// checkpoint per point on the accumulated-knowledge axis, each carrying the +// fixed eval set's accuracy, a per-trap-class breakdown, and the delivery-side +// enrichment coverage. The curve is the deliverable: accuracy and coverage as a +// function of how much promoted knowledge the enrichment layer holds. +package coldstart + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" +) + +// Manifest pins the run so results are attributable and reproducible. +type Manifest struct { + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + GitCommit string `json:"git_commit"` + PlatformVersion string `json:"platform_version"` + Target string `json:"target"` + Arm string `json:"arm"` + LLMProvider string `json:"llm_provider"` + // ClientVersion records the external client path (claude-cli: the + // `claude --version` string), empty for in-process adapters. + ClientVersion string `json:"client_version,omitempty"` + Model string `json:"model"` + Seed int64 `json:"seed"` + CurriculumID string `json:"curriculum_id"` + CurriculumHash string `json:"curriculum_hash"` + EvalSuite string `json:"eval_suite"` + TaskSetHash string `json:"task_set_hash"` + // K is the number of fresh evaluator identities per checkpoint; each answers + // the whole eval set, so a checkpoint's accuracy averages over K x eval-tasks. + K int `json:"k"` +} + +// EpisodeRecord is one teach session's telemetry (the lesson's capture episode). +type EpisodeRecord struct { + Email string `json:"email"` + SessionID string `json:"session_id,omitempty"` + ToolCalls int `json:"tool_calls"` + ToolErrors int `json:"tool_errors"` + WallMS int64 `json:"wall_ms"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + CacheReadTokens int64 `json:"cache_read_tokens,omitempty"` + CacheCreationTokens int64 `json:"cache_creation_tokens,omitempty"` + Audit auditapi.Metrics `json:"audit"` + Error string `json:"error,omitempty"` +} + +// LessonRecord captures one lesson's teach-and-promote outcome. Captured and +// Promoted are pointers so a lesson never reached (an earlier harness abort) +// is distinguishable from one that failed the transition. +type LessonRecord struct { + LessonID string `json:"lesson_id"` + Title string `json:"title"` + TrapClass string `json:"trap_class"` + Sink string `json:"sink"` + InsightID string `json:"insight_id,omitempty"` + + Captured *bool `json:"captured,omitempty"` // insight recorded and entity-linked + Promoted *bool `json:"promoted,omitempty"` // applied + changeset links the insight + + Episode EpisodeRecord `json:"episode"` + Error string `json:"error,omitempty"` // harness failure in the teach/promote +} + +// EvalAttempt is one evaluator answering one eval task at a checkpoint. Graded +// is false for a harness-level failure (connect, adapter, audit read-back), +// which is excluded from accuracy and reported separately, mirroring the S1-S3 +// and S5 pipelines. +type EvalAttempt struct { + TaskID string `json:"task_id"` + TrapClasses []string `json:"trap_classes,omitempty"` + Email string `json:"email"` + SessionID string `json:"session_id,omitempty"` + Repeat int `json:"repeat"` + Graded bool `json:"graded"` + Correct bool `json:"correct"` + FinalAnswer string `json:"final_answer,omitempty"` + WallMS int64 `json:"wall_ms"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + CacheReadTokens int64 `json:"cache_read_tokens,omitempty"` + CacheCreationTokens int64 `json:"cache_creation_tokens,omitempty"` + Audit auditapi.Metrics `json:"audit"` + Error string `json:"error,omitempty"` +} + +// ClassScore is one trap class's accuracy at a checkpoint. +type ClassScore struct { + Correct int `json:"correct"` + Graded int `json:"graded"` + Accuracy float64 `json:"accuracy"` +} + +// Checkpoint is one point on the learning curve: the eval set's outcome after a +// given number of lessons have been promoted. Index 0 is the empty baseline. +type Checkpoint struct { + Index int `json:"index"` + LessonID string `json:"lesson_id,omitempty"` // lesson promoted to reach this point + LessonTitle string `json:"lesson_title,omitempty"` // + TrapClass string `json:"trap_class,omitempty"` // that lesson's trap class + + // PromotedSoFar is the count of lessons successfully promoted at or before + // this checkpoint — the accumulated-knowledge coordinate on the x-axis. + PromotedSoFar int `json:"promoted_so_far"` + + EvalGraded int `json:"eval_graded"` + EvalCorrect int `json:"eval_correct"` + Accuracy float64 `json:"accuracy"` + + ByTrapClass map[string]ClassScore `json:"by_trap_class,omitempty"` + + AuditedCalls int `json:"audited_calls"` + EnrichedCalls int `json:"enriched_calls"` + EnrichmentCoverage float64 `json:"enrichment_coverage"` + + HarnessFailures int `json:"harness_failures"` + Attempts []EvalAttempt `json:"attempts"` +} + +// aggregate folds this checkpoint's attempts into its scores. Harness failures +// are excluded from the accuracy denominators and counted separately. +func (c *Checkpoint) aggregate() { + c.EvalGraded, c.EvalCorrect, c.HarnessFailures = 0, 0, 0 + c.AuditedCalls, c.EnrichedCalls = 0, 0 + byClass := map[string]ClassScore{} + for _, a := range c.Attempts { + if !a.Graded { + c.HarnessFailures++ + continue + } + c.EvalGraded++ + c.AuditedCalls += a.Audit.AuditedCalls + c.EnrichedCalls += a.Audit.EnrichedCalls + if a.Correct { + c.EvalCorrect++ + } + for _, class := range a.TrapClasses { + s := byClass[class] + s.Graded++ + if a.Correct { + s.Correct++ + } + byClass[class] = s + } + } + c.Accuracy = ratio(c.EvalCorrect, c.EvalGraded) + c.EnrichmentCoverage = ratio(c.EnrichedCalls, c.AuditedCalls) + for class, s := range byClass { + s.Accuracy = ratio(s.Correct, s.Graded) + byClass[class] = s + } + if len(byClass) > 0 { + c.ByTrapClass = byClass + } +} + +// Metrics is the cold-start scorecard: the curve's endpoints and totals. +type Metrics struct { + Lessons int `json:"lessons"` + LessonsCaptured int `json:"lessons_captured"` + LessonsPromoted int `json:"lessons_promoted"` + Checkpoints int `json:"checkpoints"` + EvalTasks int `json:"eval_tasks"` + HarnessFailures int `json:"harness_failures"` + + // Token totals across every episode and attempt, so a run self-reports its + // cost basis. The cache split lets a cached run's cost be computed from + // committed data (cache reads bill far below fresh input). + TotalInputTokens int64 `json:"total_input_tokens"` + TotalOutputTokens int64 `json:"total_output_tokens"` + TotalCacheReadTokens int64 `json:"total_cache_read_tokens"` + TotalCacheCreationTokens int64 `json:"total_cache_creation_tokens"` + + // Curve endpoints: the empty baseline vs the fully-taught ceiling. + BaselineAccuracy float64 `json:"baseline_accuracy"` + FinalAccuracy float64 `json:"final_accuracy"` + AccuracyLift float64 `json:"accuracy_lift"` + BaselineCoverage float64 `json:"baseline_coverage"` + FinalCoverage float64 `json:"final_coverage"` +} + +// Results is the full cold-start run output. +type Results struct { + Manifest Manifest `json:"manifest"` + Lessons []LessonRecord `json:"lessons"` + Checkpoints []Checkpoint `json:"checkpoints"` + Metrics Metrics `json:"metrics"` +} + +// Aggregate computes the checkpoint scores and the top-level metrics. +func (res *Results) Aggregate() { + m := Metrics{Lessons: len(res.Lessons), Checkpoints: len(res.Checkpoints)} + for _, l := range res.Lessons { + if boolTrue(l.Captured) { + m.LessonsCaptured++ + } + if boolTrue(l.Promoted) { + m.LessonsPromoted++ + } + if l.Error != "" { + m.HarnessFailures++ + } + m.TotalInputTokens += l.Episode.InputTokens + m.TotalOutputTokens += l.Episode.OutputTokens + m.TotalCacheReadTokens += l.Episode.CacheReadTokens + m.TotalCacheCreationTokens += l.Episode.CacheCreationTokens + } + for i := range res.Checkpoints { + res.Checkpoints[i].aggregate() + m.HarnessFailures += res.Checkpoints[i].HarnessFailures + for _, a := range res.Checkpoints[i].Attempts { + m.TotalInputTokens += a.InputTokens + m.TotalOutputTokens += a.OutputTokens + m.TotalCacheReadTokens += a.CacheReadTokens + m.TotalCacheCreationTokens += a.CacheCreationTokens + } + } + if len(res.Checkpoints) > 0 { + m.EvalTasks = distinctTasks(res.Checkpoints[0].Attempts) + first, last := res.Checkpoints[0], res.Checkpoints[len(res.Checkpoints)-1] + m.BaselineAccuracy, m.FinalAccuracy = first.Accuracy, last.Accuracy + m.AccuracyLift = last.Accuracy - first.Accuracy + m.BaselineCoverage, m.FinalCoverage = first.EnrichmentCoverage, last.EnrichmentCoverage + } + res.Metrics = m +} + +// distinctTasks counts the unique task IDs in a checkpoint's attempts (the eval +// set size, independent of k). +func distinctTasks(attempts []EvalAttempt) int { + seen := map[string]bool{} + for _, a := range attempts { + seen[a.TaskID] = true + } + return len(seen) +} + +// ratio is num/den, or 0 for an empty denominator. +func ratio(num, den int) float64 { + if den == 0 { + return 0 + } + return float64(num) / float64(den) +} + +// boolTrue reports whether a nil-able bool is set and true. +func boolTrue(b *bool) bool { return b != nil && *b } + +// WriteJSON persists the results. +func (res *Results) WriteJSON(path string) error { + raw, err := json.MarshalIndent(res, "", " ") + if err != nil { + return fmt.Errorf("marshal results: %w", err) + } + if err := os.WriteFile(path, raw, 0o600); err != nil { + return fmt.Errorf("write results: %w", err) + } + return nil +} + +// LoadJSON reads results written by WriteJSON (for -summarize). +func LoadJSON(path string) (*Results, error) { + raw, err := os.ReadFile(path) // #nosec G304 -- operator-supplied results path + if err != nil { + return nil, fmt.Errorf("read results: %w", err) + } + var res Results + if err := json.Unmarshal(raw, &res); err != nil { + return nil, fmt.Errorf("parse results: %w", err) + } + return &res, nil +} + +// HumanSummary renders the learning curve for a terminal. +func (res *Results) HumanSummary() string { + var b strings.Builder + m := res.Manifest + provider := m.LLMProvider + if m.ClientVersion != "" { + provider = fmt.Sprintf("%s via %s", m.LLMProvider, m.ClientVersion) + } + fmt.Fprintf(&b, "bench cold-start (%s): arm=%s model=%s (%s) k=%d\n", m.CurriculumID, m.Arm, m.Model, provider, m.K) + fmt.Fprintf(&b, " platform %s @ %s | commit %s | seed %d | curriculum %s | eval %s (%s)\n", + m.PlatformVersion, m.Target, short(m.GitCommit), m.Seed, short(m.CurriculumHash), m.EvalSuite, short(m.TaskSetHash)) + fmt.Fprintf(&b, " %s .. %s\n\n", m.StartedAt.Format(time.RFC3339), m.FinishedAt.Format(time.RFC3339)) + + mt := res.Metrics + fmt.Fprintf(&b, "lessons %d (captured %d, promoted %d) eval tasks %d checkpoints %d harness failures %d\n", + mt.Lessons, mt.LessonsCaptured, mt.LessonsPromoted, mt.EvalTasks, mt.Checkpoints, mt.HarnessFailures) + fmt.Fprintf(&b, "tokens: input %d output %d cache read %d cache write %d (apply current model pricing for cost)\n\n", + mt.TotalInputTokens, mt.TotalOutputTokens, mt.TotalCacheReadTokens, mt.TotalCacheCreationTokens) + + b.WriteString("learning curve (accuracy and enrichment coverage vs promoted knowledge):\n") + fmt.Fprintf(&b, " %-4s %-22s %-9s %-8s %-9s\n", "idx", "lesson promoted", "promoted", "accuracy", "coverage") + for _, c := range res.Checkpoints { + label := "(empty baseline)" + if c.LessonID != "" { + label = c.LessonID + } + fmt.Fprintf(&b, " %-4d %-22s %-8d %6.1f%% %6.1f%%\n", + c.Index, truncate(label, 22), c.PromotedSoFar, c.Accuracy*100, c.EnrichmentCoverage*100) + } + fmt.Fprintf(&b, "\naccuracy %.1f%% -> %.1f%% (lift %+.1f pts) coverage %.1f%% -> %.1f%%\n", + mt.BaselineAccuracy*100, mt.FinalAccuracy*100, mt.AccuracyLift*100, mt.BaselineCoverage*100, mt.FinalCoverage*100) + + res.writeTrapClassCurve(&b) + res.writeLessonFailures(&b) + return b.String() +} + +// writeTrapClassCurve renders the baseline vs final accuracy for each trap class +// so a reader sees which lesson unlocked which class. +func (res *Results) writeTrapClassCurve(b *strings.Builder) { + if len(res.Checkpoints) < 2 { + return + } + first, last := res.Checkpoints[0], res.Checkpoints[len(res.Checkpoints)-1] + classes := make([]string, 0, len(last.ByTrapClass)) + for class := range last.ByTrapClass { + classes = append(classes, class) + } + sort.Strings(classes) + if len(classes) == 0 { + return + } + b.WriteString("\nper-trap-class accuracy (baseline -> final):\n") + for _, class := range classes { + fmt.Fprintf(b, " %-18s %5.1f%% -> %5.1f%%\n", class, first.ByTrapClass[class].Accuracy*100, last.ByTrapClass[class].Accuracy*100) + } +} + +// writeLessonFailures lists lessons whose teach or promote did not complete. +func (res *Results) writeLessonFailures(b *strings.Builder) { + var lines []string + for _, l := range res.Lessons { + switch { + case l.Error != "": + lines = append(lines, fmt.Sprintf(" %s: %s", l.LessonID, l.Error)) + case !boolTrue(l.Captured): + lines = append(lines, fmt.Sprintf(" %s: not captured", l.LessonID)) + case !boolTrue(l.Promoted): + lines = append(lines, fmt.Sprintf(" %s: captured but not promoted", l.LessonID)) + } + } + if len(lines) == 0 { + return + } + b.WriteString("\nlesson gaps (knowledge not delivered, so its class stays flat):\n") + b.WriteString(strings.Join(lines, "\n") + "\n") +} + +// short truncates a hash for display. +func short(s string) string { + if s == "" { + return "unknown" + } + if len(s) > 12 { + return s[:12] + } + return s +} + +// truncate caps a label to n runes for the fixed-width curve table. +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n-1] + "…" +} diff --git a/bench/internal/coldstart/report_test.go b/bench/internal/coldstart/report_test.go new file mode 100644 index 000000000..9b3c1d041 --- /dev/null +++ b/bench/internal/coldstart/report_test.go @@ -0,0 +1,134 @@ +package coldstart + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" +) + +// sampleResults builds a two-lesson curve: a baseline where nothing is +// answerable and a final checkpoint where the units class is unlocked. +func sampleResults() *Results { + audited := auditapi.Metrics{AuditedCalls: 2, EnrichedCalls: 0} + enriched := auditapi.Metrics{AuditedCalls: 2, EnrichedCalls: 2} + return &Results{ + Manifest: Manifest{CurriculumID: "cs-traps", Arm: "a3", EvalSuite: "s3", K: 1}, + Lessons: []LessonRecord{ + {LessonID: "cs-units", TrapClass: "units_cents", Sink: "datahub", + Captured: new(true), Promoted: new(true), + Episode: EpisodeRecord{InputTokens: 100, OutputTokens: 20}}, + {LessonID: "cs-net", TrapClass: "net_revenue", Sink: "knowledge_page", + Captured: new(true), Promoted: new(false), + Episode: EpisodeRecord{InputTokens: 100, OutputTokens: 20}}, + }, + Checkpoints: []Checkpoint{ + {Index: 0, PromotedSoFar: 0, Attempts: []EvalAttempt{ + {TaskID: "s3-units-a", TrapClasses: []string{"units_cents"}, Graded: true, Correct: false, Audit: audited, InputTokens: 50, OutputTokens: 10}, + {TaskID: "s3-net-a", TrapClasses: []string{"net_revenue"}, Graded: true, Correct: false, Audit: audited}, + }}, + {Index: 1, LessonID: "cs-units", TrapClass: "units_cents", PromotedSoFar: 1, Attempts: []EvalAttempt{ + {TaskID: "s3-units-a", TrapClasses: []string{"units_cents"}, Graded: true, Correct: true, Audit: enriched}, + {TaskID: "s3-net-a", TrapClasses: []string{"net_revenue"}, Graded: true, Correct: false, Audit: enriched}, + {TaskID: "s3-x", Graded: false, Error: "connect: boom"}, // harness failure excluded + }}, + }, + } +} + +func TestAggregateCurveAndMetrics(t *testing.T) { + res := sampleResults() + res.Aggregate() + + base, final := res.Checkpoints[0], res.Checkpoints[1] + if base.Accuracy != 0 { + t.Errorf("baseline accuracy = %v, want 0", base.Accuracy) + } + if final.EvalGraded != 2 || final.EvalCorrect != 1 || final.Accuracy != 0.5 { + t.Errorf("final checkpoint = graded %d correct %d acc %v, want 2/1/0.5", final.EvalGraded, final.EvalCorrect, final.Accuracy) + } + if final.HarnessFailures != 1 { + t.Errorf("final harness failures = %d, want 1", final.HarnessFailures) + } + // Coverage: baseline 0/4 enriched, final 4/4. + if base.EnrichmentCoverage != 0 { + t.Errorf("baseline coverage = %v, want 0", base.EnrichmentCoverage) + } + if final.EnrichmentCoverage != 1 { + t.Errorf("final coverage = %v, want 1", final.EnrichmentCoverage) + } + // Per-trap-class: units flips 0 -> 1.0, net stays 0. + if got := final.ByTrapClass["units_cents"].Accuracy; got != 1 { + t.Errorf("final units_cents accuracy = %v, want 1", got) + } + if got := final.ByTrapClass["net_revenue"].Accuracy; got != 0 { + t.Errorf("final net_revenue accuracy = %v, want 0", got) + } + + m := res.Metrics + if m.Lessons != 2 || m.LessonsCaptured != 2 || m.LessonsPromoted != 1 { + t.Errorf("lesson metrics = %d/%d/%d, want 2/2/1", m.Lessons, m.LessonsCaptured, m.LessonsPromoted) + } + if m.EvalTasks != 2 { + t.Errorf("eval tasks = %d, want 2", m.EvalTasks) + } + if m.BaselineAccuracy != 0 || m.FinalAccuracy != 0.5 || m.AccuracyLift != 0.5 { + t.Errorf("curve endpoints = %v/%v lift %v, want 0/0.5/0.5", m.BaselineAccuracy, m.FinalAccuracy, m.AccuracyLift) + } + // Token totals: lesson inputs 100+100 plus the one baseline attempt input 50 + // = 250; outputs 20+20 plus attempt 10 = 50. + if m.TotalInputTokens != 250 || m.TotalOutputTokens != 50 { + t.Errorf("token totals = in %d out %d, want 250/50", m.TotalInputTokens, m.TotalOutputTokens) + } + if m.HarnessFailures != 1 { + t.Errorf("total harness failures = %d, want 1", m.HarnessFailures) + } +} + +func TestWriteAndLoadRoundTrip(t *testing.T) { + res := sampleResults() + res.Aggregate() + path := filepath.Join(t.TempDir(), "cold.json") + if err := res.WriteJSON(path); err != nil { + t.Fatalf("write: %v", err) + } + got, err := LoadJSON(path) + if err != nil { + t.Fatalf("load: %v", err) + } + if got.Metrics.FinalAccuracy != res.Metrics.FinalAccuracy || len(got.Checkpoints) != len(res.Checkpoints) { + t.Errorf("round-trip mismatch: %+v", got.Metrics) + } + // Every token total must serialize under a snake_case key so downstream cost + // readers find them (a dropped json tag round-trips fine but breaks consumers). + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"total_input_tokens", "total_output_tokens", "total_cache_read_tokens", "total_cache_creation_tokens"} { + if !strings.Contains(string(raw), key) { + t.Errorf("results JSON missing snake_case key %q", key) + } + } +} + +func TestHumanSummaryRendersCurve(t *testing.T) { + res := sampleResults() + res.Aggregate() + out := res.HumanSummary() + for _, want := range []string{"learning curve", "(empty baseline)", "cs-units", "per-trap-class", "units_cents", "captured but not promoted"} { + if !strings.Contains(out, want) { + t.Errorf("summary missing %q\n%s", want, out) + } + } +} + +func TestAggregateEmptyIsSafe(t *testing.T) { + res := &Results{} + res.Aggregate() // must not panic on no lessons/checkpoints + if res.Metrics.Checkpoints != 0 || res.Metrics.FinalAccuracy != 0 { + t.Errorf("empty results should aggregate to zero, got %+v", res.Metrics) + } +} diff --git a/bench/internal/coldstart/runner.go b/bench/internal/coldstart/runner.go new file mode 100644 index 000000000..22cb69107 --- /dev/null +++ b/bench/internal/coldstart/runner.go @@ -0,0 +1,379 @@ +package coldstart + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" + "github.com/txn2/mcp-data-platform/bench/internal/claudecli" + "github.com/txn2/mcp-data-platform/bench/internal/curriculum" + "github.com/txn2/mcp-data-platform/bench/internal/gen" + "github.com/txn2/mcp-data-platform/bench/internal/lifecycleapi" + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/mcpc" + "github.com/txn2/mcp-data-platform/bench/internal/pool" + "github.com/txn2/mcp-data-platform/bench/internal/promote" + "github.com/txn2/mcp-data-platform/bench/internal/target" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +// insightPollInterval is the delay between capture-verification polls (memory +// capture is synchronous, so an insight is usually visible on the first poll). +const insightPollInterval = 250 * time.Millisecond + +// AdapterFactory builds the model adapter for one episode, keyed by unit (lesson +// or task id) and stage so the scripted adapter can play a per-episode script. +type AdapterFactory func(unitID, stage string) (llm.Adapter, error) + +// Options configures a cold-start run. +type Options struct { + Target target.Target + HTTPTimeout time.Duration + Arm string // a3 (the lifecycle-and-search arm) + K int // fresh evaluator identities per checkpoint (default 1) + CurriculumDir string + TasksDir string + TranscriptDir string + Factory AdapterFactory + // ClaudeCLI, when non-nil, runs each episode through a real `claude -p` + // client instead of the in-process agent loop. Factory is unused in this mode. + ClaudeCLI *claudecli.Runner + ClientVersion string + LLMProvider string + GitCommit string + AuditTimeout time.Duration + // IdentityKeys is the identity-pool size the arm config defines. A run refuses + // to start when the lessons + per-checkpoint evaluators exceed the pool. It + // must be positive: the teacher and every checkpoint's evaluators must be + // distinct identities so the curve measures promoted (shared) knowledge, never + // an evaluator's own capture. + IdentityKeys int + // OnCheckpoint, if set, is called after every checkpoint with the aggregated + // results so far. benchrun wires it to flush the results file, so a run that + // spends real API budget always leaves every completed checkpoint on disk. + OnCheckpoint func(*Results) + Log *slog.Logger +} + +// Run drives the curriculum against an empty enrichment layer and returns the +// learning curve. A harness-level failure in any episode is recorded and +// surfaced in the returned error so the process exits nonzero. +func Run(ctx context.Context, opts Options) (*Results, error) { + cur, evalTasks, err := load(opts) + if err != nil { + return nil, err + } + if opts.K <= 0 { + opts.K = 1 + } + if err := guardPool(cur, opts); err != nil { + return nil, err + } + res := &Results{Manifest: Manifest{ + StartedAt: time.Now().UTC(), GitCommit: opts.GitCommit, Target: opts.Target.BaseURL, + Arm: opts.Arm, LLMProvider: opts.LLMProvider, Seed: gen.Seed, + CurriculumID: cur.ID, CurriculumHash: curriculum.Hash([]curriculum.Curriculum{cur}), + EvalSuite: cur.EvalSuite, TaskSetHash: task.Hash(evalTasks), K: opts.K, + }} + life := lifecycleapi.New(opts.Target.BaseURL, opts.Target.HTTPClient(opts.HTTPTimeout)) + env := &runEnv{ + opts: opts, + log: opts.Log, + audit: auditapi.New(opts.Target.BaseURL, opts.Target.HTTPClient(opts.HTTPTimeout)), + life: life, + reviewer: promote.Reviewer{Life: life, Log: opts.Log}, + } + defer env.closeAdmin() + + failures := env.run(ctx, cur, evalTasks, res) + res.Manifest.FinishedAt = time.Now().UTC() + res.Manifest.PlatformVersion = env.platformVersion + res.Manifest.Model = env.model + res.Manifest.ClientVersion = opts.ClientVersion + res.Aggregate() + if failures > 0 { + return res, fmt.Errorf("%d cold-start episode(s) failed at the harness level; see lessons[].error and checkpoints[].attempts[].error", failures) + } + return res, nil +} + +// load reads the single curriculum and the fixed eval task set. +func load(opts Options) (curriculum.Curriculum, []task.Task, error) { + curricula, err := curriculum.Load(opts.CurriculumDir) + if err != nil { + return curriculum.Curriculum{}, nil, err + } + if len(curricula) != 1 { + return curriculum.Curriculum{}, nil, fmt.Errorf("cold-start expects exactly one curriculum in %s, found %d", opts.CurriculumDir, len(curricula)) + } + cur := curricula[0] + evalTasks, err := loadEvalTasks(opts.TasksDir, cur.EvalSuite, opts.Arm) + if err != nil { + return curriculum.Curriculum{}, nil, err + } + return cur, evalTasks, nil +} + +// loadEvalTasks loads the fixed eval set: the curriculum's suite, applicable to +// the arm, deterministically graded (exec_sql is rejected — the cold-start eval +// loop has no SQL executor, and the S3 suite it targets is numeric/entity only). +func loadEvalTasks(dir, suite, arm string) ([]task.Task, error) { + all, err := task.Load(dir) + if err != nil { + return nil, err + } + var out []task.Task + for _, t := range all { + if t.Suite != suite || !t.AppliesTo(arm) { + continue + } + if t.Grading.Kind == task.GradeExecSQL { + return nil, fmt.Errorf("eval task %s uses exec_sql grading, unsupported by the cold-start eval loop", t.ID) + } + out = append(out, t) + } + if len(out) == 0 { + return nil, fmt.Errorf("no eval tasks for suite %q applicable to arm %q in %s", suite, arm, dir) + } + return out, nil +} + +// guardPool refuses to start when the identities the run needs exceed the pool. +func guardPool(cur curriculum.Curriculum, opts Options) error { + if opts.IdentityKeys <= 0 { + return errors.New("cold-start requires an identity pool (-identity-keys > 0): the teacher and every checkpoint's evaluators must be distinct identities") + } + n := len(cur.Lessons) + need := maxIdentitySeq(n, opts.K) + if need > opts.IdentityKeys { + return fmt.Errorf("%d identities needed (%d teachers + k=%d evaluators over %d checkpoints) exceed the pool of %d; raise -identity-keys and the config pool", + need, n, opts.K, n+1, opts.IdentityKeys) + } + return nil +} + +// teacherSeq is the pool sequence for lesson i's teacher (a distinct identity +// per lesson so capture verification is cleanly scoped to that lesson's insight). +func teacherSeq(lessonIndex int) int { return lessonIndex + 1 } + +// evaluatorSeq is the pool sequence for checkpoint c's repeat r evaluator. The +// teachers occupy 1..n, so evaluators start at n+1 and never collide with a +// teacher or another checkpoint's evaluator. +func evaluatorSeq(checkpointIndex, repeat, lessonCount, k int) int { + return lessonCount + checkpointIndex*k + repeat +} + +// maxIdentitySeq is the highest pool sequence a run touches: the last repeat of +// the last checkpoint's evaluators (checkpoints are 0..n, so n+1 of them). +func maxIdentitySeq(lessonCount, k int) int { + return evaluatorSeq(lessonCount, k, lessonCount, k) +} + +// runEnv holds per-run clients and mutable manifest carry-overs. +type runEnv struct { + opts Options + log *slog.Logger + audit *auditapi.Client + life *lifecycleapi.Client + reviewer promote.Reviewer + + // adminMCP is the lazily-built reviewer session that drives apply_knowledge + // (base admin credential, no rotation), shared across promotes. + adminMCP *mcp.ClientSession + adminHandle string + + platformVersion string + model string +} + +// run executes the baseline checkpoint, then each lesson's teach+promote +// followed by a fresh eval checkpoint, returning the count of harness failures. +func (e *runEnv) run(ctx context.Context, cur curriculum.Curriculum, evalTasks []task.Task, res *Results) int { + failures := 0 + n := len(cur.Lessons) + + base := e.evalCheckpoint(ctx, 0, curriculum.Lesson{}, evalTasks, n, 0) + res.Checkpoints = append(res.Checkpoints, base) + failures += base.HarnessFailures + e.flush(res) + + promoted := 0 + for i, lesson := range cur.Lessons { + lr := e.teachAndPromote(ctx, lesson, teacherSeq(i)) + if lr.Error != "" { + failures++ + } + if boolTrue(lr.Promoted) { + promoted++ + } + res.Lessons = append(res.Lessons, lr) + + cp := e.evalCheckpoint(ctx, i+1, lesson, evalTasks, n, promoted) + res.Checkpoints = append(res.Checkpoints, cp) + failures += cp.HarnessFailures + e.flush(res) + } + return failures +} + +// teachAndPromote runs the lesson's teach episode, verifies capture through the +// insights API, and promotes the insight to its sink. A harness failure lands in +// Error; a missed capture or refused apply is a measured miss (Captured/Promoted +// false, no error). +func (e *runEnv) teachAndPromote(ctx context.Context, lesson curriculum.Lesson, seq int) LessonRecord { + lr := LessonRecord{LessonID: lesson.ID, Title: lesson.Title, TrapClass: lesson.TrapClass, Sink: lesson.Sink} + rec := e.runEpisode(ctx, episodeSpec{ + stage: StageTeach, unitID: lesson.ID, seq: seq, + prompt: lesson.Teach.Prompt, system: teachScaffold, budget: lesson.BudgetToolCalls, + }) + lr.Episode = teachEpisode(rec) + if rec.err != "" { + lr.Error = rec.err + return lr + } + insight, err := promote.WaitForInsight(ctx, e.life, pool.Email(seq), lesson.EntityURN, e.opts.AuditTimeout, insightPollInterval) + if err != nil { + lr.Error = "capture verify: " + err.Error() + return lr + } + captured := insight != nil + lr.Captured = &captured + if !captured { + return lr + } + lr.InsightID = insight.ID + + session, handle, err := e.adminSession(ctx) + if err != nil { + lr.Error = "admin session: " + err.Error() + return lr + } + ok, err := e.reviewer.Apply(ctx, session, handle, promoteTarget(lesson), insight.ID) + if err != nil { + lr.Error = "promote: " + err.Error() + return lr + } + lr.Promoted = &ok + return lr +} + +// promoteTarget maps a lesson onto the shared promotion target. +func promoteTarget(l curriculum.Lesson) promote.Target { + return promote.Target{Label: l.ID, EntityURN: l.EntityURN, Sink: l.Sink, Fact: l.Fact, Page: l.Page, Notes: "bench cold-start promote"} +} + +// teachEpisode maps a teach episode's raw result into the report record. +func teachEpisode(rec episodeResult) EpisodeRecord { + return EpisodeRecord{ + Email: rec.email, SessionID: rec.sessionID, ToolCalls: rec.toolCalls, ToolErrors: rec.toolErrors, + WallMS: rec.wallMS, InputTokens: rec.usage.InputTokens, OutputTokens: rec.usage.OutputTokens, + CacheReadTokens: rec.usage.CacheReadInputTokens, CacheCreationTokens: rec.usage.CacheCreationInputTokens, + Audit: rec.audit, Error: rec.err, + } +} + +// evalCheckpoint runs the fixed eval set with k fresh evaluators and aggregates +// the checkpoint. Each evaluator answers every eval task in its own session. +func (e *runEnv) evalCheckpoint(ctx context.Context, index int, lesson curriculum.Lesson, evalTasks []task.Task, lessonCount, promotedSoFar int) Checkpoint { + cp := Checkpoint{Index: index, PromotedSoFar: promotedSoFar} + if lesson.ID != "" { + cp.LessonID, cp.LessonTitle, cp.TrapClass = lesson.ID, lesson.Title, lesson.TrapClass + } + for r := 1; r <= e.opts.K; r++ { + seq := evaluatorSeq(index, r, lessonCount, e.opts.K) + for _, t := range evalTasks { + cp.Attempts = append(cp.Attempts, e.evalAttempt(ctx, t, seq, r)) + } + } + cp.aggregate() + return cp +} + +// evalAttempt runs one eval task as a fresh evaluator and grades it. +func (e *runEnv) evalAttempt(ctx context.Context, t task.Task, seq, repeat int) EvalAttempt { + rec := e.runEpisode(ctx, episodeSpec{ + stage: StageEval, unitID: t.ID, seq: seq, + prompt: t.Prompt, system: evalSystem(t.Grading.Kind), budget: t.BudgetToolCalls, + }) + att := EvalAttempt{ + TaskID: t.ID, TrapClasses: t.TrapClasses, Email: rec.email, SessionID: rec.sessionID, + Repeat: repeat, FinalAnswer: rec.finalAnswer, WallMS: rec.wallMS, + InputTokens: rec.usage.InputTokens, OutputTokens: rec.usage.OutputTokens, + CacheReadTokens: rec.usage.CacheReadInputTokens, CacheCreationTokens: rec.usage.CacheCreationInputTokens, + Audit: rec.audit, + } + if rec.err != "" { + att.Error = rec.err + return att + } + att.Graded = true + att.Correct = gradeEval(rec.finalAnswer, t.Grading) + return att +} + +// flush aggregates and calls OnCheckpoint so an interruption never discards +// completed, paid-for work. +func (e *runEnv) flush(res *Results) { + if e.opts.OnCheckpoint == nil { + return + } + res.Aggregate() + e.opts.OnCheckpoint(res) +} + +// attemptClient builds the MCP client for one episode, authenticating as the +// pool identity. +func (e *runEnv) attemptClient(seq int) *mcpc.Client { + t := e.opts.Target + t.Credential = pool.Credential(t.Credential, seq, e.opts.IdentityKeys) + return mcpc.New(t.BaseURL, t.HTTPClient(e.opts.HTTPTimeout)) +} + +// adminSession lazily builds and caches the reviewer MCP session (base admin +// credential, no rotation), with its minted handle threaded on every apply. +func (e *runEnv) adminSession(ctx context.Context) (*mcp.ClientSession, string, error) { + if e.adminMCP != nil { + return e.adminMCP, e.adminHandle, nil + } + client := mcpc.New(e.opts.Target.BaseURL, e.opts.Target.HTTPClient(e.opts.HTTPTimeout)) + session, err := client.Connect(ctx) + if err != nil { + return nil, "", fmt.Errorf("admin session connect: %w", err) + } + info, err := mcpc.Mint(ctx, session) + if err != nil { + _ = session.Close() + return nil, "", fmt.Errorf("admin session mint: %w", err) + } + e.recordPlatformVersion(info.PlatformVersion) + e.adminMCP = session + e.adminHandle = info.Handle + return e.adminMCP, e.adminHandle, nil +} + +// closeAdmin closes the cached reviewer session at run end. +func (e *runEnv) closeAdmin() { + if e.adminMCP != nil { + _ = e.adminMCP.Close() + e.adminMCP = nil + } +} + +// recordPlatformVersion captures the platform version once. +func (e *runEnv) recordPlatformVersion(v string) { + if e.platformVersion == "" && v != "" { + e.platformVersion = v + } +} + +// recordModel captures the model name once. +func (e *runEnv) recordModel(m string) { + if e.model == "" && m != "" { + e.model = m + } +} diff --git a/bench/internal/coldstart/runner_test.go b/bench/internal/coldstart/runner_test.go new file mode 100644 index 000000000..181c76906 --- /dev/null +++ b/bench/internal/coldstart/runner_test.go @@ -0,0 +1,596 @@ +package coldstart + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "maps" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/jsonschema-go/jsonschema" + "github.com/modelcontextprotocol/go-sdk/mcp" + "gopkg.in/yaml.v3" + + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" + "github.com/txn2/mcp-data-platform/bench/internal/claudecli" + "github.com/txn2/mcp-data-platform/bench/internal/curriculum" + "github.com/txn2/mcp-data-platform/bench/internal/lifecycleapi" + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/protocol" + "github.com/txn2/mcp-data-platform/bench/internal/target" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +type authCtxKey struct{} + +func testLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +// fakePlatform is a minimal a3 platform: the MCP tools cold-start drives +// (platform_info, memory_capture, search, apply_knowledge) over streamable HTTP, +// plus the admin knowledge and audit REST. A lesson's fact is modeled as its +// trap class: capture records it on an insight, apply marks the class "applied", +// and search reports the applied classes so a downstream evaluator can act on +// promoted knowledge — exactly the surfacing path the real suite exercises. +type fakePlatform struct { + mu sync.Mutex + minted atomic.Int64 + seq int64 + insights []lifecycleapi.Insight + changesets []lifecycleapi.Changeset + events []auditapi.Event + applied map[string]bool // trap class -> promoted + httpSrv *httptest.Server +} + +func newFakePlatform(t *testing.T) *fakePlatform { + t.Helper() + fp := &fakePlatform{applied: map[string]bool{}} + server := mcp.NewServer(&mcp.Implementation{Name: "fake-coldstart", Version: "fake-1.0.0"}, nil) + fp.addPlatformInfo(server) + fp.addMemoryCapture(server) + fp.addSearch(server) + fp.addApplyKnowledge(server) + + mcpHandler := mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return server }, nil) + mux := http.NewServeMux() + mux.HandleFunc("GET /api/v1/admin/knowledge/insights", fp.listInsights) + mux.HandleFunc("GET /api/v1/admin/knowledge/insights/{id}", fp.getInsight) + mux.HandleFunc("PUT /api/v1/admin/knowledge/insights/{id}/status", fp.putStatus) + mux.HandleFunc("GET /api/v1/admin/knowledge/changesets", fp.listChangesets) + mux.HandleFunc("GET /api/v1/admin/knowledge/changesets/{id}", fp.getChangeset) + mux.HandleFunc("/api/v1/admin/audit/events", fp.serveAudit) + mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := context.WithValue(r.Context(), authCtxKey{}, r.Header.Get("Authorization")) + mcpHandler.ServeHTTP(w, r.WithContext(ctx)) + })) + fp.httpSrv = httptest.NewServer(mux) + t.Cleanup(fp.httpSrv.Close) + return fp +} + +func (fp *fakePlatform) emailFromAuth(auth string) string { + v := strings.TrimPrefix(auth, "Bearer ") + if v == "testkey" { + return "bench-admin@apikey.local" + } + if suffix, ok := strings.CutPrefix(v, "testkey-"); ok { + return "bench-agent-" + suffix + "@apikey.local" + } + return v + "@apikey.local" +} + +func (fp *fakePlatform) callerEmail(ctx context.Context) string { + auth, _ := ctx.Value(authCtxKey{}).(string) + return fp.emailFromAuth(auth) +} + +func sessionSchema(extra map[string]*jsonschema.Schema) *jsonschema.Schema { + props := map[string]*jsonschema.Schema{"session_id": {Type: "string"}} + maps.Copy(props, extra) + return &jsonschema.Schema{Type: "object", Properties: props, Required: []string{"session_id"}} +} + +func (fp *fakePlatform) addPlatformInfo(server *mcp.Server) { + mcp.AddTool(server, &mcp.Tool{Name: "platform_info", Description: "orientation"}, + func(context.Context, *mcp.CallToolRequest, map[string]any) (*mcp.CallToolResult, any, error) { + payload := map[string]any{"session_id": fmt.Sprintf("dps_%d", fp.minted.Add(1)), "version": "fake-1.0.0"} + raw, _ := json.Marshal(payload) + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(raw)}}, StructuredContent: payload}, nil, nil + }) +} + +// addMemoryCapture records a pending insight owned by the caller. The insight +// text is the fact's trap class, so apply can mark that class promoted. +func (fp *fakePlatform) addMemoryCapture(server *mcp.Server) { + schema := sessionSchema(map[string]*jsonschema.Schema{ + "text": {Type: "string"}, "category": {Type: "string"}, + "entity_urns": {Type: "array", Items: &jsonschema.Schema{Type: "string"}}, + }) + mcp.AddTool(server, &mcp.Tool{Name: "memory_capture", Description: "save knowledge", InputSchema: schema}, + func(ctx context.Context, _ *mcp.CallToolRequest, args map[string]any) (*mcp.CallToolResult, any, error) { + fp.mu.Lock() + defer fp.mu.Unlock() + fp.seq++ + id := "in-" + strconv.FormatInt(fp.seq, 10) + text, _ := args["text"].(string) + fp.insights = append(fp.insights, lifecycleapi.Insight{ + ID: id, CreatedAt: time.Unix(fp.seq, 0).UTC(), CapturedBy: fp.callerEmail(ctx), + InsightText: text, Status: "pending", EntityURNs: firstURNSlice(args["entity_urns"]), + }) + fp.recordLocked(args, "memory_capture") + return okResult("captured " + id), nil, nil + }) +} + +// addSearch reports the promoted trap classes so an evaluator can answer from +// promoted knowledge. It records an enrichment-bearing audit row once anything +// has been promoted, modeling the delivery-side coverage signal. +func (fp *fakePlatform) addSearch(server *mcp.Server) { + schema := sessionSchema(map[string]*jsonschema.Schema{"query": {Type: "string"}}) + mcp.AddTool(server, &mcp.Tool{Name: "search", Description: "discover knowledge", InputSchema: schema}, + func(_ context.Context, _ *mcp.CallToolRequest, args map[string]any) (*mcp.CallToolResult, any, error) { + fp.mu.Lock() + defer fp.mu.Unlock() + classes := make([]string, 0, len(fp.applied)) + for c := range fp.applied { + classes = append(classes, c) + } + sort.Strings(classes) + fp.recordLocked(args, "search") + return okResult("APPLIED KNOWLEDGE: " + strings.Join(classes, ",")), nil, nil + }) +} + +func (fp *fakePlatform) addApplyKnowledge(server *mcp.Server) { + schema := sessionSchema(map[string]*jsonschema.Schema{ + "action": {Type: "string"}, "entity_urn": {Type: "string"}, + "insight_ids": {Type: "array", Items: &jsonschema.Schema{Type: "string"}}, + }) + mcp.AddTool(server, &mcp.Tool{Name: "apply_knowledge", Description: "promote", InputSchema: schema}, + func(ctx context.Context, _ *mcp.CallToolRequest, args map[string]any) (*mcp.CallToolResult, any, error) { + fp.mu.Lock() + defer fp.mu.Unlock() + ids := stringSlice(args["insight_ids"]) + urn, _ := args["entity_urn"].(string) + fp.seq++ + csID := "cs-" + strconv.FormatInt(fp.seq, 10) + fp.changesets = append(fp.changesets, lifecycleapi.Changeset{ID: csID, TargetURN: urn, SourceInsightIDs: ids, AppliedBy: fp.callerEmail(ctx)}) + for _, id := range ids { + for i := range fp.insights { + if fp.insights[i].ID == id { + fp.insights[i].Status = "applied" + fp.insights[i].ChangesetRef = csID + fp.applied[fp.insights[i].InsightText] = true + } + } + } + fp.recordLocked(args, "apply_knowledge") + return okResult("applied " + csID), nil, nil + }) +} + +// recordLocked appends an audit row; a call carries enrichment once any +// knowledge has been promoted. The caller holds fp.mu. +func (fp *fakePlatform) recordLocked(args map[string]any, tool string) { + sessionID, _ := args["session_id"].(string) + if sessionID == "" { + return + } + fp.events = append(fp.events, auditapi.Event{ + Timestamp: time.Now().UTC(), DurationMS: 2, SessionID: sessionID, ToolName: tool, + Success: true, EventKind: "mcp_tool_call", EnrichmentApplied: len(fp.applied) > 0, + }) +} + +func (fp *fakePlatform) listInsights(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + fp.mu.Lock() + defer fp.mu.Unlock() + var out []lifecycleapi.Insight + for _, in := range fp.insights { + if v := q.Get("captured_by"); v != "" && in.CapturedBy != v { + continue + } + if v := q.Get("status"); v != "" && in.Status != v { + continue + } + if v := q.Get("entity_urn"); v != "" && !in.LinksEntity(v) { + continue + } + out = append(out, in) + } + writeJSON(w, map[string]any{"data": out, "total": len(out)}) +} + +func (fp *fakePlatform) getInsight(w http.ResponseWriter, r *http.Request) { + fp.mu.Lock() + defer fp.mu.Unlock() + for _, in := range fp.insights { + if in.ID == r.PathValue("id") { + writeJSON(w, in) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +func (fp *fakePlatform) putStatus(w http.ResponseWriter, r *http.Request) { + var body struct { + Status string `json:"status"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + fp.mu.Lock() + defer fp.mu.Unlock() + for i := range fp.insights { + if fp.insights[i].ID == r.PathValue("id") { + fp.insights[i].Status = body.Status + writeJSON(w, map[string]string{"status": "ok"}) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +func (fp *fakePlatform) listChangesets(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + fp.mu.Lock() + defer fp.mu.Unlock() + var out []lifecycleapi.Changeset + for _, cs := range fp.changesets { + if v := q.Get("entity_urn"); v != "" && cs.TargetURN != v { + continue + } + out = append(out, cs) + } + writeJSON(w, map[string]any{"data": out, "total": len(out)}) +} + +func (fp *fakePlatform) getChangeset(w http.ResponseWriter, r *http.Request) { + fp.mu.Lock() + defer fp.mu.Unlock() + for _, cs := range fp.changesets { + if cs.ID == r.PathValue("id") { + writeJSON(w, cs) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +func (fp *fakePlatform) serveAudit(w http.ResponseWriter, r *http.Request) { + sessionID := r.URL.Query().Get("session_id") + fp.mu.Lock() + var matched []auditapi.Event + for _, e := range fp.events { + if e.SessionID == sessionID { + matched = append(matched, e) + } + } + fp.mu.Unlock() + writeJSON(w, map[string]any{"data": matched, "total": len(matched)}) +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) +} + +func okResult(text string) *mcp.CallToolResult { + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: text}}} +} + +func firstURNSlice(v any) []string { + s := stringSlice(v) + if len(s) == 0 { + return nil + } + return s[:1] +} + +func stringSlice(v any) []string { + arr, ok := v.([]any) + if !ok { + return nil + } + out := make([]string, 0, len(arr)) + for _, e := range arr { + if s, ok := e.(string); ok { + out = append(out, s) + } + } + return out +} + +// knowledgeAdapter answers only from what the platform surfaces: a teacher +// captures its lesson's class, an evaluator searches and answers correctly iff +// its task's class has been promoted (present in the search result). +type knowledgeAdapter struct { + mode string // "teach" | "eval" + class, urn string + correct, wrong string +} + +func (a *knowledgeAdapter) Model() string { return "knowledge-test" } + +func (a *knowledgeAdapter) Complete(_ context.Context, _ string, msgs []llm.Message, _ []llm.ToolDef) (llm.Message, llm.Usage, error) { + usedTool, lastResult := false, "" + for _, m := range msgs { + for _, tr := range m.ToolResults { + usedTool, lastResult = true, tr.Text + } + } + usage := llm.Usage{InputTokens: 10, OutputTokens: 5} + if a.mode == StageTeach { + if !usedTool { + return llm.Message{Role: "assistant", ToolCalls: []llm.ToolCall{{ID: "1", Name: "memory_capture", + Args: map[string]any{"text": a.class, "category": "business_context", "entity_urns": []any{a.urn}}}}}, usage, nil + } + return llm.Message{Role: "assistant", Text: "saved " + a.class}, usage, nil + } + if !usedTool { + return llm.Message{Role: "assistant", ToolCalls: []llm.ToolCall{{ID: "1", Name: "search", Args: map[string]any{"query": a.class}}}}, usage, nil + } + ans := a.wrong + if strings.Contains(lastResult, a.class) { + ans = a.correct + } + return llm.Message{Role: "assistant", Text: "FINAL ANSWER: " + ans}, usage, nil +} + +const ordersURN = "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)" + +// testCurriculum is two lessons unlocking two eval tasks, one per class. +func testCurriculum() curriculum.Curriculum { + return curriculum.Curriculum{ + ID: "cs-test", Title: "test", EvalSuite: "s3", + Lessons: []curriculum.Lesson{ + {ID: "cs-units", Title: "units", TrapClass: "units_cents", Fact: "cents", EntityURN: ordersURN, + Sink: protocol.SinkDataHub, BudgetToolCalls: 5, Teach: protocol.TeachStage{Prompt: "remember cents"}}, + {ID: "cs-net", Title: "net", TrapClass: "net_revenue", Fact: "net", EntityURN: ordersURN, + Sink: protocol.SinkKnowledgePage, BudgetToolCalls: 5, + Page: &protocol.PagePayload{Slug: "net", Title: "Net", Body: "net policy"}, + Teach: protocol.TeachStage{Prompt: "remember net"}}, + }, + } +} + +func numericTask(id, class string, value float64) task.Task { + return task.Task{ID: id, Suite: "s3", Prompt: "q", Arms: []string{"a3"}, TrapClasses: []string{class}, + BudgetToolCalls: 5, Grading: task.Grading{Kind: task.GradeNumeric, Value: &value, AbsTolerance: 0.01}} +} + +func writeFixtures(t *testing.T, cur curriculum.Curriculum, tasks []task.Task) (curDir, tasksDir string) { + t.Helper() + curDir, tasksDir = t.TempDir(), t.TempDir() + raw, err := yaml.Marshal(cur) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(curDir, cur.ID+".yaml"), raw, 0o600); err != nil { + t.Fatal(err) + } + for _, tk := range tasks { + b, err := yaml.Marshal(tk) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tasksDir, tk.ID+".yaml"), b, 0o600); err != nil { + t.Fatal(err) + } + } + return curDir, tasksDir +} + +// testFactory maps teach units to their lesson class and eval units to their +// task's correct/wrong answers. +func testFactory(cur curriculum.Curriculum, tasks []task.Task) AdapterFactory { + lessons := map[string]curriculum.Lesson{} + for _, l := range cur.Lessons { + lessons[l.ID] = l + } + byTask := map[string]task.Task{} + for _, tk := range tasks { + byTask[tk.ID] = tk + } + return func(unitID, stage string) (llm.Adapter, error) { + if stage == StageTeach { + l := lessons[unitID] + return &knowledgeAdapter{mode: StageTeach, class: l.TrapClass, urn: l.EntityURN}, nil + } + tk := byTask[unitID] + correct := strconv.FormatFloat(*tk.Grading.Value, 'f', 2, 64) + wrong := strconv.FormatFloat(*tk.Grading.Value+1000, 'f', 2, 64) + return &knowledgeAdapter{mode: StageEval, class: tk.TrapClasses[0], correct: correct, wrong: wrong}, nil + } +} + +func testOptions(fp *fakePlatform, curDir, tasksDir string, factory AdapterFactory) Options { + return Options{ + Target: target.Target{BaseURL: fp.httpSrv.URL, Credential: "testkey"}, + HTTPTimeout: 5 * time.Second, + Arm: "a3", + K: 1, + CurriculumDir: curDir, + TasksDir: tasksDir, + Factory: factory, + AuditTimeout: 2 * time.Second, + IdentityKeys: 64, + LLMProvider: "scripted", + Log: testLogger(), + } +} + +// TestColdStartCurveClimbs is the integration test: it wires the real Run over a +// fake platform and asserts the learning curve climbs as lessons are promoted — +// baseline near zero, each promotion unlocking its trap class — proving the +// teach -> capture -> promote -> eval loop is correctly assembled. +func TestColdStartCurveClimbs(t *testing.T) { + cur := testCurriculum() + tasks := []task.Task{ + numericTask("s3-units-a", "units_cents", 100), + numericTask("s3-net-a", "net_revenue", 50), + } + curDir, tasksDir := writeFixtures(t, cur, tasks) + fp := newFakePlatform(t) + opts := testOptions(fp, curDir, tasksDir, testFactory(cur, tasks)) + opts.TranscriptDir = t.TempDir() + flushes := 0 + opts.OnCheckpoint = func(*Results) { flushes++ } + + res, err := Run(context.Background(), opts) + if err != nil { + t.Fatalf("run: %v", err) + } + if flushes != 3 { + t.Errorf("OnCheckpoint fired %d times, want one per checkpoint (3)", flushes) + } + if entries, _ := os.ReadDir(opts.TranscriptDir); len(entries) == 0 { + t.Error("expected per-episode transcripts to be written") + } + + if len(res.Checkpoints) != 3 { + t.Fatalf("want 3 checkpoints (baseline + 2 lessons), got %d", len(res.Checkpoints)) + } + acc := []float64{res.Checkpoints[0].Accuracy, res.Checkpoints[1].Accuracy, res.Checkpoints[2].Accuracy} + if acc[0] != 0 || acc[1] != 0.5 || acc[2] != 1 { + t.Errorf("learning curve = %v, want [0 0.5 1]", acc) + } + // Every lesson captured and promoted. + if res.Metrics.LessonsCaptured != 2 || res.Metrics.LessonsPromoted != 2 { + t.Errorf("lessons captured/promoted = %d/%d, want 2/2", res.Metrics.LessonsCaptured, res.Metrics.LessonsPromoted) + } + // The delivery signal climbs: no enrichment at the empty baseline, full at the end. + if res.Checkpoints[0].EnrichmentCoverage != 0 || res.Checkpoints[2].EnrichmentCoverage == 0 { + t.Errorf("coverage should climb from 0, got %v -> %v", res.Checkpoints[0].EnrichmentCoverage, res.Checkpoints[2].EnrichmentCoverage) + } + // Per-trap-class flips at the right checkpoint. + if res.Checkpoints[1].ByTrapClass["units_cents"].Accuracy != 1 || res.Checkpoints[1].ByTrapClass["net_revenue"].Accuracy != 0 { + t.Errorf("after units lesson, units should be 1.0 and net 0.0, got %+v", res.Checkpoints[1].ByTrapClass) + } + // Teacher and evaluator identities are disjoint. + assertDistinctIdentities(t, res) +} + +func assertDistinctIdentities(t *testing.T, res *Results) { + t.Helper() + teachers := map[string]bool{} + for _, l := range res.Lessons { + teachers[l.Episode.Email] = true + } + for _, cp := range res.Checkpoints { + for _, a := range cp.Attempts { + if teachers[a.Email] { + t.Errorf("evaluator %s reused a teacher identity", a.Email) + } + } + } +} + +func TestGuardPoolRejectsSmallPool(t *testing.T) { + cur := testCurriculum() + if err := guardPool(cur, Options{K: 3, IdentityKeys: 2}); err == nil { + t.Error("expected guardPool to reject a pool smaller than the run needs") + } + if err := guardPool(cur, Options{K: 1, IdentityKeys: 0}); err == nil { + t.Error("expected guardPool to reject a zero pool") + } + // A pool sized exactly to the need is accepted. + need := maxIdentitySeq(len(cur.Lessons), 1) + if err := guardPool(cur, Options{K: 1, IdentityKeys: need}); err != nil { + t.Errorf("exact-fit pool rejected: %v", err) + } +} + +// claudeEvalStream is a canned `claude -p` eval transcript: mint, search, answer. +const claudeEvalStream = `{"type":"system","subtype":"init","mcp_servers":[{"name":"bench","status":"connected"}]} +{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"i0","name":"mcp__bench__platform_info","input":{}}]}} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"i0","is_error":false,"content":"{\"session_id\":\"dps_cc_1\",\"version\":\"fake-1.0.0\"}"}]}} +{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"s1","name":"mcp__bench__search","input":{"query":"units"}}]}} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"s1","is_error":false,"content":"APPLIED KNOWLEDGE: units_cents"}]}} +{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"FINAL ANSWER: 100.00"}]}} +{"type":"result","subtype":"success","is_error":false,"result":"FINAL ANSWER: 100.00","session_id":"cc-1","usage":{"input_tokens":30,"output_tokens":8}}` + +// TestClaudeCLIEpisode isolates the claude-cli episode path: a stubbed client +// returns a canned eval transcript, and the harness maps the client result, +// reads audit best effort by the threaded handle, and writes the transcript. +func TestClaudeCLIEpisode(t *testing.T) { + fp := newFakePlatform(t) + fp.events = append(fp.events, auditapi.Event{DurationMS: 5, SessionID: "dps_cc_1", ToolName: "search", Success: true}) + + runner, err := claudecli.New(claudecli.Options{ + Model: "claude-sonnet-5", + Exec: func(context.Context, claudecli.CommandSpec) ([]byte, []byte, error) { + return []byte(claudeEvalStream), nil, nil + }, + }) + if err != nil { + t.Fatalf("claudecli.New: %v", err) + } + env := &runEnv{ + opts: Options{ + Target: target.Target{BaseURL: fp.httpSrv.URL, Credential: "testkey"}, + HTTPTimeout: 10 * time.Second, Arm: "a3", ClaudeCLI: runner, + IdentityKeys: 32, AuditTimeout: 5 * time.Second, TranscriptDir: t.TempDir(), + }, + log: testLogger(), + audit: auditapi.New(fp.httpSrv.URL, target.Target{BaseURL: fp.httpSrv.URL, Credential: "testkey"}.HTTPClient(10*time.Second)), + } + rec := env.runEpisode(context.Background(), episodeSpec{stage: StageEval, unitID: "s3-units-a", seq: 7, prompt: "q", system: "sys", budget: 5}) + if rec.err != "" { + t.Fatalf("claude-cli episode error: %s", rec.err) + } + if !gradeEval(rec.finalAnswer, task.Grading{Kind: task.GradeNumeric, Value: new(100.0), AbsTolerance: 0.01}) { + t.Errorf("claude-cli answer %q did not grade correct", rec.finalAnswer) + } + if rec.sessionID != "dps_cc_1" { + t.Errorf("handle = %q, want dps_cc_1", rec.sessionID) + } + if entries, _ := os.ReadDir(env.opts.TranscriptDir); len(entries) == 0 { + t.Error("claude-cli transcript not written") + } +} + +func TestGradeEval(t *testing.T) { + numeric := task.Grading{Kind: task.GradeNumeric, Value: new(42.0), AbsTolerance: 0.5} + if !gradeEval("FINAL ANSWER: 42.1", numeric) { + t.Error("in-tolerance numeric should grade correct") + } + if gradeEval("FINAL ANSWER: 99", numeric) { + t.Error("out-of-tolerance numeric should grade incorrect") + } + if gradeEval("FINAL ANSWER: 42", task.Grading{Kind: task.GradeNumeric}) { + t.Error("nil expected value must not grade correct") + } + entity := task.Grading{Kind: task.GradeEntity, Aliases: []string{"North"}, WrongAliases: []string{"South"}} + if !gradeEval("FINAL ANSWER: North", entity) { + t.Error("matching alias should grade correct") + } + if gradeEval("FINAL ANSWER: South", entity) { + t.Error("wrong alias should grade incorrect") + } + if gradeEval("FINAL ANSWER: x", task.Grading{Kind: task.GradeExecSQL}) { + t.Error("unsupported grading kind must not grade correct") + } +} + +func TestLoadEvalTasksRejectsExecSQL(t *testing.T) { + dir := t.TempDir() + raw, _ := yaml.Marshal(task.Task{ID: "s3-x", Suite: "s3", Prompt: "q", Arms: []string{"a3"}, + BudgetToolCalls: 5, ExpectedSQL: "SELECT 1", Grading: task.Grading{Kind: task.GradeExecSQL}}) + _ = os.WriteFile(filepath.Join(dir, "s3-x.yaml"), raw, 0o600) + if _, err := loadEvalTasks(dir, "s3", "a3"); err == nil { + t.Error("expected exec_sql eval task to be rejected") + } +} diff --git a/bench/internal/curriculum/curriculum.go b/bench/internal/curriculum/curriculum.go new file mode 100644 index 000000000..ab604d8b4 --- /dev/null +++ b/bench/internal/curriculum/curriculum.go @@ -0,0 +1,194 @@ +// Package curriculum defines the cold-start knowledge-growth schema (issue +// #963): an ordered sequence of teaching lessons that, run against an empty +// enrichment layer, progressively promote knowledge into the platform's sinks +// (DataHub entity descriptions and portal knowledge pages). A fixed evaluation +// suite is re-run after each lesson so the harness can plot a learning curve — +// answer accuracy and enrichment coverage as a function of accumulated, +// promoted knowledge. +// +// A curriculum is the teach-and-promote half of the S5 lifecycle (issue #944) +// generalized into a sequence: each lesson states a fact conversationally, +// captures it as an insight, and a reviewer promotes it to a sink, exactly as a +// protocol's teach+promote stages do. The difference is what is measured — not +// a single teach-once-answer-forever transition, but the whole eval set's +// accuracy climbing from the empty floor toward the fully-documented ceiling as +// lessons accumulate. Ground truth for the eval set is the committed S3 task +// suite, whose answers are computed from the seeded dataset (never hand-typed), +// exactly as the S1-S3 and S5 truths are. +package curriculum + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/txn2/mcp-data-platform/bench/internal/protocol" +) + +// Curriculum is one cold-start knowledge-growth script: an ordered lesson +// sequence plus the fixed eval suite re-run at every checkpoint. It is pure +// generated data over a fixed runner, like the task and protocol sets. +type Curriculum struct { + // ID is unique across the curriculum set, e.g. "cs-traps". + ID string `yaml:"id" json:"id"` + // Title is a human label for the report. + Title string `yaml:"title" json:"title"` + // EvalSuite names the committed task suite re-run at each checkpoint + // (e.g. "s3", the knowledge-trap suite). The runner loads it from the tasks + // directory, so the eval ground truth stays generated, never duplicated here. + EvalSuite string `yaml:"eval_suite" json:"eval_suite"` + // Lessons are the ordered teaching steps. The learning curve is measured at + // the empty baseline and after each lesson is promoted, so lesson order is + // the curve's x-axis: earlier lessons unlock the traps that depend only on + // them, later multi-fact traps flip once all their lessons have landed. + Lessons []Lesson `yaml:"lessons" json:"lessons"` +} + +// Lesson is one teach-and-promote step. It carries everything the runner needs +// to state the fact, verify capture, and promote it to a sink — the same fields +// a protocol's teach+promote path uses. +type Lesson struct { + // ID is unique within the curriculum, e.g. "cs-units-cents". + ID string `yaml:"id" json:"id"` + // Title is a human label for the curve breakdown. + Title string `yaml:"title" json:"title"` + // TrapClass names the S3 trap class this lesson primarily unlocks + // (e.g. "units_cents"), so the report can attribute a curve step to a class. + TrapClass string `yaml:"trap_class" json:"trap_class"` + // Fact is the domain fact taught, verified as a captured insight, and — for + // the datahub sink — promoted as the entity description detail. + Fact string `yaml:"fact" json:"fact"` + // EntityURN anchors the captured insight and is the datahub-sink apply target. + EntityURN string `yaml:"entity_urn" json:"entity_urn"` + // Sink selects the promotion destination: protocol.SinkDataHub or + // protocol.SinkKnowledgePage, the two knowledge-delivery channels. + Sink string `yaml:"sink" json:"sink"` + // Page is the knowledge_page payload, required when Sink is a page sink. + Page *protocol.PagePayload `yaml:"page,omitempty" json:"page,omitempty"` + // BudgetToolCalls caps tool calls in the teach episode. + BudgetToolCalls int `yaml:"budget_tool_calls" json:"budget_tool_calls"` + // Teach is the fact-capture episode (prompt states the fact, agent saves it). + Teach protocol.TeachStage `yaml:"teach" json:"teach"` +} + +// Validate rejects a malformed curriculum at load time so a broken set fails +// before any session is spent on it. +func (c Curriculum) Validate() error { + switch { + case c.ID == "": + return errors.New("curriculum with empty id") + case c.Title == "": + return fmt.Errorf("curriculum %s: empty title", c.ID) + case c.EvalSuite == "": + return fmt.Errorf("curriculum %s: empty eval_suite", c.ID) + case len(c.Lessons) == 0: + return fmt.Errorf("curriculum %s: no lessons", c.ID) + } + seen := map[string]bool{} + for _, l := range c.Lessons { + if err := l.validate(c.ID); err != nil { + return err + } + if seen[l.ID] { + return fmt.Errorf("curriculum %s: duplicate lesson id %s", c.ID, l.ID) + } + seen[l.ID] = true + } + return nil +} + +// validate checks one lesson's required fields and sink payload. +func (l Lesson) validate(curriculumID string) error { + switch { + case l.ID == "": + return fmt.Errorf("curriculum %s: lesson with empty id", curriculumID) + case l.Title == "": + return fmt.Errorf("curriculum %s: lesson %s empty title", curriculumID, l.ID) + case l.TrapClass == "": + return fmt.Errorf("curriculum %s: lesson %s empty trap_class", curriculumID, l.ID) + case l.Fact == "": + return fmt.Errorf("curriculum %s: lesson %s empty fact", curriculumID, l.ID) + case l.EntityURN == "": + return fmt.Errorf("curriculum %s: lesson %s empty entity_urn", curriculumID, l.ID) + case l.BudgetToolCalls <= 0: + return fmt.Errorf("curriculum %s: lesson %s budget_tool_calls must be positive", curriculumID, l.ID) + case l.Teach.Prompt == "": + return fmt.Errorf("curriculum %s: lesson %s empty teach prompt", curriculumID, l.ID) + } + return l.validateSink(curriculumID) +} + +// validateSink checks the sink is known and its payload is present, mirroring +// the protocol sink contract so a lesson promotes exactly as a protocol does. +func (l Lesson) validateSink(curriculumID string) error { + switch l.Sink { + case protocol.SinkDataHub: + return nil + case protocol.SinkKnowledgePage: + if l.Page == nil || l.Page.Slug == "" || l.Page.Title == "" || l.Page.Body == "" { + return fmt.Errorf("curriculum %s: lesson %s knowledge_page sink requires a complete page payload", curriculumID, l.ID) + } + return nil + default: + return fmt.Errorf("curriculum %s: lesson %s unknown sink %q", curriculumID, l.ID, l.Sink) + } +} + +// Load reads every *.yaml curriculum in dir (sorted by filename), validating +// each and rejecting duplicate IDs. +func Load(dir string) ([]Curriculum, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read curriculum dir: %w", err) + } + var curricula []Curriculum + seen := map[string]bool{} + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yaml") { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) // #nosec G304 -- operator-supplied curriculum dir + if err != nil { + return nil, fmt.Errorf("read %s: %w", e.Name(), err) + } + var c Curriculum + if err := yaml.Unmarshal(raw, &c); err != nil { + return nil, fmt.Errorf("parse %s: %w", e.Name(), err) + } + if err := c.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", e.Name(), err) + } + if seen[c.ID] { + return nil, fmt.Errorf("duplicate curriculum id %s", c.ID) + } + seen[c.ID] = true + curricula = append(curricula, c) + } + if len(curricula) == 0 { + return nil, fmt.Errorf("no curricula found in %s", dir) + } + return curricula, nil +} + +// Hash returns the canonical SHA-256 of the curriculum set (sorted by ID, JSON +// encoded) for the run manifest, mirroring task.Hash and protocol.Hash. +func Hash(curricula []Curriculum) string { + sorted := make([]Curriculum, len(curricula)) + copy(sorted, curricula) + slices.SortFunc(sorted, func(a, b Curriculum) int { return strings.Compare(a.ID, b.ID) }) + raw, err := json.Marshal(sorted) + if err != nil { + // Curriculum is a plain data struct; marshal cannot fail on validated input. + panic(fmt.Sprintf("marshal curriculum set: %v", err)) + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} diff --git a/bench/internal/curriculum/curriculum_test.go b/bench/internal/curriculum/curriculum_test.go new file mode 100644 index 000000000..5e842004f --- /dev/null +++ b/bench/internal/curriculum/curriculum_test.go @@ -0,0 +1,149 @@ +package curriculum + +import ( + "os" + "path/filepath" + "testing" + + "github.com/txn2/mcp-data-platform/bench/internal/protocol" +) + +// validLesson returns a minimal well-formed datahub-sink lesson. +func validLesson() Lesson { + return Lesson{ + ID: "cs-a", Title: "A", TrapClass: "units_cents", Fact: "amounts are cents", + EntityURN: "urn:li:dataset:x", Sink: protocol.SinkDataHub, BudgetToolCalls: 5, + Teach: protocol.TeachStage{Prompt: "remember this"}, + } +} + +func validCurriculum() Curriculum { + return Curriculum{ID: "cs-traps", Title: "traps", EvalSuite: "s3", Lessons: []Lesson{validLesson()}} +} + +func TestValidateAcceptsWellFormed(t *testing.T) { + if err := validCurriculum().Validate(); err != nil { + t.Fatalf("valid curriculum rejected: %v", err) + } + // A page-sink lesson with a complete payload is valid. + c := validCurriculum() + c.Lessons[0].Sink = protocol.SinkKnowledgePage + c.Lessons[0].Page = &protocol.PagePayload{Slug: "s", Title: "T", Body: "B"} + if err := c.Validate(); err != nil { + t.Fatalf("valid page-sink curriculum rejected: %v", err) + } +} + +func TestValidateRejectsMalformed(t *testing.T) { + cases := map[string]func(*Curriculum){ + "empty id": func(c *Curriculum) { c.ID = "" }, + "empty title": func(c *Curriculum) { c.Title = "" }, + "empty eval_suite": func(c *Curriculum) { c.EvalSuite = "" }, + "no lessons": func(c *Curriculum) { c.Lessons = nil }, + "lesson empty id": func(c *Curriculum) { c.Lessons[0].ID = "" }, + "lesson empty fact": func(c *Curriculum) { + c.Lessons[0].Fact = "" + }, + "lesson empty entity": func(c *Curriculum) { c.Lessons[0].EntityURN = "" }, + "lesson empty trap": func(c *Curriculum) { c.Lessons[0].TrapClass = "" }, + "lesson zero budget": func(c *Curriculum) { c.Lessons[0].BudgetToolCalls = 0 }, + "lesson empty teach": func(c *Curriculum) { c.Lessons[0].Teach.Prompt = "" }, + "unknown sink": func(c *Curriculum) { c.Lessons[0].Sink = "s3" }, + "page sink no payload": func(c *Curriculum) { + c.Lessons[0].Sink = protocol.SinkKnowledgePage + c.Lessons[0].Page = nil + }, + "page sink partial payload": func(c *Curriculum) { + c.Lessons[0].Sink = protocol.SinkKnowledgePage + c.Lessons[0].Page = &protocol.PagePayload{Slug: "s", Title: "", Body: "B"} + }, + "duplicate lesson id": func(c *Curriculum) { + c.Lessons = append(c.Lessons, c.Lessons[0]) + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + c := validCurriculum() + mutate(&c) + if err := c.Validate(); err == nil { + t.Errorf("%s: expected validation error, got nil", name) + } + }) + } +} + +func TestLoadReadsAndValidates(t *testing.T) { + dir := t.TempDir() + writeYAML(t, filepath.Join(dir, "cs-traps.yaml"), ` +id: cs-traps +title: traps +eval_suite: s3 +lessons: + - id: cs-a + title: A + trap_class: units_cents + fact: amounts are cents + entity_urn: urn:li:dataset:x + sink: datahub + budget_tool_calls: 5 + teach: + prompt: remember this +`) + got, err := Load(dir) + if err != nil { + t.Fatalf("load: %v", err) + } + if len(got) != 1 || got[0].ID != "cs-traps" || len(got[0].Lessons) != 1 { + t.Fatalf("unexpected load result: %+v", got) + } +} + +func TestLoadRejectsInvalidAndDuplicates(t *testing.T) { + dir := t.TempDir() + writeYAML(t, filepath.Join(dir, "bad.yaml"), "id: x\ntitle: t\neval_suite: s3\nlessons: []\n") + if _, err := Load(dir); err == nil { + t.Error("expected load to reject a curriculum with no lessons") + } + + dir2 := t.TempDir() + one := ` +id: dup +title: t +eval_suite: s3 +lessons: + - {id: l, title: L, trap_class: units_cents, fact: f, entity_urn: u, sink: datahub, budget_tool_calls: 1, teach: {prompt: p}} +` + writeYAML(t, filepath.Join(dir2, "a.yaml"), one) + writeYAML(t, filepath.Join(dir2, "b.yaml"), one) + if _, err := Load(dir2); err == nil { + t.Error("expected load to reject duplicate curriculum ids") + } + + if _, err := Load(t.TempDir()); err == nil { + t.Error("expected load to fail on an empty directory") + } +} + +func TestHashIsStableAndOrderIndependent(t *testing.T) { + a := validCurriculum() + b := validCurriculum() + b.ID = "cs-other" + if Hash([]Curriculum{a, b}) != Hash([]Curriculum{b, a}) { + t.Error("hash must not depend on curriculum order") + } + if Hash([]Curriculum{a}) == Hash([]Curriculum{b}) { + t.Error("distinct curricula must hash differently") + } + changed := validCurriculum() + changed.Lessons[0].Fact = "different fact" + if Hash([]Curriculum{a}) == Hash([]Curriculum{changed}) { + t.Error("a changed lesson must change the hash") + } +} + +func writeYAML(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/bench/internal/gen/emit_curriculum.go b/bench/internal/gen/emit_curriculum.go new file mode 100644 index 000000000..824561171 --- /dev/null +++ b/bench/internal/gen/emit_curriculum.go @@ -0,0 +1,124 @@ +package gen + +import ( + "github.com/txn2/mcp-data-platform/bench/internal/curriculum" + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/protocol" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +// The cold-start curriculum (issue #963) teaches, one lesson at a time, the S3 +// trap facts the A2 seed pre-loads — but starting from the empty baseline +// (DataHubMCEsEmpty, no knowledge pages). Each lesson promotes its fact to the +// same channel A2 delivers it through: the datahub entity description (units, +// freshness, deprecation) or a portal knowledge page (net-revenue policy, fiscal +// calendar, tier definitions). apply_knowledge restores the description text and +// knowledge pages that carry the answer-bearing facts, so the S3 trap suite (all +// of whose disambiguating facts live in that text) reaches its A2 accuracy +// ceiling once all six lessons are promoted. It does not re-create A2's auxiliary +// aspects — globalTags, the structured deprecation aspect, editableSchema column +// docs, customProperties — so the final enrichment layer is not the whole A2 +// catalog, only the fact-bearing channels the trap suite reads. +// +// Lesson order is the curve's x-axis. It runs foundational-first so a multi-fact +// trap flips to correct only once every fact it needs has landed: units before +// net-revenue (net figures are also in cents), then the calendar/freshness/tier/ +// deprecation facts each independent trap classes depend on. + +// coldStartBudget caps tool calls in a cold-start teach episode, matching the S5 +// protocol budget (a capture episode uses well under this; the cap bounds a +// thrashing search that never reaches the capture tool). +const coldStartBudget = protocolBudget + +// Datahub-sink lesson facts. Each becomes the promoted entity description, so it +// must carry the knowledge the trap suite needs on its own. The freshness and +// deprecation facts reuse the exact A2 description text (dailyDescription, +// legacyDescription), so the promoted description is byte-identical to A2's for +// those entities (the auxiliary A2 aspects are not re-created; see the package +// comment above). +const ( + unitsCentsFact = "In memory.bench.orders the monetary columns amount and discount are stored as " + + "INTEGERS IN US CENTS, not dollars; divide by 100 to get USD. Any total computed without dividing " + + "by 100 is off by a factor of 100." + netRevenueFact = "Company revenue reporting policy: revenue = amount - discount, over COMPLETED orders " + + "only (refunded and pending orders excluded). Amounts are in US cents. A gross figure that ignores " + + "discounts or includes non-completed orders is not policy revenue." + fiscalCalendarFact = "The company fiscal year runs February 1 through January 31: fiscal year 2025 is " + + "2025-02-01 through 2026-01-31. Fiscal figures must not be computed over the January-December calendar year." + tierBoundaryFact = "A 'key account' is any customer on the plus OR enterprise tier — a derived segment " + + "broader than the enterprise tier alone and not stored in any column. Counting only enterprise customers " + + "undercounts key accounts." +) + +// Curriculum builds the committed cold-start curriculum from the dataset. Its +// eval set is the S3 trap suite (loaded from the tasks directory at run time), +// whose ground truth is generated, so nothing is duplicated here. +func (d *Dataset) Curriculum() curriculum.Curriculum { + orders := benchURN("orders") + customers := benchURN("customers") + daily := benchURN("daily_region_revenue") + legacy := benchURN("legacy_orders") + return curriculum.Curriculum{ + ID: "cs-traps", + Title: "Cold-start knowledge growth over the S3 trap suite", + EvalSuite: "s3", + Lessons: []curriculum.Lesson{ + datahubLesson("cs-units-cents", "Monetary columns are integer cents", "units_cents", unitsCentsFact, orders), + pageLesson("cs-net-revenue", "Net-revenue reporting policy", "net_revenue", netRevenueFact, orders, + "revenue-reporting-policy", "Revenue Reporting Policy", revenuePolicyBody), + pageLesson("cs-fiscal-calendar", "Fiscal calendar boundaries", "fiscal_calendar", fiscalCalendarFact, orders, + "fiscal-calendar-policy", "Fiscal Calendar Policy", fiscalCalendarBody), + datahubLesson("cs-freshness-cutoff", "Daily index freshness cutoff", "freshness_cutoff", dailyDescription, daily), + pageLesson("cs-tier-boundary", "Key-account tier definition", "tier_boundary", tierBoundaryFact, customers, + "customer-tier-definitions", "Customer Tier Definitions", tierDefinitionsBody), + datahubLesson("cs-deprecated-table", "legacy_orders is deprecated", "deprecated_table", legacyDescription, legacy), + }, + } +} + +// datahubLesson builds a lesson that promotes its fact to the entity's catalog +// description (delivered to any identity via cross-enrichment). +func datahubLesson(id, title, trapClass, fact, urn string) curriculum.Lesson { + return curriculum.Lesson{ + ID: id, Title: title, TrapClass: trapClass, Fact: fact, + EntityURN: urn, Sink: protocol.SinkDataHub, BudgetToolCalls: coldStartBudget, + Teach: protocol.TeachStage{Prompt: teachPrompt(fact)}, + } +} + +// pageLesson builds a lesson that promotes its fact to a portal knowledge page +// (delivered to any identity via the search tool). The page reuses the A2 seed's +// slug/title/body so the promoted page is identical to the documented baseline. +func pageLesson(id, title, trapClass, fact, urn, slug, pageTitle, body string) curriculum.Lesson { + return curriculum.Lesson{ + ID: id, Title: title, TrapClass: trapClass, Fact: fact, + EntityURN: urn, Sink: protocol.SinkKnowledgePage, BudgetToolCalls: coldStartBudget, + Page: &protocol.PagePayload{Slug: slug, Title: pageTitle, Body: body}, + Teach: protocol.TeachStage{Prompt: teachPrompt(fact)}, + } +} + +// ScriptedColdStartSmoke builds the deterministic per-episode playback for the +// no-API-key cold-start smoke. Each lesson's teach stage captures the fact via +// memory_capture (so the harness verifies capture and drives the real +// promotion), and each eval task answers with its computed ground truth via the +// search-then-answer path. One smoke run validates handle threading, the +// insight/changeset APIs, apply_knowledge promotion, deterministic grading, and +// the learning-curve metrics against the live platform with no model. The eval +// answers are always correct (the smoke measures plumbing, not model behavior), +// so its curve is flat-high; the climbing curve is a property of a real model +// run against the empty baseline. +func ScriptedColdStartSmoke(cur curriculum.Curriculum, evalTasks []task.Task) map[string]map[string][]llm.Step { + out := make(map[string]map[string][]llm.Step, len(cur.Lessons)+len(evalTasks)) + for _, l := range cur.Lessons { + out[l.ID] = map[string][]llm.Step{ + "teach": {captureStep(l.Fact, l.EntityURN, "business_context"), {FinalText: "saved the definition"}}, + } + } + for _, t := range evalTasks { + out[t.ID] = map[string][]llm.Step{ + "eval": {searchStep(), {FinalText: "FINAL ANSWER: " + answerString(t.Grading)}}, + } + } + return out +} diff --git a/bench/internal/gen/emit_datahub.go b/bench/internal/gen/emit_datahub.go index eef60acf3..937cfb65d 100644 --- a/bench/internal/gen/emit_datahub.go +++ b/bench/internal/gen/emit_datahub.go @@ -10,6 +10,11 @@ func benchURN(table string) string { return fmt.Sprintf("urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.%s,PROD)", table) } +// benchTables is the fixed set of bench warehouse datasets, in emission order. +// Both the fully-documented A2 seed and the cold-start empty seed iterate it, so +// the two DataHub baselines cover exactly the same entities. +var benchTables = []string{"orders", "customers", "legacy_orders", "daily_region_revenue"} + // mcp is one metadata change proposal in the MCP file format `datahub ingest` // consumes (file source): the aspect payload is a GenericAspect wrapped as // {"json": {...}}. The legacy `datahub put --file` bulk mode that the e2e @@ -65,12 +70,28 @@ func (d *Dataset) DataHubMCEs() ([]byte, error) { deprecation("legacy_orders", "Deprecated. Use memory.bench.orders instead."), datasetProps("daily_region_revenue", dailyDescription, map[string]string{"team": "bench", "grain": "day,region"}), ) - for _, table := range []string{"orders", "customers", "legacy_orders", "daily_region_revenue"} { + for _, table := range benchTables { proposals = append(proposals, tag(table)) } return json.MarshalIndent(proposals, "", " ") } +// DataHubMCEsEmpty emits the cold-start baseline (#963): each bench dataset +// exists as an entity with its name but carries NO knowledge — empty +// description, no column docs, no tags, no deprecation. The enrichment layer is +// therefore bare, so a fresh identity cannot answer a knowledge-trap question +// from the catalog. The cold-start curriculum backfills this knowledge one +// lesson at a time (promoting into these same datasetProperties descriptions and +// into knowledge pages), and the learning curve measures accuracy climbing from +// this empty floor toward the fully-documented A2 ceiling that DataHubMCEs seeds. +func (d *Dataset) DataHubMCEsEmpty() ([]byte, error) { + proposals := make([]mcp, 0, len(benchTables)) + for _, table := range benchTables { + proposals = append(proposals, datasetProps(table, "", map[string]string{})) + } + return json.MarshalIndent(proposals, "", " ") +} + // datasetProps builds a datasetProperties proposal. func datasetProps(table, description string, custom map[string]string) mcp { return mcp{ diff --git a/bench/internal/gen/gen_test.go b/bench/internal/gen/gen_test.go index 19d5662e0..e55f66225 100644 --- a/bench/internal/gen/gen_test.go +++ b/bench/internal/gen/gen_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/txn2/mcp-data-platform/bench/internal/curriculum" "github.com/txn2/mcp-data-platform/bench/internal/task" ) @@ -24,6 +25,11 @@ func TestGenerateDeterministic(t *testing.T) { if string(am) != string(bm) { t.Fatal("mce emitter is not deterministic") } + ae, _ := a.DataHubMCEsEmpty() + be, _ := b.DataHubMCEsEmpty() + if string(ae) != string(be) { + t.Fatal("empty mce emitter is not deterministic") + } } func TestTrapInvariants(t *testing.T) { @@ -122,6 +128,80 @@ func TestScriptedSmokeCoversAllTasks(t *testing.T) { } } +// TestEmptyMCEsAreBare asserts the cold-start baseline carries the entities but +// none of the knowledge the A2 seed does: every bench table is present as a +// datasetProperties skeleton, and no description, column doc, tag, or +// deprecation leaks in (any of those would let a fresh identity answer a trap +// before the curriculum teaches it, flattening the learning curve). +func TestEmptyMCEsAreBare(t *testing.T) { + ds := Generate() + raw, err := ds.DataHubMCEsEmpty() + if err != nil { + t.Fatal(err) + } + var proposals []map[string]any + if err := json.Unmarshal(raw, &proposals); err != nil { + t.Fatalf("empty mces are not valid json: %v", err) + } + if len(proposals) != len(benchTables) { + t.Fatalf("empty seed has %d proposals, want one per table (%d)", len(proposals), len(benchTables)) + } + present := map[string]bool{} + for _, p := range proposals { + if aspect := p["aspectName"]; aspect != "datasetProperties" { + t.Errorf("empty seed emits aspect %v, want only datasetProperties", aspect) + } + urn, _ := p["entityUrn"].(string) + present[urn] = true + aspect, _ := p["aspect"].(map[string]any) + body, _ := aspect["json"].(map[string]any) + if desc, _ := body["description"].(string); desc != "" { + t.Errorf("%s carries a description in the empty seed: %q", urn, desc) + } + } + for _, table := range benchTables { + if !present[benchURN(table)] { + t.Errorf("empty seed missing entity for %s", table) + } + } + // None of the A2 knowledge markers may appear in the bare baseline. + for _, needle := range []string{"US CENTS", "deprecation", "GROSS of discounts", "urn:li:tag:bench", "editableSchemaMetadata"} { + if strings.Contains(string(raw), needle) { + t.Errorf("empty seed leaks A2 knowledge marker %q", needle) + } + } +} + +// TestScriptedColdStartSmokeCoversUnits asserts the smoke has a teach playback +// for every lesson and an eval playback ending in a final answer for every eval +// task — so one scripted run exercises the whole cold-start loop. +func TestScriptedColdStartSmokeCoversUnits(t *testing.T) { + ds := Generate() + cur := ds.Curriculum() + var evalTasks []task.Task + for _, tk := range ds.Tasks() { + if tk.Suite == cur.EvalSuite { + evalTasks = append(evalTasks, tk) + } + } + smoke := ScriptedColdStartSmoke(cur, evalTasks) + for _, l := range cur.Lessons { + steps := smoke[l.ID]["teach"] + if len(steps) == 0 || len(steps[0].ToolCalls) == 0 || steps[0].ToolCalls[0].Name != "memory_capture" { + t.Errorf("lesson %s teach must open with a memory_capture", l.ID) + } + } + for _, tk := range evalTasks { + steps := smoke[tk.ID]["eval"] + if len(steps) == 0 || steps[len(steps)-1].FinalText == "" { + t.Errorf("eval task %s must end in a final answer", tk.ID) + } + } + if len(evalTasks) == 0 { + t.Fatal("no eval tasks found for the smoke") + } +} + // TestCommittedArtifactsMatch is the reproducibility gate: the committed seed // artifacts and task set must regenerate byte-identically from the fixed seed. func TestCommittedArtifactsMatch(t *testing.T) { @@ -131,8 +211,13 @@ func TestCommittedArtifactsMatch(t *testing.T) { if err != nil { t.Fatal(err) } + emptyMCEs, err := ds.DataHubMCEsEmpty() + if err != nil { + t.Fatal(err) + } compareFile(t, filepath.Join(root, "seed/trino/setup.sql"), []byte(ds.TrinoSQL())) compareFile(t, filepath.Join(root, "seed/datahub/bench_mces.json"), mces) + compareFile(t, filepath.Join(root, "seed/datahub/bench_mces_empty.json"), emptyMCEs) compareFile(t, filepath.Join(root, "seed/postgres/knowledge_pages.sql"), []byte(ds.KnowledgePagesSQL())) committed, err := task.Load(filepath.Join(root, "tasks")) @@ -147,6 +232,27 @@ func TestCommittedArtifactsMatch(t *testing.T) { t.Fatal(err) } compareFile(t, filepath.Join(root, "tasks/scripted-smoke.json"), append(smoke, '\n')) + + committedCur, err := curriculum.Load(filepath.Join(root, "curriculum")) + if err != nil { + t.Fatalf("load committed curriculum: %v", err) + } + regen := []curriculum.Curriculum{ds.Curriculum()} + if got, want := curriculum.Hash(committedCur), curriculum.Hash(regen); got != want { + t.Errorf("committed curriculum hash %s != regenerated %s; run `make bench-gen`", got, want) + } + + var evalTasks []task.Task + for _, tk := range ds.Tasks() { + if tk.Suite == ds.Curriculum().EvalSuite { + evalTasks = append(evalTasks, tk) + } + } + csSmoke, err := json.MarshalIndent(ScriptedColdStartSmoke(ds.Curriculum(), evalTasks), "", " ") + if err != nil { + t.Fatal(err) + } + compareFile(t, filepath.Join(root, "curriculum/scripted-cold-start-smoke.json"), append(csSmoke, '\n')) } func compareFile(t *testing.T, path string, want []byte) { diff --git a/bench/internal/lifecycle/promote.go b/bench/internal/lifecycle/promote.go index e49277b5b..1c6201128 100644 --- a/bench/internal/lifecycle/promote.go +++ b/bench/internal/lifecycle/promote.go @@ -9,6 +9,7 @@ import ( "github.com/txn2/mcp-data-platform/bench/internal/lifecycleapi" "github.com/txn2/mcp-data-platform/bench/internal/mcpc" + "github.com/txn2/mcp-data-platform/bench/internal/promote" "github.com/txn2/mcp-data-platform/bench/internal/protocol" ) @@ -17,136 +18,28 @@ import ( // the loop covers request-scheduling slack only. const insightPollInterval = 250 * time.Millisecond -// insightSuperseded is the insight status a clean recall-first supersede leaves -// on the prior insight (mirrors knowledge.StatusSuperseded). -const insightSuperseded = "superseded" - -// insightPending is the status a freshly captured, unreviewed insight carries; -// capture verification looks for it so a prior run's applied/superseded insights -// on the same entity are not mistaken for this episode's capture. -const insightPending = "pending" - -// waitForInsight polls the insights API until a pending insight captured by the -// given identity and anchored to the entity appears, returning the newest, or nil -// when none lands within the audit timeout (a missed capture, not a harness -// error). The pending filter scopes the read to this episode's fresh capture: -// insights an earlier run left on the same entity under a reused pool identity -// have since moved to applied or superseded and are skipped. +// waitForInsight polls for a pending insight captured by the identity and +// anchored to the entity, using the shared promote path (see promote.WaitForInsight). func (e *runEnv) waitForInsight(ctx context.Context, email, urn string) (*lifecycleapi.Insight, error) { - deadline := time.Now().Add(e.opts.AuditTimeout) - for { - insights, err := e.life.ListInsights(ctx, lifecycleapi.InsightFilter{CapturedBy: email, EntityURN: urn, Status: insightPending}) - if err != nil { - return nil, err - } - if newest := newestInsight(insights); newest != nil { - return newest, nil - } - if time.Now().After(deadline) { - return nil, nil //nolint:nilnil // no insight is a graded miss, not an error - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(insightPollInterval): - } - } -} - -// newestInsight returns the most recently created insight, or nil for an empty -// slice. -func newestInsight(insights []lifecycleapi.Insight) *lifecycleapi.Insight { - var newest *lifecycleapi.Insight - for i := range insights { - if newest == nil || insights[i].CreatedAt.After(newest.CreatedAt) { - newest = &insights[i] - } - } - return newest + return promote.WaitForInsight(ctx, e.life, email, urn, e.opts.AuditTimeout, insightPollInterval) } -// promote plays the reviewer: it approves the insight (pending -> approved) and -// applies it via apply_knowledge to the protocol's sink, then verifies through -// the knowledge API that the insight is applied and a live changeset links it. -// A transport-level failure is a harness error (returned); an apply the platform -// refuses, or a promotion the API cannot confirm, is a measured miss (false). -func (e *runEnv) promote(ctx context.Context, p protocol.Protocol, insightID string) (bool, error) { - if err := e.life.Approve(ctx, insightID, "bench lifecycle promote"); err != nil { - return false, fmt.Errorf("approve insight: %w", err) - } +// promoteInsight plays the reviewer: it approves the insight and applies it to +// the protocol's sink over the cached admin session, then verifies through the +// knowledge API (see promote.Reviewer.Apply). A transport-level failure is a +// harness error; an apply the platform refuses is a measured miss (false). +func (e *runEnv) promoteInsight(ctx context.Context, p protocol.Protocol, insightID string) (bool, error) { session, handle, err := e.adminSession(ctx) if err != nil { return false, err } - r := mcpc.Call(ctx, session, applyToolName, applyArgs(p, insightID), handle) - if r.TransportErr != nil { - return false, fmt.Errorf("apply transport: %w", r.TransportErr) - } - if r.ToolErr { - e.log.Warn("apply_knowledge returned an error", "protocol", p.ID, "text", r.Text) - return false, nil - } - return e.verifyPromotion(ctx, p, insightID) -} - -// verifyPromotion confirms the insight is applied and a non-rolled-back -// changeset lists it as a source. It reads the linkage from the insight's -// changeset_ref (set by MarkApplied) and falls back to listing the entity's -// changesets when the ref is absent. -func (e *runEnv) verifyPromotion(ctx context.Context, p protocol.Protocol, insightID string) (bool, error) { - in, err := e.life.GetInsight(ctx, insightID) - if err != nil { - return false, fmt.Errorf("get insight after apply: %w", err) - } - if in.Status != "applied" { - return false, nil - } - if in.ChangesetRef != "" { - cs, err := e.life.GetChangeset(ctx, in.ChangesetRef) - if err != nil { - return false, fmt.Errorf("get changeset %s: %w", in.ChangesetRef, err) - } - return !cs.RolledBack && cs.Sourced(insightID), nil - } - // Fallback: the datahub sink targets the entity URN, so its changeset is - // listable by entity. (The changeset_ref path above covers both sinks; this - // only guards a ref the adapter left unset.) - changesets, err := e.life.ListChangesets(ctx, lifecycleapi.ChangesetFilter{EntityURN: p.EntityURN}) - if err != nil { - return false, fmt.Errorf("list changesets: %w", err) - } - for _, cs := range changesets { - if !cs.RolledBack && cs.Sourced(insightID) { - return true, nil - } - } - return false, nil + return e.reviewer.Apply(ctx, session, handle, promoteTarget(p), insightID) } -// applyArgs builds the apply_knowledge arguments for the protocol's sink. -func applyArgs(p protocol.Protocol, insightID string) map[string]any { - args := map[string]any{ - "action": "apply", - "entity_urn": p.EntityURN, - "insight_ids": []string{insightID}, - "confirm": true, - } - if p.Sink == protocol.SinkKnowledgePage { - args["sink"] = "knowledge_page" - args["page"] = map[string]any{ - "slug": p.Page.Slug, - "title": p.Page.Title, - "body": p.Page.Body, - } - return args - } - args["sink"] = "datahub" - args["changes"] = []map[string]any{{ - "change_type": "update_description", - "target": "", - "detail": p.Fact, - }} - return args +// promoteTarget maps a protocol onto the shared promotion target. The approve +// note is the fixed string the pre-extraction lifecycle path always recorded. +func promoteTarget(p protocol.Protocol) promote.Target { + return promote.Target{Label: p.ID, EntityURN: p.EntityURN, Sink: p.Sink, Fact: p.Fact, Page: p.Page, Notes: "bench lifecycle promote"} } // adminSession lazily builds and caches the reviewer MCP session (base admin diff --git a/bench/internal/lifecycle/runner.go b/bench/internal/lifecycle/runner.go index 155d86533..a3eabcce5 100644 --- a/bench/internal/lifecycle/runner.go +++ b/bench/internal/lifecycle/runner.go @@ -20,14 +20,11 @@ import ( "github.com/txn2/mcp-data-platform/bench/internal/llm" "github.com/txn2/mcp-data-platform/bench/internal/mcpc" "github.com/txn2/mcp-data-platform/bench/internal/pool" + "github.com/txn2/mcp-data-platform/bench/internal/promote" "github.com/txn2/mcp-data-platform/bench/internal/protocol" "github.com/txn2/mcp-data-platform/bench/internal/target" ) -// applyToolName is the reviewer-side promotion tool the harness drives (the -// promote stage is scripted, not agent-driven). -const applyToolName = "apply_knowledge" - // identitiesPerRun is the number of distinct pool identities each protocol // attempt consumes: one teacher (teach, recall, update, abstain) and one learner // (transfer). Each attempt uses fresh identities so the search-first gate's @@ -100,11 +97,13 @@ func Run(ctx context.Context, opts Options) (*Results, error) { return nil, fmt.Errorf("%d identities needed (%d protocols x k=%d x %d) exceed the pool of %d; raise -identity-keys and the config pool", need, len(protocols), opts.K, identitiesPerRun, opts.IdentityKeys) } + life := lifecycleapi.New(opts.Target.BaseURL, opts.Target.HTTPClient(opts.HTTPTimeout)) env := &runEnv{ - opts: opts, - log: opts.Log, - audit: auditapi.New(opts.Target.BaseURL, opts.Target.HTTPClient(opts.HTTPTimeout)), - life: lifecycleapi.New(opts.Target.BaseURL, opts.Target.HTTPClient(opts.HTTPTimeout)), + opts: opts, + log: opts.Log, + audit: auditapi.New(opts.Target.BaseURL, opts.Target.HTTPClient(opts.HTTPTimeout)), + life: life, + reviewer: promote.Reviewer{Life: life, Log: opts.Log}, } defer env.closeAdmin() @@ -153,10 +152,11 @@ func (e *runEnv) checkpoint(res *Results) { // runEnv holds per-run clients and mutable manifest carry-overs. type runEnv struct { - opts Options - log *slog.Logger - audit *auditapi.Client - life *lifecycleapi.Client + opts Options + log *slog.Logger + audit *auditapi.Client + life *lifecycleapi.Client + reviewer promote.Reviewer // adminMCP is the lazily-built reviewer session that drives apply_knowledge // (base admin credential, no identity rotation), shared across promotes, with @@ -255,7 +255,7 @@ func (e *runEnv) teachAndRecall(ctx context.Context, p protocol.Protocol, teache // promoteAndTransfer approves and applies the insight, then runs the // cross-identity transfer episode. Returns true on a harness abort. func (e *runEnv) promoteAndTransfer(ctx context.Context, p protocol.Protocol, learnerSeq int, run *ProtocolRun) bool { - promoted, err := e.promote(ctx, p, run.InsightID) + promoted, err := e.promoteInsight(ctx, p, run.InsightID) if err != nil { run.Error = "promote: " + err.Error() return true @@ -325,7 +325,7 @@ func (e *runEnv) duplicated(ctx context.Context, teachInsightID string) (bool, e if err != nil { return false, err } - return in.Status != insightSuperseded, nil + return in.Status != promote.StatusSuperseded, nil } // gradeUpdate scores the post-update recall: correct only when it matches the diff --git a/bench/internal/promote/promote.go b/bench/internal/promote/promote.go new file mode 100644 index 000000000..faee96bc2 --- /dev/null +++ b/bench/internal/promote/promote.go @@ -0,0 +1,183 @@ +// Package promote is the shared reviewer-promotion path for the benchmark's +// knowledge-lifecycle orchestrators (the S5 lifecycle suite and the cold-start +// knowledge-growth suite). Both drive the same transition — a captured, pending +// insight is approved and applied to a sink (a DataHub entity description or a +// portal knowledge page) via the apply_knowledge tool, and the result is +// verified through the platform's own insights and changesets APIs, never +// inferred from a transcript. Keeping this one implementation avoids two +// subtly-diverging copies of the correctness-critical approve/apply/verify +// sequence. +package promote + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/txn2/mcp-data-platform/bench/internal/lifecycleapi" + "github.com/txn2/mcp-data-platform/bench/internal/mcpc" + "github.com/txn2/mcp-data-platform/bench/internal/protocol" +) + +// ApplyToolName is the platform tool that applies an approved insight to a sink. +const ApplyToolName = "apply_knowledge" + +// Insight lifecycle statuses the harness reads back through the knowledge API. +const ( + // StatusPending is a freshly captured, unreviewed insight; capture + // verification filters on it so an earlier run's applied/superseded insight + // on the same entity is never mistaken for this episode's capture. + StatusPending = "pending" + // StatusApplied is the status a successful promotion leaves. + StatusApplied = "applied" + // StatusSuperseded is the status a clean recall-first supersede leaves on the + // prior insight (mirrors knowledge.StatusSuperseded). + StatusSuperseded = "superseded" +) + +// Target describes what to promote and where — the minimal shape both suites +// share. A protocol or a curriculum lesson maps onto it identically. +type Target struct { + // Label identifies the source (protocol or lesson id) for log lines. + Label string + // EntityURN anchors the insight and is the datahub-sink apply target. + EntityURN string + // Sink is protocol.SinkDataHub or protocol.SinkKnowledgePage. + Sink string + // Fact is the datahub-sink description detail (unused for the page sink). + Fact string + // Page is the knowledge_page payload (required for the page sink). + Page *protocol.PagePayload + // Notes is the reviewer approval note recorded on the insight status update. + // Each suite sets its own fixed string so the recorded reason is stable. + Notes string +} + +// WaitForInsight polls the insights API until a pending insight captured by the +// given identity and anchored to the entity appears, returning the newest, or +// nil when none lands within the timeout (a missed capture, not a harness +// error). The pending filter scopes the read to this episode's fresh capture. +func WaitForInsight(ctx context.Context, life *lifecycleapi.Client, email, urn string, timeout, poll time.Duration) (*lifecycleapi.Insight, error) { + deadline := time.Now().Add(timeout) + for { + insights, err := life.ListInsights(ctx, lifecycleapi.InsightFilter{CapturedBy: email, EntityURN: urn, Status: StatusPending}) + if err != nil { + return nil, err + } + if newest := newestInsight(insights); newest != nil { + return newest, nil + } + if time.Now().After(deadline) { + return nil, nil //nolint:nilnil // no insight is a graded miss, not an error + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(poll): + } + } +} + +// newestInsight returns the most recently created insight, or nil for an empty +// slice. +func newestInsight(insights []lifecycleapi.Insight) *lifecycleapi.Insight { + var newest *lifecycleapi.Insight + for i := range insights { + if newest == nil || insights[i].CreatedAt.After(newest.CreatedAt) { + newest = &insights[i] + } + } + return newest +} + +// Reviewer plays the platform reviewer: it approves an insight and applies it, +// then verifies the result through the knowledge API. +type Reviewer struct { + Life *lifecycleapi.Client + Log *slog.Logger +} + +// Apply approves the insight (pending -> approved) and applies it via +// apply_knowledge to the target's sink over the given admin session, then +// verifies through the knowledge API that the insight is applied and a live +// changeset links it. A transport-level failure is a harness error (returned); +// an apply the platform refuses, or a promotion the API cannot confirm, is a +// measured miss (false, nil). +func (r Reviewer) Apply(ctx context.Context, session *mcp.ClientSession, handle string, t Target, insightID string) (bool, error) { + if err := r.Life.Approve(ctx, insightID, t.Notes); err != nil { + return false, fmt.Errorf("approve insight: %w", err) + } + res := mcpc.Call(ctx, session, ApplyToolName, BuildApplyArgs(t, insightID), handle) + if res.TransportErr != nil { + return false, fmt.Errorf("apply transport: %w", res.TransportErr) + } + if res.ToolErr { + if r.Log != nil { + r.Log.Warn("apply_knowledge returned an error", "target", t.Label, "text", res.Text) + } + return false, nil + } + return r.verify(ctx, t, insightID) +} + +// verify confirms the insight is applied and a non-rolled-back changeset lists +// it as a source. It reads the linkage from the insight's changeset_ref (set by +// MarkApplied) and falls back to listing the entity's changesets when absent. +func (r Reviewer) verify(ctx context.Context, t Target, insightID string) (bool, error) { + in, err := r.Life.GetInsight(ctx, insightID) + if err != nil { + return false, fmt.Errorf("get insight after apply: %w", err) + } + if in.Status != StatusApplied { + return false, nil + } + if in.ChangesetRef != "" { + cs, err := r.Life.GetChangeset(ctx, in.ChangesetRef) + if err != nil { + return false, fmt.Errorf("get changeset %s: %w", in.ChangesetRef, err) + } + return !cs.RolledBack && cs.Sourced(insightID), nil + } + // Fallback: the datahub sink targets the entity URN, so its changeset is + // listable by entity. (The changeset_ref path above covers both sinks; this + // only guards a ref the adapter left unset.) + changesets, err := r.Life.ListChangesets(ctx, lifecycleapi.ChangesetFilter{EntityURN: t.EntityURN}) + if err != nil { + return false, fmt.Errorf("list changesets: %w", err) + } + for _, cs := range changesets { + if !cs.RolledBack && cs.Sourced(insightID) { + return true, nil + } + } + return false, nil +} + +// BuildApplyArgs builds the apply_knowledge arguments for the target's sink. +func BuildApplyArgs(t Target, insightID string) map[string]any { + args := map[string]any{ + "action": "apply", + "entity_urn": t.EntityURN, + "insight_ids": []string{insightID}, + "confirm": true, + } + if t.Sink == protocol.SinkKnowledgePage { + args["sink"] = "knowledge_page" + args["page"] = map[string]any{ + "slug": t.Page.Slug, + "title": t.Page.Title, + "body": t.Page.Body, + } + return args + } + args["sink"] = "datahub" + args["changes"] = []map[string]any{{ + "change_type": "update_description", + "target": "", + "detail": t.Fact, + }} + return args +} diff --git a/bench/internal/promote/promote_test.go b/bench/internal/promote/promote_test.go new file mode 100644 index 000000000..167c84266 --- /dev/null +++ b/bench/internal/promote/promote_test.go @@ -0,0 +1,275 @@ +package promote + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/google/jsonschema-go/jsonschema" + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/txn2/mcp-data-platform/bench/internal/lifecycleapi" + "github.com/txn2/mcp-data-platform/bench/internal/mcpc" + "github.com/txn2/mcp-data-platform/bench/internal/protocol" +) + +func TestBuildApplyArgsDataHub(t *testing.T) { + args := BuildApplyArgs(Target{ + Label: "lc-x", EntityURN: "urn:x", Sink: protocol.SinkDataHub, Fact: "amounts are cents", + }, "ins-1") + if args["sink"] != "datahub" || args["entity_urn"] != "urn:x" || args["confirm"] != true { + t.Fatalf("datahub args wrong: %+v", args) + } + if ids, ok := args["insight_ids"].([]string); !ok || len(ids) != 1 || ids[0] != "ins-1" { + t.Fatalf("insight_ids wrong: %+v", args["insight_ids"]) + } + changes, ok := args["changes"].([]map[string]any) + if !ok || len(changes) != 1 || changes[0]["change_type"] != "update_description" || changes[0]["detail"] != "amounts are cents" { + t.Fatalf("datahub changes wrong: %+v", args["changes"]) + } + if _, hasPage := args["page"]; hasPage { + t.Error("datahub sink must not carry a page payload") + } +} + +func TestBuildApplyArgsKnowledgePage(t *testing.T) { + args := BuildApplyArgs(Target{ + Label: "lc-y", EntityURN: "urn:y", Sink: protocol.SinkKnowledgePage, + Page: &protocol.PagePayload{Slug: "s", Title: "T", Body: "B"}, + }, "ins-2") + if args["sink"] != "knowledge_page" { + t.Fatalf("page sink wrong: %+v", args) + } + page, ok := args["page"].(map[string]any) + if !ok || page["slug"] != "s" || page["title"] != "T" || page["body"] != "B" { + t.Fatalf("page payload wrong: %+v", args["page"]) + } + if _, hasChanges := args["changes"]; hasChanges { + t.Error("page sink must not carry datahub changes") + } +} + +// TestWaitForInsight covers the newest-of-many pick, the pending status filter +// passed through, and the timeout (nil, not error) when nothing lands. +func TestWaitForInsight(t *testing.T) { + older := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + newer := older.Add(time.Hour) + var gotStatus string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotStatus = r.URL.Query().Get("status") + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []lifecycleapi.Insight{ + {ID: "old", CreatedAt: older, Status: "pending"}, + {ID: "new", CreatedAt: newer, Status: "pending"}, + }, + "total": 2, + }) + })) + defer srv.Close() + life := lifecycleapi.New(srv.URL, srv.Client()) + + got, err := WaitForInsight(context.Background(), life, "a@b", "urn:x", time.Second, time.Millisecond) + if err != nil { + t.Fatalf("wait: %v", err) + } + if got == nil || got.ID != "new" { + t.Fatalf("expected newest insight 'new', got %+v", got) + } + if gotStatus != StatusPending { + t.Errorf("capture verify must filter on pending status, got %q", gotStatus) + } +} + +func TestWaitForInsightTimeoutIsMiss(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"data": []lifecycleapi.Insight{}, "total": 0}) + })) + defer srv.Close() + life := lifecycleapi.New(srv.URL, srv.Client()) + + got, err := WaitForInsight(context.Background(), life, "a@b", "urn:x", 20*time.Millisecond, time.Millisecond) + if err != nil { + t.Fatalf("a missed capture must not be an error: %v", err) + } + if got != nil { + t.Fatalf("expected nil for no insight, got %+v", got) + } +} + +// TestReviewerVerify exercises the three verification outcomes: applied with a +// changeset_ref that sources the insight (pass), not-yet-applied (fail), and the +// fallback that lists the entity's changesets when the ref is unset. +func TestReviewerVerify(t *testing.T) { + cases := []struct { + name string + insight lifecycleapi.Insight + handler http.HandlerFunc + want bool + }{ + { + name: "applied with sourcing changeset ref", + insight: lifecycleapi.Insight{ID: "i1", Status: StatusApplied, ChangesetRef: "cs1"}, + want: true, + }, + { + name: "not yet applied", + insight: lifecycleapi.Insight{ID: "i2", Status: "approved"}, + want: false, + }, + { + name: "applied, ref unset, found by entity fallback", + insight: lifecycleapi.Insight{ID: "i3", Status: StatusApplied}, + want: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/api/v1/admin/knowledge/insights/"): + _ = json.NewEncoder(w).Encode(tc.insight) + case r.URL.Path == "/api/v1/admin/knowledge/changesets/cs1": + _ = json.NewEncoder(w).Encode(lifecycleapi.Changeset{ID: "cs1", SourceInsightIDs: []string{tc.insight.ID}}) + case r.URL.Path == "/api/v1/admin/knowledge/changesets": + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []lifecycleapi.Changeset{{ID: "cs9", SourceInsightIDs: []string{tc.insight.ID}}}, + "total": 1, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + r := Reviewer{Life: lifecycleapi.New(srv.URL, srv.Client())} + got, err := r.verify(context.Background(), Target{EntityURN: "urn:x"}, tc.insight.ID) + if err != nil { + t.Fatalf("verify: %v", err) + } + if got != tc.want { + t.Errorf("verify = %v, want %v", got, tc.want) + } + }) + } +} + +// applyFake is a minimal platform: an MCP server exposing platform_info + +// apply_knowledge over streamable HTTP, plus the admin insight/changeset REST +// the Reviewer verifies against. It lets Apply be tested end to end (approve -> +// apply_knowledge -> API verify) through genuine protocol wiring. +type applyFake struct { + mu sync.Mutex + approved map[string]bool + applyError bool + srv *httptest.Server +} + +func newApplyFake(t *testing.T, applyError bool) *applyFake { + t.Helper() + f := &applyFake{approved: map[string]bool{}, applyError: applyError} + server := mcp.NewServer(&mcp.Implementation{Name: "apply-fake", Version: "1.0.0"}, nil) + mcp.AddTool(server, &mcp.Tool{Name: "platform_info", Description: "orientation"}, + func(context.Context, *mcp.CallToolRequest, map[string]any) (*mcp.CallToolResult, any, error) { + payload := map[string]any{"session_id": "dps_test", "version": "fake-1.0.0"} + raw, _ := json.Marshal(payload) + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(raw)}}, StructuredContent: payload}, nil, nil + }) + schema := &jsonschema.Schema{Type: "object", Properties: map[string]*jsonschema.Schema{ + "session_id": {Type: "string"}, "action": {Type: "string"}, "entity_urn": {Type: "string"}, + }, Required: []string{"session_id"}} + mcp.AddTool(server, &mcp.Tool{Name: ApplyToolName, Description: "promote", InputSchema: schema}, + func(context.Context, *mcp.CallToolRequest, map[string]any) (*mcp.CallToolResult, any, error) { + if f.applyError { + return &mcp.CallToolResult{IsError: true, Content: []mcp.Content{&mcp.TextContent{Text: "write refused"}}}, nil, nil + } + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: "applied"}}}, nil, nil + }) + mcpHandler := mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return server }, nil) + mux := http.NewServeMux() + mux.HandleFunc("PUT /api/v1/admin/knowledge/insights/{id}/status", func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.approved[r.PathValue("id")] = true + f.mu.Unlock() + w.WriteHeader(http.StatusOK) + }) + mux.HandleFunc("GET /api/v1/admin/knowledge/insights/{id}", func(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + // The insight reports applied only once approved and applied without error. + status := "approved" + f.mu.Lock() + if f.approved[id] && !f.applyError { + status = StatusApplied + } + f.mu.Unlock() + _ = json.NewEncoder(w).Encode(lifecycleapi.Insight{ID: id, Status: status, ChangesetRef: "cs1"}) + }) + mux.HandleFunc("GET /api/v1/admin/knowledge/changesets/{id}", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(lifecycleapi.Changeset{ID: r.PathValue("id"), SourceInsightIDs: []string{"ins-1"}}) + }) + mux.Handle("/", mcpHandler) + f.srv = httptest.NewServer(mux) + t.Cleanup(f.srv.Close) + return f +} + +func TestReviewerApply(t *testing.T) { + cases := []struct { + name string + applyError bool + want bool + }{ + {name: "promotes and verifies", applyError: false, want: true}, + {name: "apply tool error is a measured miss", applyError: true, want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newApplyFake(t, tc.applyError) + ctx := context.Background() + session, err := mcpc.New(f.srv.URL, f.srv.Client()).Connect(ctx) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer func() { _ = session.Close() }() + info, err := mcpc.Mint(ctx, session) + if err != nil { + t.Fatalf("mint: %v", err) + } + r := Reviewer{Life: lifecycleapi.New(f.srv.URL, f.srv.Client())} + got, err := r.Apply(ctx, session, info.Handle, Target{Label: "cs-x", EntityURN: "urn:x", Sink: protocol.SinkDataHub, Fact: "f"}, "ins-1") + if err != nil { + t.Fatalf("apply: %v", err) + } + if got != tc.want { + t.Errorf("Apply = %v, want %v", got, tc.want) + } + if !f.approved["ins-1"] { + t.Error("Apply must approve the insight before applying") + } + }) + } +} + +// TestReviewerVerifyRolledBackFails ensures a rolled-back changeset does not +// count as a live promotion. +func TestReviewerVerifyRolledBackFails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/changesets/cs1") { + _ = json.NewEncoder(w).Encode(lifecycleapi.Changeset{ID: "cs1", SourceInsightIDs: []string{"i1"}, RolledBack: true}) + return + } + _ = json.NewEncoder(w).Encode(lifecycleapi.Insight{ID: "i1", Status: StatusApplied, ChangesetRef: "cs1"}) + })) + defer srv.Close() + r := Reviewer{Life: lifecycleapi.New(srv.URL, srv.Client())} + got, err := r.verify(context.Background(), Target{EntityURN: "urn:x"}, "i1") + if err != nil { + t.Fatalf("verify: %v", err) + } + if got { + t.Error("a rolled-back changeset must not count as promoted") + } +} diff --git a/bench/seed/datahub/bench_mces_empty.json b/bench/seed/datahub/bench_mces_empty.json new file mode 100644 index 000000000..22bbf5bae --- /dev/null +++ b/bench/seed/datahub/bench_mces_empty.json @@ -0,0 +1,54 @@ +[ + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "json": { + "customProperties": {}, + "description": "", + "name": "orders" + } + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.customers,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "json": { + "customProperties": {}, + "description": "", + "name": "customers" + } + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.legacy_orders,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "json": { + "customProperties": {}, + "description": "", + "name": "legacy_orders" + } + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.daily_region_revenue,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "json": { + "customProperties": {}, + "description": "", + "name": "daily_region_revenue" + } + } + } +] \ No newline at end of file diff --git a/bench/seedgen/main.go b/bench/seedgen/main.go index b0b7a6522..ad6078690 100644 --- a/bench/seedgen/main.go +++ b/bench/seedgen/main.go @@ -14,6 +14,7 @@ import ( "gopkg.in/yaml.v3" + "github.com/txn2/mcp-data-platform/bench/internal/curriculum" "github.com/txn2/mcp-data-platform/bench/internal/gen" "github.com/txn2/mcp-data-platform/bench/internal/protocol" "github.com/txn2/mcp-data-platform/bench/internal/task" @@ -23,15 +24,16 @@ func main() { seedDir := flag.String("seed-dir", "seed", "output directory for seed artifacts") tasksDir := flag.String("tasks-dir", "tasks", "output directory for task YAML and the smoke script") protocolsDir := flag.String("protocols-dir", "protocols", "output directory for S5 lifecycle protocol YAML and smoke") + curriculumDir := flag.String("curriculum-dir", "curriculum", "output directory for the cold-start curriculum YAML") flag.Parse() - if err := run(*seedDir, *tasksDir, *protocolsDir); err != nil { + if err := run(*seedDir, *tasksDir, *protocolsDir, *curriculumDir); err != nil { fmt.Fprintln(os.Stderr, "seedgen:", err) os.Exit(1) } } // run generates the dataset and writes every artifact. -func run(seedDir, tasksDir, protocolsDir string) error { +func run(seedDir, tasksDir, protocolsDir, curriculumDir string) error { ds := gen.Generate() if err := writeSeedArtifacts(ds, seedDir); err != nil { return err @@ -50,8 +52,16 @@ func run(seedDir, tasksDir, protocolsDir string) error { if err := writeLifecycleSmoke(protocols, protocolsDir); err != nil { return err } - fmt.Printf("seedgen: %d customers, %d orders, %d tasks (hash %s), %d protocols (hash %s)\n", - len(ds.Customers), len(ds.Orders), len(tasks), task.Hash(tasks), len(protocols), protocol.Hash(protocols)) + cur := ds.Curriculum() + if err := writeCurriculum(cur, curriculumDir); err != nil { + return err + } + if err := writeColdStartSmoke(cur, tasks, curriculumDir); err != nil { + return err + } + fmt.Printf("seedgen: %d customers, %d orders, %d tasks (hash %s), %d protocols (hash %s), curriculum %s (%d lessons, hash %s)\n", + len(ds.Customers), len(ds.Orders), len(tasks), task.Hash(tasks), len(protocols), protocol.Hash(protocols), + cur.ID, len(cur.Lessons), curriculum.Hash([]curriculum.Curriculum{cur})) return nil } @@ -61,10 +71,15 @@ func writeSeedArtifacts(ds *gen.Dataset, dir string) error { if err != nil { return fmt.Errorf("render datahub mces: %w", err) } + emptyMCEs, err := ds.DataHubMCEsEmpty() + if err != nil { + return fmt.Errorf("render empty datahub mces: %w", err) + } files := map[string][]byte{ - filepath.Join(dir, "trino", "setup.sql"): []byte(ds.TrinoSQL()), - filepath.Join(dir, "datahub", "bench_mces.json"): mces, - filepath.Join(dir, "postgres", "knowledge_pages.sql"): []byte(ds.KnowledgePagesSQL()), + filepath.Join(dir, "trino", "setup.sql"): []byte(ds.TrinoSQL()), + filepath.Join(dir, "datahub", "bench_mces.json"): mces, + filepath.Join(dir, "datahub", "bench_mces_empty.json"): emptyMCEs, + filepath.Join(dir, "postgres", "knowledge_pages.sql"): []byte(ds.KnowledgePagesSQL()), } for path, content := range files { if err := writeFile(path, content); err != nil { @@ -117,6 +132,35 @@ func writeProtocols(protocols []protocol.Protocol, dir string) error { return nil } +// writeCurriculum emits the cold-start curriculum as a single YAML document. +func writeCurriculum(cur curriculum.Curriculum, dir string) error { + if err := cur.Validate(); err != nil { + return fmt.Errorf("generated curriculum %s invalid: %w", cur.ID, err) + } + raw, err := yaml.Marshal(cur) + if err != nil { + return fmt.Errorf("marshal curriculum %s: %w", cur.ID, err) + } + header := "# Generated by bench/seedgen; the eval ground truth is the committed S3 task suite.\n# Do not edit; regenerate with `make bench-gen`.\n" + return writeFile(filepath.Join(dir, cur.ID+".yaml"), append([]byte(header), raw...)) +} + +// writeColdStartSmoke emits the deterministic per-episode playback for the +// no-API-key cold-start smoke, over the curriculum lessons and the eval suite. +func writeColdStartSmoke(cur curriculum.Curriculum, tasks []task.Task, dir string) error { + var evalTasks []task.Task + for _, t := range tasks { + if t.Suite == cur.EvalSuite { + evalTasks = append(evalTasks, t) + } + } + raw, err := json.MarshalIndent(gen.ScriptedColdStartSmoke(cur, evalTasks), "", " ") + if err != nil { + return fmt.Errorf("marshal cold-start smoke: %w", err) + } + return writeFile(filepath.Join(dir, "scripted-cold-start-smoke.json"), append(raw, '\n')) +} + // writeLifecycleSmoke emits the deterministic per-episode playback for the // no-API-key lifecycle smoke. func writeLifecycleSmoke(protocols []protocol.Protocol, dir string) error { diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 22b2ae34b..de1f6def8 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -1077,7 +1077,7 @@ An opt-in pprof listener supports the harness: setting the `PPROF_ADDR` environm ## Agent-effectiveness benchmark -A second measurement harness lives in `bench/` (also a separate Go module): the agent-effectiveness benchmark (issue #930). Where the load harness answers "how much" (throughput, latency, memory), the benchmark answers "how well": it holds the model, prompt scaffold, seed data, and task set constant and ablates only the platform configuration (arms as config profiles: `a0` raw toolkit tools vs `a2` the full semantic-first platform), measuring answer accuracy, pass^k reliability, and tool-call efficiency. Ground truth is generated from a fixed-seed dataset (Trino tables, DataHub metadata, knowledge pages) with the disambiguating facts for its knowledge-trap tasks (units-in-cents, gross-vs-net revenue policy) living only in the metadata and knowledge layers. Efficiency metrics are read back from the platform's own admin audit API (`audit.delivery: sync`), and a run fails loudly when a session's audit rows are incomplete. A deterministic scripted adapter validates the whole pipeline against a live stack with no model API key (`make bench-up`, `make bench-smoke`, `make bench-down`); real runs use a pinned Anthropic model. Like load testing, it is deliberately NOT part of `make verify`. See `bench/README.md`. The S5 suite adds multi-episode memory-insight-knowledge lifecycle protocols (`make bench-lifecycle`, on the `a3` arm): each protocol teaches a novel definition in one session and, across fresh sessions, measures personal recall, cross-identity transfer after the reviewer promotes the insight via `apply_knowledge` (to an entity description or a knowledge page), recall-first supersede when the fact is later corrected, and abstention on facts never taught — every lifecycle state transition verified through the admin insights and changesets APIs, not inferred from transcripts. Published results live in `docs/reference/benchmarks.md` with raw run JSON under `bench/results/`. The page measures one knowledge system from two coupled angles: a surfacing half (cross-enrichment and search deliver curated knowledge from its sinks) and a capture-and-propagation half (memory -> insight -> apply_knowledge promotes facts into those sinks -- DataHub entity/column descriptions or knowledge pages); the lifecycle populates what enrichment and search deliver, so they are two ends of one pipe, not independent features. Surfacing half, validated by S1-S3 (four arms a0/a1/a2/a3, claude-sonnet-5, k=3, 261 graded attempts per arm, zero harness failures): S3 knowledge-trap accuracy climbs 42.7% (raw tools) -> 57.3% (enrichment) -> 98.7% (full platform), a +56-point gain for the platform over bare tools (95% CI +44 to +67), while S1 discovery and S2 analytical accuracy are near ceiling for every arm; median knowledge-trap tool calls drop 16 -> 10. Arm a3 (platform + memory lifecycle) ties a2 (platform) on S1-S3 because these single-session tasks exercise only the surfacing half, NOT because capture lacks value; capture-and-propagation is what produces the sink contents S1-S3 shows are effective once present, and it is measured separately by S5. Capture-and-propagation half (S5 lifecycle protocols, a3, k=3, 15 multi-session protocols): a capture/personal-recall/unprompted-surface/cross-identity-transfer/update-correctness/duplicate/abstention scorecard plus full-lifecycle pass^3, measuring whether the teach-once-answer-forever mechanism works reliably across fresh sessions rather than an accuracy delta over a (trivially-zero) no-memory baseline, with every state transition verified through the admin insights and changesets APIs. The S5 lifecycle is reported as three runs: a shared-store run (45 attempts, one accumulating knowledge store, k-repeats coupled) and two isolated replicates (each three independent k=1 passes with a full clean platform reset between passes, merged to k=3). The two isolated replicates, identical in configuration, disagree materially (duplicate rate 0% vs 42.9%, pass^3 26.7% vs 20.0%, capture/recall a few points each), which shows most metric-by-metric differences at this scale (15 protocols, small supersede denominators) are run-to-run sampling noise rather than the shared-store confound; the apparent isolation "lift" seen in the first replicate is not reproduced by the second. What is stable across all three runs and is read as the firm result: unprompted surface 100%, update correctness 100%, abstention 96-100%, and cross-identity transfer stuck at ~42-47% (the clearest reproducible reliability ceiling). Because S1-S3 already validates surfacing, the transfer gap is a capture/propagation limit upstream (the apply_knowledge promotion landing in an aspect enrichment reads -- mcp-datahub GetEntity fully populates only datasets/dashboards -- or the second identity surfacing the fact but not using it), not a delivery failure. All three runs recorded zero harness failures and zero episode errors, so transient Claude API outages (which surface as excluded harness failures, never mis-graded answers) did not affect the data. Spend is published as token counts (not dollars), computed from committed per-attempt records: ~158.6M tokens total across all runs (25,182 fresh input, 2,664,630 output, 147,183,943 cache-read, 8,676,898 cache-write), of which cache reads are ~93% -- apply current claude-sonnet-5 per-token pricing to these counts for a cost. S1-S3 and the shared-store S5 run used development build v1.102.0-9-gadfb9d90-dirty; the isolated runs used v1.102.0-10-g32d61254-dirty (platform code byte-identical between the two build commits, verified by diff; only benchmark-harness commits differ); each is pinned in its manifest. A `benchrun -baseline ` regression gate exits nonzero when a fresh run falls below a committed baseline beyond per-suite thresholds; an earlier phase-1 pilot and phase-3 S5 pilot remain in the result history. +A second measurement harness lives in `bench/` (also a separate Go module): the agent-effectiveness benchmark (issue #930). Where the load harness answers "how much" (throughput, latency, memory), the benchmark answers "how well": it holds the model, prompt scaffold, seed data, and task set constant and ablates only the platform configuration (arms as config profiles: `a0` raw toolkit tools vs `a2` the full semantic-first platform), measuring answer accuracy, pass^k reliability, and tool-call efficiency. Ground truth is generated from a fixed-seed dataset (Trino tables, DataHub metadata, knowledge pages) with the disambiguating facts for its knowledge-trap tasks (units-in-cents, gross-vs-net revenue policy) living only in the metadata and knowledge layers. Efficiency metrics are read back from the platform's own admin audit API (`audit.delivery: sync`), and a run fails loudly when a session's audit rows are incomplete. A deterministic scripted adapter validates the whole pipeline against a live stack with no model API key (`make bench-up`, `make bench-smoke`, `make bench-down`); real runs use a pinned Anthropic model. Like load testing, it is deliberately NOT part of `make verify`. See `bench/README.md`. The S5 suite adds multi-episode memory-insight-knowledge lifecycle protocols (`make bench-lifecycle`, on the `a3` arm): each protocol teaches a novel definition in one session and, across fresh sessions, measures personal recall, cross-identity transfer after the reviewer promotes the insight via `apply_knowledge` (to an entity description or a knowledge page), recall-first supersede when the fact is later corrected, and abstention on facts never taught — every lifecycle state transition verified through the admin insights and changesets APIs, not inferred from transcripts. A cold-start knowledge-growth suite (`make bench-cold-start`, issue #963) closes the loop between the two halves: it boots the `a3` arm against an empty enrichment layer (undocumented DataHub via `bench_mces_empty.json`, no knowledge pages via `BENCH_SEED_PAGES=0`), teaches a six-lesson curriculum (one fact per S3 trap class, promoted to the same DataHub descriptions and knowledge pages the A2 seed pre-loads), and re-runs the fixed S3 trap suite with a fresh, never-taught evaluator identity after each promotion — producing a learning curve of accuracy, per-trap-class resistance, and enrichment coverage as a function of accumulated promoted knowledge (the ceiling is the fully-documented A2 level, so the capture-and-propagation half visibly populates what the surfacing half delivers). Published results live in `docs/reference/benchmarks.md` with raw run JSON under `bench/results/`. The page measures one knowledge system from two coupled angles: a surfacing half (cross-enrichment and search deliver curated knowledge from its sinks) and a capture-and-propagation half (memory -> insight -> apply_knowledge promotes facts into those sinks -- DataHub entity/column descriptions or knowledge pages); the lifecycle populates what enrichment and search deliver, so they are two ends of one pipe, not independent features. Surfacing half, validated by S1-S3 (four arms a0/a1/a2/a3, claude-sonnet-5, k=3, 261 graded attempts per arm, zero harness failures): S3 knowledge-trap accuracy climbs 42.7% (raw tools) -> 57.3% (enrichment) -> 98.7% (full platform), a +56-point gain for the platform over bare tools (95% CI +44 to +67), while S1 discovery and S2 analytical accuracy are near ceiling for every arm; median knowledge-trap tool calls drop 16 -> 10. Arm a3 (platform + memory lifecycle) ties a2 (platform) on S1-S3 because these single-session tasks exercise only the surfacing half, NOT because capture lacks value; capture-and-propagation is what produces the sink contents S1-S3 shows are effective once present, and it is measured separately by S5. Capture-and-propagation half (S5 lifecycle protocols, a3, k=3, 15 multi-session protocols): a capture/personal-recall/unprompted-surface/cross-identity-transfer/update-correctness/duplicate/abstention scorecard plus full-lifecycle pass^3, measuring whether the teach-once-answer-forever mechanism works reliably across fresh sessions rather than an accuracy delta over a (trivially-zero) no-memory baseline, with every state transition verified through the admin insights and changesets APIs. The S5 lifecycle is reported as three runs: a shared-store run (45 attempts, one accumulating knowledge store, k-repeats coupled) and two isolated replicates (each three independent k=1 passes with a full clean platform reset between passes, merged to k=3). The two isolated replicates, identical in configuration, disagree materially (duplicate rate 0% vs 42.9%, pass^3 26.7% vs 20.0%, capture/recall a few points each), which shows most metric-by-metric differences at this scale (15 protocols, small supersede denominators) are run-to-run sampling noise rather than the shared-store confound; the apparent isolation "lift" seen in the first replicate is not reproduced by the second. What is stable across all three runs and is read as the firm result: unprompted surface 100%, update correctness 100%, abstention 96-100%, and cross-identity transfer stuck at ~42-47% (the clearest reproducible reliability ceiling). Because S1-S3 already validates surfacing, the transfer gap is a capture/propagation limit upstream (the apply_knowledge promotion landing in an aspect enrichment reads -- mcp-datahub GetEntity fully populates only datasets/dashboards -- or the second identity surfacing the fact but not using it), not a delivery failure. All three runs recorded zero harness failures and zero episode errors, so transient Claude API outages (which surface as excluded harness failures, never mis-graded answers) did not affect the data. Spend is published as token counts (not dollars), computed from committed per-attempt records: ~158.6M tokens total across all runs (25,182 fresh input, 2,664,630 output, 147,183,943 cache-read, 8,676,898 cache-write), of which cache reads are ~93% -- apply current claude-sonnet-5 per-token pricing to these counts for a cost. S1-S3 and the shared-store S5 run used development build v1.102.0-9-gadfb9d90-dirty; the isolated runs used v1.102.0-10-g32d61254-dirty (platform code byte-identical between the two build commits, verified by diff; only benchmark-harness commits differ); each is pinned in its manifest. A `benchrun -baseline ` regression gate exits nonzero when a fresh run falls below a committed baseline beyond per-suite thresholds; an earlier phase-1 pilot and phase-3 S5 pilot remain in the result history. Environment-only configuration: `OTEL_TRACES_ENABLED` (default false), `OTEL_EXPORTER_OTLP_ENDPOINT` (OTLP/gRPC `host:port`, default `localhost:4317`), `OTEL_EXPORTER_OTLP_INSECURE` (default true — the common in-cluster topology; set false for a TLS remote collector), `OTEL_TRACES_SAMPLER_ARG` (head-based sampling ratio in [0,1] applied to root spans, default 0.1), and `OTEL_SERVICE_NAME` (service.name resource attribute, default `mcp-data-platform`). The OTLP exporter connects lazily: an unreachable or unconfigured collector never blocks or fails startup; spans are batched and dropped if undeliverable. diff --git a/docs/llms.txt b/docs/llms.txt index 16bc74f32..4b4c3008b 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -91,7 +91,7 @@ mcp-data-platform is the orchestration layer for the txn2 MCP ecosystem. DataHub - [Configuration Reference](https://mcp-data-platform.txn2.com/reference/configuration/): Redirect page; the full YAML schema lives at [Configuration](https://mcp-data-platform.txn2.com/server/configuration/) - [Providers](https://mcp-data-platform.txn2.com/reference/providers/): Semantic, query, and storage provider interfaces - [Middleware](https://mcp-data-platform.txn2.com/reference/middleware/): Request processing chain: tool visibility, description overrides, the search-first gate, icon enrichment, client logging, progress notifications -- [Benchmarks](https://mcp-data-platform.txn2.com/reference/benchmarks/): Agent-effectiveness benchmark results (bench/) measuring one knowledge system from two angles -- the surfacing half (cross-enrichment + search delivering curated knowledge from its sinks) and the capture-and-propagation half (memory -> insight -> apply_knowledge promoting facts into DataHub descriptions or knowledge pages); the lifecycle populates the sinks the enrichment/search layer delivers from, so they are two ends of one pipe, not independent features. S1-S3 (four-arm ablation) validates surfacing: accuracy with bootstrap CIs, pass^k, the S3 knowledge-trap breakdown showing each channel carrying its own facts (descriptions via enrichment, pages via search), and audit-derived tool-call efficiency. S5 (memory-insight-knowledge lifecycle protocols) tests capture-and-propagation: capture, personal recall, cross-identity transfer, supersede, abstention, verified through the admin insights and changesets APIs. a2 ties a3 on S1-S3 because those tasks exercise only surfacing; the transfer gap (~45%) is a capture/propagation limit upstream of the already-validated surfacing. Token spend published from committed per-attempt cache tokens; benchrun -baseline regression gate +- [Benchmarks](https://mcp-data-platform.txn2.com/reference/benchmarks/): Agent-effectiveness benchmark results (bench/) measuring one knowledge system from two angles -- the surfacing half (cross-enrichment + search delivering curated knowledge from its sinks) and the capture-and-propagation half (memory -> insight -> apply_knowledge promoting facts into DataHub descriptions or knowledge pages); the lifecycle populates the sinks the enrichment/search layer delivers from, so they are two ends of one pipe, not independent features. S1-S3 (four-arm ablation) validates surfacing: accuracy with bootstrap CIs, pass^k, the S3 knowledge-trap breakdown showing each channel carrying its own facts (descriptions via enrichment, pages via search), and audit-derived tool-call efficiency. S5 (memory-insight-knowledge lifecycle protocols) tests capture-and-propagation: capture, personal recall, cross-identity transfer, supersede, abstention, verified through the admin insights and changesets APIs. a2 ties a3 on S1-S3 because those tasks exercise only surfacing; the transfer gap (~45%) is a capture/propagation limit upstream of the already-validated surfacing. A cold-start knowledge-growth suite (make bench-cold-start, #963) starts a3 from an empty enrichment layer (undocumented DataHub, no knowledge pages) and teaches a six-lesson curriculum, re-running the S3 trap suite with a fresh evaluator after each promotion to plot a learning curve (accuracy, per-trap-class resistance, enrichment coverage) as promoted knowledge accumulates toward the A2 ceiling. Token spend published from committed per-attempt cache tokens; benchrun -baseline regression gate - [Tuning and Scaling](https://mcp-data-platform.txn2.com/reference/tuning-and-scaling/): Resource limits, Go runtime tuning, horizontal scaling characteristics, connection pool sizing, autoscaling guidance, and measured single-replica throughput/latency limits from the load harness (test/load); the sibling agent-effectiveness benchmark harness (bench/) measures arm-ablated accuracy and efficiency with audit-derived metrics ## Key Capabilities diff --git a/docs/reference/benchmarks.md b/docs/reference/benchmarks.md index 23f574167..b022b556c 100644 --- a/docs/reference/benchmarks.md +++ b/docs/reference/benchmarks.md @@ -543,3 +543,18 @@ figure. | 2026-07-14 | Memory (S5, shared-store) | a3 | claude-sonnet-5 | 15 protocols × 3 | [`bench/results/s5-anthropic-k3/`](https://github.com/txn2/mcp-data-platform/tree/main/bench/results/s5-anthropic-k3/) | | 2026-07-14 | Memory (S5, isolated run 1) | a3 | claude-sonnet-5 | 15 protocols × 3 passes | [`bench/results/s5-anthropic-k3-isolated/`](https://github.com/txn2/mcp-data-platform/tree/main/bench/results/s5-anthropic-k3-isolated/) | | 2026-07-14 | Memory (S5, isolated run 2) | a3 | claude-sonnet-5 | 15 protocols × 3 passes | [`bench/results/s5-anthropic-k3-isolated-v2/`](https://github.com/txn2/mcp-data-platform/tree/main/bench/results/s5-anthropic-k3-isolated-v2/) | + +### Cold-start knowledge growth (upcoming) + +A cold-start knowledge-growth suite (`make bench-cold-start`, issue #963) is +implemented and runnable but not yet published here: it starts the `a3` arm from +an empty enrichment layer (undocumented DataHub, no knowledge pages) and teaches +a six-lesson curriculum — one fact per S3 trap class, promoted to the same +DataHub descriptions and knowledge pages the A2 seed pre-loads — re-running the +fixed S3 trap suite with a fresh, never-taught evaluator identity after each +promotion. The result is a learning curve of accuracy, per-trap-class trap +resistance, and enrichment coverage as promoted knowledge accumulates toward the +A2 ceiling, directly exercising the coupling between the two halves above (the +lifecycle populates the sinks the surfacing layer delivers from). Its learning +curve will be added to this page once a real model run completes; the loop +itself is validated with no API key by `make bench-cold-start-smoke`.