Skip to content

feat(bench): cold-start knowledge-growth suite (#963) #967

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cjimti merged 1 commit into from
Jul 15, 2026
Merged

feat(bench): cold-start knowledge-growth suite (#963) #967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 46 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -822,7 +822,7 @@ BENCH_COMPOSE := DOCKER_DEFAULT_PLATFORM= docker compose -f docker-compose.e2e.y

## bench-gen: Regenerate seed artifacts, the task set, and the S5 protocols from the fixed seed
bench-gen:
@cd bench && $(GO) run ./seedgen -seed-dir seed -tasks-dir tasks -protocols-dir protocols
@cd bench && $(GO) run ./seedgen -seed-dir seed -tasks-dir tasks -protocols-dir protocols -curriculum-dir curriculum

## bench-up: Start the compose stack, seed the bench warehouse, and run the platform (BENCH_ARM=a0|a1|a2|a3)
bench-up: e2e-up
Expand Down Expand Up @@ -856,9 +856,13 @@ bench-up: e2e-up
if ! kill -0 $$(cat $(BENCH_PID)) 2>/dev/null; then \
echo "ERROR: bench platform exited after start (another server answered readiness?); see $(BENCH_LOG)"; \
tail -20 $(BENCH_LOG); exit 1; fi
@echo "Seeding knowledge pages (requires platform migrations, just applied on boot)..."
@$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \
< bench/seed/postgres/knowledge_pages.sql
@if [ "$(BENCH_SEED_PAGES)" = "0" ]; then \
echo "Skipping knowledge-page seeding (BENCH_SEED_PAGES=0, cold-start empty baseline)."; \
else \
echo "Seeding knowledge pages (requires platform migrations, just applied on boot)..."; \
$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \
< bench/seed/postgres/knowledge_pages.sql; \
fi
@echo "Platform ready (pid $$(cat $(BENCH_PID)), arm $(BENCH_ARM))."

## bench-seed-datahub: Push bench metadata into a running DataHub quickstart (a2 arm)
Expand All @@ -869,6 +873,13 @@ bench-seed-datahub:
@printf 'source:\n type: file\n config:\n path: %s/bench/seed/datahub/bench_mces.json\nsink:\n type: datahub-rest\n config:\n server: %s\n' "$$(pwd)" "$(BENCH_DATAHUB_GMS)" > $(BUILD_DIR)/bench-datahub-recipe.yml
datahub ingest -c $(BUILD_DIR)/bench-datahub-recipe.yml

## bench-seed-datahub-empty: Push the cold-start empty baseline into DataHub (entities present, undocumented; issue #963)
bench-seed-datahub-empty:
@command -v datahub >/dev/null 2>&1 || { echo "ERROR: datahub CLI not found (pip install acryl-datahub)"; exit 1; }
@mkdir -p $(BUILD_DIR)
@printf 'source:\n type: file\n config:\n path: %s/bench/seed/datahub/bench_mces_empty.json\nsink:\n type: datahub-rest\n config:\n server: %s\n' "$$(pwd)" "$(BENCH_DATAHUB_GMS)" > $(BUILD_DIR)/bench-datahub-empty-recipe.yml
datahub ingest -c $(BUILD_DIR)/bench-datahub-empty-recipe.yml

## bench-run: Run the benchmark (ARM must match bench-up; LLM=anthropic|scripted|claude-cli, SUITE=, K=, MODEL=)
bench-run:
@mkdir -p build/bench-results
Expand Down Expand Up @@ -923,6 +934,37 @@ bench-lifecycle-report:
@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
$(BUILD_DIR)/benchrun -lifecycle -summarize build/bench-results/lifecycle-a3.json

## bench-cold-start: Run the cold-start knowledge-growth curriculum (issue #963; needs an empty-seeded a3: bench-up BENCH_ARM=a3 BENCH_SEED_PAGES=0 + bench-seed-datahub-empty; LLM=anthropic|scripted|claude-cli, K=, MODEL=)
bench-cold-start:
@mkdir -p build/bench-results
@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
@echo "Resetting cold-start state so the baseline is truly empty (search gate, prior insights/changesets, and any promoted knowledge pages persist in Postgres across runs)..."
@echo " (CASCADE also clears portal_threads, which FK-references knowledge pages; the bench stack is disposable scratch state.)"
@$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \
-c "TRUNCATE search_gate_discovery, memory_records, knowledge_changesets, portal_knowledge_pages CASCADE"
$(BUILD_DIR)/benchrun \
-cold-start \
-arm a3 \
-url $(BENCH_URL) \
-credential $(BENCH_KEY) \
-curriculum bench/curriculum \
-tasks bench/tasks \
-git-commit $$(git rev-parse HEAD) \
-out build/bench-results/cold-start-a3.json \
$(if $(LLM),-llm $(LLM),) \
$(if $(SCRIPT),-script $(SCRIPT),) \
$(if $(K),-k $(K),) \
$(if $(MODEL),-model $(MODEL),)

## bench-cold-start-smoke: Run the scripted (no-API-key) cold-start smoke against the running a3 platform
bench-cold-start-smoke:
@$(MAKE) bench-cold-start LLM=scripted SCRIPT=bench/curriculum/scripted-cold-start-smoke.json K=1

## bench-cold-start-report: Print the human summary (learning curve) of the last cold-start run
bench-cold-start-report:
@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
$(BUILD_DIR)/benchrun -cold-start -summarize build/bench-results/cold-start-a3.json

## bench-report: Print the human summary of the last run for BENCH_ARM
bench-report:
@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
Expand Down
79 changes: 77 additions & 2 deletions bench/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,53 @@ applicable lifecycle). Harness-level failures (connect, adapter, API read-back)
are excluded from the metrics and reported separately, mirroring the S1–S3
pipeline.

## Cold-start knowledge growth (#963)

The S1-S3 and S5 suites ablate the platform with a **pre-seeded** knowledge base.
The cold-start suite (`curriculum/`, generated) instead starts from an **empty
enrichment layer** and measures the platform getting smarter as knowledge
accumulates — a learning curve whose independent variable is the amount of
**promoted (shared)** knowledge, holding the model, prompt, task set, and dataset
constant.

It runs on the `a3` arm against an **empty baseline**: an undocumented DataHub
(`seed/datahub/bench_mces_empty.json` — entities present, but no descriptions,
column docs, tags, or glossary) and **no knowledge pages** (`bench-up` with
`BENCH_SEED_PAGES=0`). Over an ordered **curriculum** of six lessons — one per S3
trap class — the harness:

1. **Teaches** each fact (a teacher identity states it and captures it via
`memory_capture`), then **promotes** it to its sink through `apply_knowledge`:
a DataHub entity description (units, freshness, deprecation) or a portal
knowledge page (net-revenue policy, fiscal calendar, tier definitions). Each
lesson teaches the same S3 trap fact the A2 seed pre-loads, so the trap
suite reaches its A2 accuracy ceiling once all six are promoted (the
fact-bearing description and page channels are restored; A2's auxiliary
aspects — tags, the structured deprecation flag, column docs — are not, but
the S3 traps read the fact text, not those). Capture and promotion are verified through
the admin insights and changesets APIs, reusing the same reviewer-promotion
path (`internal/promote`) the S5 lifecycle uses.
2. **Evaluates** at every checkpoint (the empty baseline and after each lesson)
by re-running the fixed S3 trap suite with a **fresh, never-taught evaluator
identity**. Its only knowledge source is what the platform surfaces —
cross-enrichment for the DataHub-sink facts, `search` for the page-sink facts
— so accuracy climbs only because promotion pushed the fact into shared
knowledge. This isolates the delivery of *promoted* knowledge (the coupling
between the lifecycle and the enrichment layer), not an evaluator's own memory.

The report is a **learning curve**: per checkpoint, the eval set's accuracy, a
per-trap-class breakdown (which lesson unlocked which class), and the
delivery-side **enrichment coverage** (the fraction of tool calls whose response
carried cross-enrichment, from the audit trail). Lesson order is the x-axis, run
foundational-first (units before net-revenue, then the calendar/freshness/tier/
deprecation facts) so a multi-fact trap flips to correct only once every fact it
needs has landed.

Grading is the deterministic S3 grading (numeric tolerance, entity alias); the
suite reuses one identity pool (a distinct teacher per lesson, fresh evaluators
per checkpoint), and a run refuses to start when the lessons plus per-checkpoint
evaluators exceed the pool.

## Running

From the repository root:
Expand Down Expand Up @@ -251,6 +298,30 @@ drives the reviewer-side promotion, and abstains — validating handle threading
the insight/changeset APIs, supersede, grading, and the metrics against the live
platform with no API key and no model variance.

The cold-start suite (#963) boots the same `a3` arm but with the empty baseline
(no knowledge pages, undocumented DataHub), then teaches the curriculum:

```bash
# Boot a3 with an empty enrichment layer: no knowledge pages, empty DataHub.
make bench-up BENCH_ARM=a3 BENCH_SEED_PAGES=0
make bench-seed-datahub-empty # entities present, undocumented

make bench-cold-start-smoke # scripted no-API-key loop validation
make bench-cold-start K=1 # real learning-curve run (needs a model)
make bench-cold-start LLM=claude-cli MODEL=sonnet K=1 # subscription run
make bench-cold-start-report # print the learning curve
```

The **scripted cold-start smoke** (`-llm scripted`) plays
`curriculum/scripted-cold-start-smoke.json` (generated): each lesson captures its
fact and the harness drives the real promotion; each eval task answers with its
computed ground truth. One run validates the whole teach → capture → promote →
eval loop, the insight/changeset APIs, deterministic grading, and the
learning-curve metrics against the live platform with no model. Its eval answers
are always correct (the smoke measures plumbing, not model behavior), so its
curve is flat-high; the climbing curve is a property of a real model run against
the empty baseline.

For the DataHub arms (`a1`, `a2`, `a3`), start a DataHub quickstart first (same
external convention as e2e and load), then `make bench-seed-datahub` and
`make bench-up BENCH_ARM=a2` (or `a1`/`a3`). Run each arm, then
Expand Down Expand Up @@ -334,21 +405,25 @@ bench/
├── seed/ generated seed artifacts (committed; bench-gen)
├── tasks/ generated task YAML + smoke script (committed)
├── protocols/ generated S5 lifecycle protocol YAML + smoke (committed)
├── curriculum/ generated cold-start curriculum YAML + smoke (committed)
├── judge/ versioned rubric + human-labeled calibration set
└── internal/
├── gen/ dataset model, emitters, ground-truth computation, protocols
├── gen/ dataset model, emitters, ground-truth computation, protocols, curriculum
├── task/ task schema, loader, task-set hash
├── protocol/ S5 lifecycle protocol schema, loader, protocol-set hash
├── curriculum/ cold-start curriculum schema, loader, curriculum-set hash
├── llm/ adapter interface + anthropic + scripted
├── claudecli/ real Claude Code client path (claude -p) + stream parse
├── agent/ model-driven tool loop with budget
├── mcpc/ MCP session, handle mint, session_id threading
├── auditapi/ admin audit API read-back + metrics
├── auditapi/ admin audit API read-back + metrics (+ enrichment coverage)
├── lifecycleapi/ admin insights + changesets read-back, approve + apply drivers
├── promote/ shared reviewer-promotion path (approve + apply_knowledge + verify), used by S5 and cold-start
├── grade/ deterministic graders (numeric, entity, execution-result)
├── judge/ LLM judge + calibration harness
├── pipeline/ task x k orchestration
├── lifecycle/ S5 protocol runner, stage graders, metrics, results model
├── coldstart/ cold-start curriculum runner, learning-curve metrics, results model
├── report/ results model, aggregates, cross-arm comparison
└── target/ endpoint + Bearer auth
```
142 changes: 127 additions & 15 deletions bench/benchrun/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"time"

"github.com/txn2/mcp-data-platform/bench/internal/claudecli"
"github.com/txn2/mcp-data-platform/bench/internal/coldstart"
"github.com/txn2/mcp-data-platform/bench/internal/judge"
"github.com/txn2/mcp-data-platform/bench/internal/lifecycle"
"github.com/txn2/mcp-data-platform/bench/internal/llm"
Expand Down Expand Up @@ -54,6 +55,8 @@ type config struct {
protocolsDir string
baseline string
merge string
coldStart bool
curriculumDir string
}

func main() {
Expand Down Expand Up @@ -95,6 +98,8 @@ func parseFlags() config {
flag.StringVar(&cfg.protocolsDir, "protocols", "protocols", "protocol YAML directory (with -lifecycle)")
flag.StringVar(&cfg.baseline, "baseline", "", "committed baseline results JSON: after the run, gate on per-suite regression and exit nonzero if the candidate falls below it")
flag.StringVar(&cfg.merge, "merge", "", "comma-separated per-pass lifecycle result JSONs (with -lifecycle): merge independent k=1 passes into one k=N result and exit")
flag.BoolVar(&cfg.coldStart, "cold-start", false, "run the cold-start knowledge-growth curriculum (issue #963) instead of the task suites")
flag.StringVar(&cfg.curriculumDir, "curriculum", "curriculum", "curriculum YAML directory (with -cold-start)")
flag.Parse()
return cfg
}
Expand All @@ -105,6 +110,9 @@ func run(cfg config) error {
if handled, err := runReadOnly(cfg); handled {
return err
}
if cfg.coldStart {
return runColdStart(cfg)
}
if cfg.lifecycle {
return runLifecycle(cfg)
}
Expand All @@ -116,20 +124,8 @@ func run(cfg config) error {
// live benchmark run.
func runReadOnly(cfg config) (bool, error) {
switch {
case cfg.summarize != "" && cfg.lifecycle:
res, err := lifecycle.LoadJSON(cfg.summarize)
if err != nil {
return true, err
}
fmt.Print(res.HumanSummary())
return true, nil
case cfg.summarize != "":
res, err := report.LoadJSON(cfg.summarize)
if err != nil {
return true, err
}
fmt.Print(res.HumanSummary())
return true, nil
return true, runSummarize(cfg)
case cfg.merge != "":
// -merge only makes sense for lifecycle results; refuse rather than fall
// through to a live (paid) benchmark run when -lifecycle is forgotten.
Expand All @@ -145,6 +141,32 @@ func runReadOnly(cfg config) (bool, error) {
return false, nil
}

// runSummarize prints the human summary of an existing results JSON, choosing
// the result shape from the run-mode flags.
func runSummarize(cfg config) error {
switch {
case cfg.coldStart:
res, err := coldstart.LoadJSON(cfg.summarize)
if err != nil {
return err
}
fmt.Print(res.HumanSummary())
case cfg.lifecycle:
res, err := lifecycle.LoadJSON(cfg.summarize)
if err != nil {
return err
}
fmt.Print(res.HumanSummary())
default:
res, err := report.LoadJSON(cfg.summarize)
if err != nil {
return err
}
fmt.Print(res.HumanSummary())
}
return nil
}

// runLifecycle executes the S5 lifecycle protocols and writes outputs. Like the
// task benchmark, the results JSON is written even on failure so partial
// evidence is never discarded.
Expand Down Expand Up @@ -202,9 +224,99 @@ func runLifecycle(cfg config) error {
return runErr
}

// runColdStart executes the cold-start knowledge-growth curriculum and writes
// outputs. Like the other runs, the results JSON is flushed per checkpoint so an
// interruption never discards paid-for work. The -baseline gate scores the
// S1-S3 report shape, so it is refused here (cold-start produces a curve, not
// per-suite accuracy).
func runColdStart(cfg config) error {
if cfg.arm == "" {
return errors.New("-arm is required")
}
if cfg.baseline != "" {
return errors.New("-baseline is not supported with -cold-start (the regression gate scores S1-S3 task suites, not the learning curve)")
}
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
opts := coldstart.Options{
Target: target.Target{BaseURL: cfg.url, Credential: cfg.credential},
HTTPTimeout: cfg.httpTimeout,
Arm: cfg.arm,
K: cfg.k,
CurriculumDir: cfg.curriculumDir,
TasksDir: cfg.tasksDir,
TranscriptDir: transcriptDir(cfg.out),
LLMProvider: cfg.llmProvider,
GitCommit: cfg.gitCommit,
AuditTimeout: cfg.auditTimeout,
IdentityKeys: cfg.identityKeys,
OnCheckpoint: func(r *coldstart.Results) {
if err := r.WriteJSON(cfg.out); err != nil {
log.Warn("checkpoint write", "error", err)
}
},
Log: log,
}
if cfg.llmProvider == claudeCLIProvider {
runner, version, err := buildClaudeRunner(cfg)
if err != nil {
return err
}
opts.ClaudeCLI, opts.ClientVersion = runner, version
} else {
factory, err := buildColdStartFactory(cfg)
if err != nil {
return err
}
opts.Factory = factory
}
res, runErr := coldstart.Run(context.Background(), opts)
if res != nil {
if err := writeAndSummarize(res, cfg.out); err != nil {
return err
}
}
return runErr
}

// buildColdStartFactory constructs the per-episode adapter factory: a shared
// stateless model adapter, or a fresh scripted adapter per episode keyed by unit
// (lesson or task id) and stage. The scripted map has the same shape as the
// lifecycle smoke (unit -> stage -> steps), so it reuses the same loader.
func buildColdStartFactory(cfg config) (coldstart.AdapterFactory, error) {
switch cfg.llmProvider {
case "anthropic":
adapter, err := llm.NewAnthropic(cfg.model, cfg.maxTokens, cfg.llmTimeout)
if err != nil {
return nil, err
}
return func(string, string) (llm.Adapter, error) { return adapter, nil }, nil
case "scripted":
if cfg.script == "" {
return nil, errors.New("-script is required for -llm scripted")
}
script, err := llm.LoadLifecycleScript(cfg.script)
if err != nil {
return nil, err
}
return func(unitID, stage string) (llm.Adapter, error) {
stages, ok := script[unitID]
if !ok {
return nil, fmt.Errorf("cold-start script has no unit %s", unitID)
}
steps, ok := stages[stage]
if !ok {
return nil, fmt.Errorf("cold-start script has no %s/%s stage", unitID, stage)
}
return llm.NewScripted(steps), nil
}, nil
default:
return nil, fmt.Errorf("unknown -llm provider %q", cfg.llmProvider)
}
}

// summarizable is any run result that can persist itself and render a summary,
// satisfied by both report.Results (S1-S3) and lifecycle.Results (S5), so the
// two run paths share one write-and-print block.
// satisfied by report.Results (S1-S3), lifecycle.Results (S5), and
// coldstart.Results (#963), so the run paths share one write-and-print block.
type summarizable interface {
WriteJSON(path string) error
HumanSummary() string
Expand Down
Loading
Loading