diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e9de242..de295ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,15 +19,15 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Set up pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 with: version: 10.4.1 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: ${{ matrix.node-version }} cache: pnpm @@ -46,3 +46,71 @@ jobs: - name: Run build run: pnpm run build + + clean-consumer: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: [22, 24] + + steps: + - name: Check out repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: 10.4.1 + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: ${{ matrix.node-version }} + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build package + run: pnpm run build + + - name: Pack and exercise clean consumer + run: node scripts/clean-consumer.mjs --mode happy + + engine-mismatch: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: 10.4.1 + + - name: Set up Node.js for pack + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build package + run: pnpm run build + + - name: Pack release candidate + run: | + mkdir -p "$RUNNER_TEMP/kit-pack" + npm pack --ignore-scripts --pack-destination "$RUNNER_TEMP/kit-pack" + echo "TARBALL=$(ls "$RUNNER_TEMP/kit-pack"/*.tgz)" >> "$GITHUB_ENV" + + - name: Set up Node.js 20 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + + - name: Surface declared engine mismatch + run: node scripts/clean-consumer.mjs --mode engine-mismatch --tarball "$TARBALL" diff --git a/.gitignore b/.gitignore index 001696b..e93f79d 100644 --- a/.gitignore +++ b/.gitignore @@ -79,5 +79,7 @@ session-exports/ !.omo/plans/ # Local agent planning outside OmO -/plans/ /.grok/ + +# Prometheus scratch context (repo AGENTS.md public-tree list) +/prometheus-implementation-context.md diff --git a/.omo/plans/omo-native-adapter.md b/.omo/plans/omo-native-adapter.md new file mode 100644 index 0000000..c2cc0a8 --- /dev/null +++ b/.omo/plans/omo-native-adapter.md @@ -0,0 +1,345 @@ +# omo-native-adapter - Work Plan + +## TL;DR (For humans) + + + +**What you'll get:** A complete attach adapter for OmO-native (senpi-engine) coding sessions inside this library: it can discover and list every session across all your projects, read a session's real conversation history correctly even though sessions branch like a git tree, follow live sessions as they grow, validate and run senpi's native command hooks, and - only with explicit consent - register hook integrations for a desktop observer app. + +**Why this approach:** Senpi stores conversations as branching trees, not linear logs - so the one load-bearing decision is that this library (not any consumer) converts each tree into the correct linear history, including compaction checkpoints; and every wire format is copied byte-for-byte from a pinned, checksum-verified copy of the engine's own contract files, so upstream changes are caught by tests before they bite. + +**What it will NOT do:** It never drives, spawns, or controls senpi sessions - observe only. It never reads credentials or telemetry state. It never grants hook trust silently - trust is written only through an explicit consent operation. It does not unify with the Claude or Grok adapters' internals. + +**Effort:** Large +**Risk:** Medium - tree-linearization correctness is the hard part; mitigated by golden fixtures from real transcripts and two-way drift tests against the pinned engine artifacts. +**Decisions to sanity-check:** Pin to senpi 2026.8.19; no new runtime npm dependencies (internal file locking); Windows support is best-effort non-goal for v1; the kit PR merges only after the Cockpit integration phase is tested against it. + +Your next move: approve to start execution via /start-work, or request changes. Full execution detail follows below. + +--- + +> TL;DR (machine): Large/Medium-risk; 23 todos in 5 waves + 4 final verifiers; deliverables: src/senpi adapter (/senpi + /senpi/processing), vendored 2026.8.19 pin + drift tests, tree-linear tail/watch/discovery, 7-event hooks library with consent-gated trust writer, reference doc, cockpit Phase-C brief. + +## Scope +### Must have + +- `src/senpi/` attach adapter with two exact barrel subpaths `/senpi` and `/senpi/processing` (no wildcards), ESM NodeNext, Zod-validated, zero runtime dependency on the senpi engine or OmO. +- Session v3 contract: Zod schemas for header + the 9 known non-header entry tags (message, model_change, thinking_level_change, compaction, branch_summary, custom, custom_message, label, session_info), 7-role AgentMessage union, text/image/thinking/toolCall content blocks, usage/cost. +- Agent-home resolution: `OMO_CODING_AGENT_DIR`/`SENPI_CODING_AGENT_DIR`/`PI_CODING_AGENT_DIR` env → `/.omo/agent` (sentinel settings.json) → legacy flat `/.omo` → `/.senpi/agent`, injectable env/homeDir/exists. +- Observe processing: discovery (top-level *.jsonl only; skip `*-artifacts/` dirs and per-cwd `extensions/`; verify header cwd — never trust dash-encoded dirname), `listSenpiSessions(cwd)`, `listAllSenpiSessions(agentHome?)` with SessionInfo-parity fields MINUS allMessagesText, `parseSenpiEntry` known/unknown/invalid policy, pure tree projection (`resolveSenpiLeaf`, `projectSenpiBranch`) honoring latest-compaction `retainedTail`/legacy `firstKeptEntryId`, internal jsonl cursor, senpi-prefixed checkpoint marker filenames, `tailSenpiSession` with revisioned suffix-splice mutations, `watchSenpiSession` (watch = wakeup hint; quiescence = configurable stable-cursor window, default 30s), Senpi-native block reduce/fold. +- Hooks library: `validateSenpiHooksConfig` (exactly 7 events, command handlers only, pinned diagnostic-code vocabulary), `hook-wire` schemas for HookInputWire/HookOutputWire with camelCase primaries + snake_case aliases, `executeSenpiHook` runner (exit 2 = block with stderr reason; stdout JSON otherwise), `SenpiHookOutputBuilder`, read-only trust state inspection + pure `hookTrustId`/`hashCommandHook` parity. +- Phase D groundwork: consent-gated `trust-writer` (explicit API, file lock, atomic replacement, preserves unknown entries, fail-closed on malformed state), forwarder asset variant + hooks.json registration helper that never writes trust state implicitly. +- Vendored contract artifacts under `docs/upstream/senpi/` (session-format.md + hooks dist .d.ts files, MIT NOTICE) with sha256 `pin.json` ({engineVersion: "2026.8.19", files, fixtureRedump, notes}) and tarball-based `scripts/sync-upstream-senpi.mjs --check` + two-way drift tests. +- Sanitized real-transcript fixtures from ~/.omo/agent/sessions plus synthetic edge-case fixtures under tests/fixtures/senpi/. +- Platform/runtime constraints: runtime floor Node >=22 (kit `engines`); zod ^4 API surface; NO new runtime npm dependencies (internal mkdir/O_EXCL locking instead of proper-lockfile); Windows is best-effort non-goal for v1 (runner honors commandWindows; cursor inode/device identity assumes POSIX); macOS/Linux CI is the supported matrix. +- docs/reference/senpi-adapter.md mirroring docs/reference/grok-adapter.md. +- Cross-repo Cockpit Phase C execution brief at plans/omo-native-adapter/cockpit-phase-c.md (spec only; cockpit code is a separate effort in that repo). + +### Must NOT have (guardrails, anti-slop, scope boundaries) + +- No runtime import of `@code-yeongyu/senpi`, `oh-my-openagent`, or any OmO package. +- No import from `src/grok/*` or Claude `src/processing/*` internals; no changes to Claude-path files, markers, block types, or root exports. +- No shared SessionBlock / hook-event / decision unification across adapters. +- No http/prompt/agent/mcp_tool hook handler types; no extension-API (in-process ExtensionAPI) surface. +- No reads of auth.json; no use of omo-senpi telemetry state as session contract. +- No drive/RPC/spawn control plane (attach-observe only); no multi-home auto-merge (extra stores reachable only via env/option). +- No silent trust-state writes outside the consent-gated writer; no trust writes from runner/settings/install paths. +- No wildcard `/senpi/*` exports; cursor/checkpoint internals stay unbarreled. +- No new runtime npm dependencies (dev/test deps follow existing policy). + +## Verification strategy +> Zero human intervention - all verification is agent-executed. +- Test decision: tests-after + vitest (repo standard); every todo runs `pnpm run check` (type-check + lint) and targeted `pnpm run test:run -- tests/` before commit. +- Contract truth: vendored pin artifacts are the wire authority; drift test must pass both directions (vendored→Zod, Zod→vendored) plus a mutation-detection case. +- Runtime proof: tail of a sanitized REAL transcript fixture must reproduce the expected linear leaf-path history (golden snapshot), including one branch-switch splice and both compaction forms. +- Evidence: every todo's `Evidence /task--....log` path resolves against evidence-root = currentAttemptDir from 'omo-agent-toolkit ulw-loop status --json' (.omo/evidence/ulw///a) when running under ulw-loop, else .omo/evidence/. +- CI wiring decision: `sync-upstream-senpi.mjs --check` is deliberately NOT wired into CI (grok precedent - it needs the tarball); the drift TESTS are vendored-artifact-only and DO run in CI on every push. + +## Execution strategy +### Parallel execution waves +> Target 5-8 todos per wave. Fewer than 3 (except the final) means you under-split. + +- Wave 1 — Contracts & pins: todos 1-5. +- Wave 2 — Processing core + fixtures: todos 6-10 and 14. +- Wave 3 — Integration & observe surface: todos 11-13 and 15. +- Wave 4 — Hooks library & docs: todos 16-20. +- Wave 5 — Managed-hooks groundwork & cross-repo brief: todos 21-24. + +### Dependency matrix +| Todo | Depends on | Blocks | Can parallelize with | +| --- | --- | --- | --- | +| 1 | — | 4, 5, 16, 19 | 2, 3 | +| 2 | — | 5, 6, 7, 10, 11, 14 | 1, 3 | +| 3 | — | 11 | 1, 2 | +| 4 | 1 | 5, 15 | 2, 3 | +| 5 | 1, 2, 4 | 16, 23 | — | +| 6 | 2 | 7, 12, 15 | 8, 14 | +| 7 | 2, 6, 14 | 10, 12 | 8, 9 | +| 8 | — | 9, 12, 15 | 6, 7, 14 | +| 9 | 8 | 12, 15 | 6, 7, 10, 14 | +| 10 | 2, 7 | 12, 15 | 8, 9 | +| 11 | 2, 3 | 12, 15 | 13 | +| 12 | 6, 7, 8, 9, 10, 11 | 13, 15, 23 | — | +| 13 | 12 | 15 | 11 | +| 14 | 2 | 7, 12 | 6, 8, 9 | +| 15 | 4, 6-13 | 16-22, 24 | — | +| 16 | 1, 5, 15 | 17, 18, 20 | 19 | +| 17 | 15, 16 | 20, 22 | 18, 19 | +| 18 | 15, 16 | 20 | 17, 19 | +| 19 | 1, 15 | 20, 21 | 16, 17, 18 | +| 20 | 15, 16, 17, 18, 19, 24 | — | 21, 22, 23 | +| 21 | 15, 19 | 22 | 20, 23 | +| 22 | 15, 17, 21 | — | 20, 23 | +| 23 | 5, 12, 15 | — | 20, 21, 22 | +| 24 | 15, 16, 17, 18, 19, 21 | 20 | 22, 23 | + +## Todos +> Implementation + Test = ONE todo. Never separate. + +- [x] 1. Vendor senpi contract artifacts, pin.json, and tarball sync script + Recommended task executor category: unspecified-low - multi-file mechanical vendoring + script + What to do / Must NOT do: Fetch the artifact of record: `npm pack @code-yeongyu/senpi@2026.8.19` (record the registry integrity sha512 in pin.json notes). Extract EXACTLY: docs/session-format.md, docs/settings.md, docs/environment-variables.md; ALL 21 hooks .d.ts files from dist/core/extensions/builtin/hooks/ (types, index, schema, trust, trust-storage, config-loader, command-runner, dispatcher, output-parser, output-bounds, safety, matcher, command, handler, diagnostics, plugin-loader, plugin-manifest, lifecycle-adapter, tool-adapter, stop-adapter, prompt-adapter - count must equal 21); AND the implementation files needed because .d.ts signatures alone are insufficient: hooks/trust.js (hash algorithm), hooks/output-parser.js (SYSTEM_MESSAGE_EVENTS, output parsing incl. decision "allow"), hooks/trust-storage.js (lock + scope paths), hooks/output-bounds.js, hooks/types.js. Store under docs/upstream/senpi/ preserving relative names (hooks files under docs/upstream/senpi/hooks/). Write NOTICE (MIT, code-yeongyu/senpi) - no Apache file. Write pin.json: {engineVersion:"2026.8.19", pinnedAt:, source:"npm", registryIntegrity, files:{:{upstreamPath, sha256}}, fixtureRedump, notes[]} with per-file sha256 of the vendored bytes. The pin is COMPLETE at this todo; later todos must never add vendored files post-hoc. Write scripts/sync-upstream-senpi.mjs accepting `node scripts/sync-upstream-senpi.mjs --tarball [--check]`: re-extract, re-hash, compare to pin.json; --check writes nothing and exits non-zero printing `Senpi upstream vendor drift detected: ` on any mismatch; success prints `Senpi upstream vendor is in sync.` Must NOT fetch at test time or import engine code. + Parallelization: Wave 1 | Blocked by: — | Blocks: 4, 5, 16, 19 + References (executor has NO interview context - be exhaustive): docs/upstream/grok/pin.json (schema shape); scripts/sync-upstream-grok.mjs:53-89,324-389 (CLI + drift contract); plans/omo-native-adapter/planning-brief.md "Contract source of truth"; local reference copy: ~/dev-admin/oh-my-openagent/node_modules/@code-yeongyu/senpi (verify version 2026.8.19 in its package.json; tarball is the artifact of record). + Acceptance criteria (agent-executable): `node scripts/sync-upstream-senpi.mjs --tarball --check` exits 0 printing the in-sync line; corrupting one vendored byte makes it exit non-zero naming that file; pin.json covers every vendored file with matching sha256. + QA scenarios (name the exact tool + invocation): happy - run --check against pristine tarball, capture exit 0, Evidence /task-1-check.log; failure - flip one byte in a vendored .d.ts copy, rerun, capture named-file drift error and non-zero exit, Evidence /task-1-drift.log + Commit: Y | feat(senpi): vendor senpi 2026.8.19 contract artifacts with pin and sync script + +- [x] 2. Session v3 Zod schemas in src/senpi/types.ts + Recommended task executor category: unspecified-high - core contract schemas need strictness care + What to do / Must NOT do: Define Zod schemas + z.infer exports for: entry base {type,id,parentId:string|null,timestamp}; session header {type:"session",version,id,timestamp,cwd,parentSession?}; the EXACTLY 9 known non-header entry tags (message, model_change, thinking_level_change, compaction, branch_summary, custom, custom_message, label, session_info) with payloads: message{message:AgentMessage}; model_change{provider,modelId}; thinking_level_change{thinkingLevel}; compaction{summary,tokensBefore,retainedTail?:AgentMessage[],firstKeptEntryId?,usage?,details?,fromHook?}; branch_summary{fromId,summary,...}; custom{customType,data?}; custom_message{customType,content,display,details?}; label{targetId,label}; session_info{name}; AgentMessage union user/assistant/toolResult/bashExecution/custom/branchSummary/compactionSummary with content blocks text/image/thinking{startedAt?,endedAt?}/toolCall and usage{input,output,cacheRead,cacheWrite,totalTokens,cost{input,output,cacheRead,cacheWrite,total}}. Use z.looseObject for additive forward-compat on all entry/message objects. No filesystem access, no hook schemas, no tree logic in this file. + Parallelization: Wave 1 | Blocked by: — | Blocks: 5,6,7,8,10,14 + References: vendored docs/upstream/senpi/session-format.md (field authority); src/grok/validation.ts:1-226 (zod conventions: looseObject, discriminatedUnion); tsconfig.json:31-62 strictness (noUncheckedIndexedAccess, exactOptionalPropertyTypes). + Acceptance criteria: `pnpm run check` passes; every cleanly-parsing JSON example fence in vendored session-format.md round-trips (assert >=10 fences yield >=10 parsed entries so silent under-extraction fails; skip `usage:{...}` ellipsis placeholder fences, which are intentionally invalid JSON); unknown extra fields survive parse. Zod floor: repo pins zod ^4.3.6 - looseObject/discriminatedUnion are zod-v4 APIs. + QA scenarios: happy - fence-parse unit tests/senpi-types.test.ts with >=10-parsed-entries assertion, Evidence /task-2-types.log; failure - compaction missing `summary` parses as kind:'invalid', never throws, same evidence file + Commit: Y | feat(senpi): session v3 entry and message schemas + +- [x] 3. Agent-home resolution in src/senpi/home.ts + Recommended task executor category: quick - single small pure module + What to do / Must NOT do: Implement `resolveSenpiAgentHome(options?: {env?, homeDir?, exists?}): string` with exact precedence: first non-empty of OMO_CODING_AGENT_DIR, SENPI_CODING_AGENT_DIR, PI_CODING_AGENT_DIR (resolved absolute); `/.omo/agent` if it contains settings.json; `/.omo` if it contains settings.json; else `/.senpi/agent`. Pure/injectable. Also export AGENT_HOME_SENTINEL="settings.json" and AGENT_DIR_ENV_NAMES. No session discovery here. + Parallelization: Wave 1 | Blocked by: — | Blocks: 11 + References: ~/dev-admin/oh-my-openagent/packages/omo-senpi/src/components/agent-home/resolve-agent-home.ts (semantic authority); src/grok/processing/discovery.ts getGrokHome (per-call evaluation convention). + Acceptance criteria: vitest table test covering env-wins, sentinel detection, flat fallback, senpi fallback, homeDir injection passes. + QA scenarios: happy - precedence matrix assertions green, Evidence /task-3-home.log; failure - empty-string env value skipped not returned, same evidence + Commit: Y | feat(senpi): agent-home resolution + +- [x] 4. Hooks config validator in src/senpi/settings.ts + Recommended task executor category: unspecified-high - diagnostic vocabulary parity is contract-critical + What to do / Must NOT do: `validateSenpiHooksConfig(json: unknown): SenpiHooksConfig` validating: hooks record keyed by EXACTLY the 7 supported events (PreToolUse, PostToolUse, UserPromptSubmit, SessionStart, PreCompact, PostCompact, Stop); handler groups {matcher?, hooks:[{type:"command", command, commandWindows?, timeout?, statusMessage?}]}; unknown event keys and unsupported handler types (http/prompt/agent/mcp_tool) produce typed diagnostics from the pinned HookDiagnosticCode vocabulary in vendored hooks/types.d.ts + diagnostics.d.ts (invalid_root, invalid_hooks, invalid_event_config, invalid_matcher, invalid_handler_group, invalid_handler_list, invalid_handler, invalid_command, invalid_command_windows, invalid_command_target, missing_command_target, invalid_timeout, invalid_status_message, unknown_event, unsupported_event, unsupported_field, unsupported_handler_type, unsupported_async_handler, unsupported_command_variant). Return {executableHandlers, diagnostics} mirroring ParsedHookConfig. No file discovery, no TOML, no execution. + Parallelization: Wave 1 | Blocked by: 1 | Blocks: 5,15 + References: vendored docs/upstream/senpi/hooks/types.d.ts (CommandHookConfig, SupportedHookEvent, UNSUPPORTED_* consts, HookDiagnosticCode); vendored hooks/schema.d.ts + config-loader.d.ts (config sources: /hooks.json, /.senpi/hooks.json, settings.json hooks keys); src/grok/settings.ts:1-214 (superRefine/transform style). + Acceptance criteria: vitest: valid 7-event config parses; every diagnostic code reproducible by a dedicated malformed input; snake_case/camelCase event keys rejected as unknown_event (config-side takes only the 7 canonical names - unlike Grok aliases). + QA scenarios: happy - full-valid config round-trip, Evidence /task-4-settings.log; failure - one http handler yields unsupported_handler_type diagnostic, not throw, same evidence + Commit: Y | feat(senpi): hooks configuration validator with pinned diagnostics + +- [x] 5. Upstream drift tests in tests/senpi-upstream-drift.test.ts + Recommended task executor category: unspecified-high - two-way extraction logic against vendored sources + What to do / Must NOT do: Parse vendored artifacts, assert BOTH directions against our code: (a) extract SUPPORTED_HOOK_EVENTS / UNSUPPORTED_KNOWN_HOOK_EVENTS / UNSUPPORTED_HANDLER_TYPES from hooks/types.d.ts and compare element-for-element with constants exported from src/senpi; (b) CREATE src/senpi/hook-contract.ts exporting HOOK_INPUT_BRANCHES (per-event required-field manifests copied VERBATIM from the HookInputWire union in vendored types.d.ts - e.g. SessionStart requires camelCase sessionId; permission_mode appears only on UserPromptSubmit; PreCompact has no accepted field; PostToolUse has no transcript_path) plus HOOK_DECISIONS = ["approve","block","deny","ask","allow"]; task 16 implements schemas strictly against this manifest; (c) extract entry `type` literals from vendored session-format.md json fences (regex-based, JSON.parse-independent) and compare with src/senpi/types.ts known-entry tags; (d) assert the vendored hooks .d.ts count equals 21; (e) include one mutation-detection case (tampered string makes the test fail with a named diff). Must NOT read the npm package at test time. + Parallelization: Wave 1 | Blocked by: 1,2,4 | Blocks: 16,23 + References: tests/grok-upstream-drift.test.ts:1-145 (both-directions + mutation pattern); tests/docs-round-trip.test.ts:8-61 (md json-fence extraction). + Acceptance criteria: `pnpm run test:run -- tests/senpi-upstream-drift.test.ts` green; renaming one Zod key intentionally makes it fail naming the mismatch. + QA scenarios: happy - green run log, Evidence /task-5-drift.log; failure - mutation case red with named mismatch, same evidence + Commit: Y | test(senpi): two-way drift tests against vendored 2026.8.19 artifacts + +- [x] 6. Entry parser in src/senpi/processing/parse.ts + Recommended task executor category: quick - single-file parser with fully specified policy + What to do / Must NOT do: `parseSenpiEntry(raw: unknown): SenpiEntryParseResult` returning {kind:'known', entry} for the 9 known non-header tags (message, model_change, thinking_level_change, compaction, branch_summary, custom, custom_message, label, session_info) plus the session header; {kind:'unknown', tag, entry, raw} for any other tag whose base {type,id,parentId,timestamp} validates (unknown entries are tree participants); {kind:'invalid', error, raw} for malformed known tags or base-invalid lines. Tag-peek dispatch per grok policy. No ordering/tree/IO logic. + Parallelization: Wave 2 | Blocked by: 2 | Blocks: 7,12,15 + References: src/grok/processing/events.ts:1-320 (parse policy); src/senpi/types.ts (task 2). + Acceptance criteria: vitest truth table over known/unknown/invalid samples incl. unknown-tag-with-valid-base joining the tree; zero throws on any JSON-decoded input. + QA scenarios: happy - truth table green, Evidence /task-6-parse.log; failure - malformed compaction returns invalid with message, never throws, same evidence + Commit: Y | feat(senpi): known/unknown/invalid entry parser + +- [x] 7. Tree projection in src/senpi/processing/projection.ts + Recommended task executor category: deep - one cohesive hard algorithmic problem, keep whole + What to do / Must NOT do: Implement the approved design: SenpiTreeIndex {byId,parentById,childrenByParent,appendOrder,leafId,structuralLeaves,sessionName,labelsByTargetId}; index rules (reject duplicate id without overwrite; require parent already indexed; leaf = last accepted physical entry regardless of type); `resolveSenpiLeaf`; `projectSenpiBranch(entries, leafId, opts?)` -> root-to-leaf path records with stable keys; latest-compaction handling (retainedTail incl. [] authoritative, synthetic keys retained::; legacy firstKeptEntryId range excluding older compactions; missing/off-path firstKept => warn + incomplete); off-path records disposition off_branch|summarized; cycle/missing-parent guards; warnings array. Pure helper `computeProjectionMutation(prevKeys, nextRecords)` returning {index,deleteCount,records,removedRecordKeys} | null via longest-common-prefix. Pure module: no IO. + Parallelization: Wave 2 | Blocked by: 2,6,14 | Blocks: 10,12 + References: approved algorithm recorded in .omo/drafts/omo-native-adapter.md Findings (ULTRABRAIN design, full interface + edge table); vendored session-format.md "Tree Structure" + "Context Building" sections; fixtures from task 14. + Acceptance criteria: golden tests over synthetic fixtures cover at least 20 design-table edge cases (branch switch, duplicate id, orphan parent, multiple roots, retainedTail:[], dangling tool ref, nested compactions, unicode ids, out-of-order timestamps); LCP mutation property test passes. + QA scenarios: happy - golden snapshot of projected linear history matches expected for the branch-switch fixture, Evidence /task-7-projection.log; failure - cycle input yields invalid result with cycle diagnostic, no hang, same evidence + Commit: Y | feat(senpi): v3 tree index, leaf resolution, compaction-aware projection + +- [x] 8. Internal JSONL cursor in src/senpi/processing/jsonl-cursor.ts + Recommended task executor category: unspecified-high - concurrency/digest edge cases across a file reader + What to do / Must NOT do: Model on grok's cursor but senpi-owned: byte-offset scanning (0x0a), complete-line-only decode, partial-line deferral, inode/device identity, SHA-256 head+boundary digests, generation counter, oversized-line stream-discard diagnostic. Internal only - NOT exported from any barrel. + Parallelization: Wave 2 | Blocked by: — | Blocks: 9,12 + References: src/grok/processing/jsonl-cursor.ts:7-260 (behavioral template); tests/grok-jsonl-cursor.test.ts (edge-case inventory to replicate). + Acceptance criteria: replicated edge cases pass: append mid-line, truncate+regrow same inode, inode swap, oversized line, CRLF, multibyte boundary. + QA scenarios: happy - cursor suite green, Evidence /task-8-cursor.log; failure - same-size rewrite detected via boundary digest mismatch, same evidence + Commit: Y | feat(senpi): internal bounded jsonl cursor + +- [x] 9. Checkpoint marker in src/senpi/processing/checkpoint.ts + Recommended task executor category: quick - small marker module, fully specified + What to do / Must NOT do: Senpi marker schema {sessionPathDigest, sessionId, device, inode, generation, offset, lineNumber, headDigest, boundaryDigest, revision, leafId, projectedRecordKeys[], markerVersion}; senpi-prefixed marker filenames; allowedMarkerRoots validation mirroring src/processing/tail.ts:1817-1870 gate semantics but independently implemented; atomic write via temp+rename under file lock; `commitSenpiSessionCheckpoint`; pure invalidation predicate. Never parses entries or decides branches. + Parallelization: Wave 2 | Blocked by: 8 | Blocks: 12 + References: src/grok/processing/tail.ts:740-820 (grok marker-independence precedent); src/processing/tail.ts:96-104,1817-1870 (root-gate semantics to mirror). + Acceptance criteria: marker round-trip; tampered/malformed marker reports invalid, not throw; custom markerDir outside allowed roots rejected. + QA scenarios: happy - commit+reread equality, Evidence /task-9-checkpoint.log; failure - offset-not-at-line-boundary forces invalidate=true, same evidence + Commit: Y | feat(senpi): revisioned checkpoint markers with root gating + +- [x] 10. Block model + fold in src/senpi/processing/blocks.ts + Recommended task executor category: unspecified-high - native block model + reduce/fold semantics + What to do / Must NOT do: SenpiSessionBlock native model (role, entryId, parentId, branch status, origin entry|retained_tail, content blocks, usage, isError, timestamps, customType passthrough for custom/custom_message metadata blocks); `reduceSenpiProjection(previous, current)` producing upserts/deletes driven by projection mutations; `foldSenpiBlockChanges(changes): SenpiSessionBlock[]`. Stable block ids = entryId (+: where split). Unknown/custom payloads stay metadata. No Claude/Grok type sharing. + Parallelization: Wave 2 | Blocked by: 2,7 | Blocks: 12 + References: src/grok/processing/blocks.ts:1-165,166-410 (reduce/fold shape); planning-brief.md risk 2 (neutral custom density, O(n) fold). + Acceptance criteria: reduce over a branch-splice fixture yields correct upsert+delete sets matching removedRecordKeys; fold is idempotent. + QA scenarios: happy - splice reduction golden, Evidence /task-10-blocks.log; failure - dense custom stream (1000 senpi.todo-state entries) folds in O(n) within test timeout, same evidence + Commit: Y | feat(senpi): native block reduction and folding + +- [x] 11. Discovery + listing in src/senpi/processing/discovery.ts and listing.ts + Recommended task executor category: unspecified-high - discovery + header-verified listing + What to do / Must NOT do: discovery.ts: `getSenpiSessionsRoot(agentHome?)`, dash-encoded cwd dirname encoder (path with / replaced by -; document ambiguity, never decode), `findSenpiSessionDirs(projectCwd, agentHome?)` scanning `/sessions/` for candidate dirs; candidate generation may over-match (encoding ambiguity) because listing verifies headers. Skip `*-artifacts/` directories and per-cwd `extensions/` subdirs. listing.ts: `listSenpiSessions(projectCwd, agentHome?)` and `listAllSenpiSessions(agentHome?)` returning SenpiSessionInfo {path, id, cwd, name?, parentSessionPath?, created, modified, messageCount, firstMessage} (deliberate engine-parity minus allMessagesText) parsed from validated headers; firstMessage from the first user message; messageCount from a streaming full scan that is O(store) time and O(1) memory per file (bounded, no retention) - acceptable for a ~100 MB store; per-file failure isolation returning {kind:'valid'}|{kind:'invalid', error} entries; verify header cwd matches projectCwd for the per-project variant. No folding, no tailing. + Parallelization: Wave 3 | Blocked by: 2,3 | Blocks: 12 + References: src/grok/processing/discovery.ts:10-133 (valid/invalid listing pattern); engine parity shape SessionManager.listAll -> SessionInfo (vendored session-format.md + plans/omo-native-adapter/planning-brief.md); resolveSenpiAgentHome (task 3). + Acceptance criteria: vitest over temp-dir stores: per-project listing returns only header-cwd-matching sessions; listAll returns all; -artifacts and extensions/ excluded; invalid JSONL file surfaces as invalid entry, not throw. + QA scenarios: happy - store with 3 projects lists correctly per-project and all (Evidence /task-11-discovery.log); failure - ambiguous dirname (a-b vs a/b) resolved by header cwd check, same evidence + Commit: Y | feat(senpi): session discovery and listing with header verification + +- [x] 12. Tail orchestration in src/senpi/processing/tail.ts + Recommended task executor category: deep - integration of cursor/projection/blocks, shared insight + What to do / Must NOT do: `tailSenpiSession(file, opts?)` integrating cursor scan -> parseSenpiEntry -> index update -> resolveSenpiLeaf -> projectSenpiBranch -> computeProjectionMutation vs prior projectedRecordKeys -> reduceSenpiProjection -> SenpiSessionTailResult {records, mutations(0..1 splice), offPath, diagnostics, leaf, previousByteOffset, nextByteOffset, fileSize, generation, revision, reset, checkpoint}. Automatic checkpoint mode persists only after full successful parse+projection; manual mode returns checkpoint for caller commit. Checkpoint invalidation per task 9 predicate (inode, size leaf resolution invalid. Never expose cursor/marker types in the result. + Parallelization: Wave 3 | Blocked by: 6,7,8,9,10,11 | Blocks: 13,15,23 + References: approved algorithm + interfaces in .omo/drafts/omo-native-adapter.md Findings; src/grok/processing/tail.ts:120-337 (orchestration shape); fixtures from tasks 11+14. + Acceptance criteria: integration test over branch-switch fixture: first tail emits full path; append cross-branch entry => exactly one splice mutation with correct index/deleteCount; compaction append => splice or append per LCP; reset scenarios emit splice from 0 with reset=true; `pnpm run check` green. + QA scenarios: happy - live-append sequence (write, tail, append, tail) golden log (Evidence /task-12-tail.log); failure - truncated final line is deferred and reread next pass, same evidence + Commit: Y | feat(senpi): leaf-linear session tail with splice mutations + +- [x] 13. Watch + quiescence in src/senpi/processing/watch.ts + Recommended task executor category: unspecified-high - watch + injected-clock quiescence + What to do / Must NOT do: async-generator `watchSenpiSession(file, opts?)`: fs.watch events are wake-up hints only; every wake reconciles via tailSenpiSession; initial readiness yield; quiescence signal after configurable stable-cursor window (option quiescenceMs, default 30000) using injected clock (no fixed sleeps in tests); abort-signal cleanup; missing file retains prior checkpoint, no reset until replacement observed. Never duplicate projection/checkpoint logic. + Parallelization: Wave 3 | Blocked by: 12 | Blocks: 15 + References: src/grok/processing/tail.ts watch section (generator shape); planning-brief.md risk 4 (no SessionEnd; quiescence recipe); vitest fake-timer conventions in tests/grok-tail.test.ts. + Acceptance criteria: injected-clock test: append -> wake -> result; silence for quiescenceMs -> terminal quiescent yield; abort cleans watcher without leaks (handle count assertion). + QA scenarios: happy - append-then-quiet sequence emits quiescence exactly once (Evidence /task-13-watch.log); failure - watch-event storm coalesces to one tail per quiet interval, same evidence + Commit: Y | feat(senpi): watch generator with stable-cursor quiescence + +- [x] 14. Fixtures: sanitized real transcripts + synthetic edge cases + Recommended task executor category: unspecified-low - mechanical redaction + synthetic fixtures + What to do / Must NOT do: Build tests/fixtures/senpi/: (a) 2-3 sanitized real session JSONL files copied from ~/.omo/agent/sessions (pick sessions with branch traffic and compaction; REDACT per category: message text content, tool arguments/outputs, cwd/home paths, session ids AND user-visible strings in every entry type - session_info.name, label.label strings, custom.data payloads replaced with type-preserving synthetic stubs (todo-state content leaks user text) - keep structure, entry types, tree shape, customType density); record redaction script tests/fixtures/senpi/redact.mjs so fixtures are reproducible; (b) synthetic fixtures: branch-switch, retainedTail compaction, legacy firstKeptEntryId compaction, duplicate-id, orphan-parent, multi-root, header-only, empty, unicode cwd. Manifest README.md documenting provenance. Must NOT commit unredacted personal data. + Parallelization: Wave 2 | Blocked by: 2 | Blocks: 7,12 + References: docs/upstream/grok/pin.json fixtureRedump note (redaction precedent); real store ~/.omo/agent/sessions/<--encoded-cwd-->/*.jsonl; entry format from vendored session-format.md. + Acceptance criteria: redact.mjs rerun produces byte-identical fixtures from the same source; every fixture parses with parseSenpiEntry with zero invalid except the intentionally-corrupt ones; privacy gate is a MULTI-SAMPLE grep set, not one substring: machine username, the literal $HOME path prefix, and at least 3 distinct substrings sampled per redaction category (message text, custom.data, session_info.name, label strings) must all return zero matches across every fixture. + QA scenarios: happy - fixtures parse + redaction idempotence log (Evidence /task-14-fixtures.log); failure - privacy grep finds no original substrings (assertion passes = failure scenario avoided; document the negative check), same evidence + Commit: Y | test(senpi): sanitized real and synthetic session fixtures + +- [x] 15. Processing barrel, package export map, and export-surface tests (initial surface) + Recommended task executor category: quick - barrels + export-map edits + test extension + What to do / Must NOT do: Create src/senpi/processing/index.ts (/senpi/processing barrel: types, parse, discovery/listing, projection, tail/watch/checkpoint commit, blocks) and an INITIAL src/senpi/index.ts exporting ONLY what exists after Wave 1-3 (home, session types re-export if desired, settings validator, hook-contract manifest, SenpiHookEventName consts - NO hook wire/runner/builder/trust modules; those land via todo 24). Cursor + marker internals NOT exported. Add exact package.json exports entries "./senpi" and "./senpi/processing" (no wildcards). Extend tests/package-exports.test.ts: exact export lists, cursor absence assertions, root barrel stays senpi-processing-free. Do not re-export from src/index.ts. + Parallelization: Wave 3 | Blocked by: 4,6,7,8,9,10,11,12,13 | Blocks: 16,17,18,19,20,21,22,23,24 + References: package.json exports map (root + subpath entries); src/grok/index.ts:13-58 + src/grok/processing/index.ts:3-57 barrel shape; tests/package-exports.test.ts:116-218 (exact-list + absence assertion pattern, verified). + Acceptance criteria: `pnpm run check` and `pnpm run test:run` green; importing '@libar-dev/agent-harness-kit/senpi/processing' resolves in a vitest smoke import; exported-name snapshot matches approved list exactly. + QA scenarios: happy - export snapshot test green for the initial surface (Evidence /task-15-exports.log); failure - the exact-list assertion rejects any name not in the approved list, including wildcard subpath entries, same evidence + Commit: Y | feat(senpi): public barrels and package export surface + +- [x] 16. Hook wire schemas in src/senpi/hook-wire.ts + Recommended task executor category: unspecified-high - wire union parity with vendored types + What to do / Must NOT do: Single canonical source for the hook wire: INPUT side = 7-event discriminated union whose per-branch fields are copied VERBATIM from the vendored HookInputWire union in hooks/types.d.ts and validated against HOOK_INPUT_BRANCHES from src/senpi/hook-contract.ts (note the asymmetries: SessionStart requires camelCase sessionId; permission_mode only on UserPromptSubmit; PreCompact has no accepted field; PostToolUse has no transcript_path). Accept camelCase primaries AND snake_case aliases (hook_event_name, session_id, tool_name, tool_input, tool_response, tool_use_id, will_retry, custom_instructions), normalizing once at this boundary. OUTPUT side = sourced from vendored output-parser.js / ParsedHookOutput["output"], NOT from types.d.ts HookOutputWire (which has only 6 fields): decision accepts "allow" as well as approve/block/deny/ask; reason, additionalContext, updatedInput, updatedToolOutput, continue; stopReason/suppressOutput/systemMessage are parser-level universal fields gated by SYSTEM_MESSAGE_EVENTS; hookSpecificOutput handled per vendored output-parser.js behavior. Export senpiHookInputSchema/senpiHookOutputSchema, validateSenpiHookInput, z.infer types. No settings/trust/IO. + Parallelization: Wave 4 | Blocked by: 1,5,15 | Blocks: 17,18,20 + References: vendored docs/upstream/senpi/hooks/types.d.ts (HookInputWire/HookOutputWire authority); src/grok/validation.ts:1-226 (looseObject + discriminatedUnion conventions); drift manifest from task 5. + Acceptance criteria: every wire branch round-trips; alias inputs normalize to primary fields; unknown extra fields pass through; drift manifest assertions green. + QA scenarios: happy - per-event fixture envelope validation (hand-authored one JSON per event under tests/fixtures/senpi/hook-inputs/), Evidence /task-16-hook-wire.log; failure - PostToolUse with tool_response alias parses to toolOutput, same evidence + Commit: Y | feat(senpi): hook input/output wire schemas with alias normalization + +- [x] 17. Runner in src/senpi/execute.ts + Recommended task executor category: unspecified-high - runner exit semantics parity + What to do / Must NOT do: `readSenpiStdinJson(options?)` (bounded stdin read, 30s cap like grok), `outputSenpiJson(output)`, `executeSenpiHook(handler, options?)` with injectable stdin/stdout/exit seams. Semantics per VENDORED output-parser.js: exit 2 => {decision:'block', reason:}; otherwise stdout JSON parsed (universal fields continue/stopReason/suppressOutput/systemMessage event-gated to the 5-event SYSTEM_MESSAGE_EVENTS set; hookSpecificOutput per parser behavior); malformed/non-object stdout => diagnostic no-op exit 0; validation failure exit 1 with stderr log. Platform: win32 selects handler.commandWindows when present (mirror selectCommandForPlatform) via injectable platform option; POSIX is the tested path. Never read CLAUDE_* env, never reuse executeHook, never grant trust. + Parallelization: Wave 4 | Blocked by: 15,16 | Blocks: 20,22 + References: vendored hooks/output-parser.js (authoritative: SYSTEM_MESSAGE_EVENTS set, exit-2 rule, allow decision, hookSpecificOutput handling); src/grok/execute.ts:47-76,134-246 (runner shape + seams). + Acceptance criteria: vitest over injected seams: exit-2 block, valid JSON pass-through, malformed stdout no-op, observe events ignore decision JSON; `node -e` pipe smoke test mirrors package.json hook:test pattern. + QA scenarios: happy - scripted stdin/stdout round-trip per event (Evidence /task-17-execute.log); failure - oversized stdin beyond cap truncates per documented policy without hang, same evidence + Commit: Y | feat(senpi): hook command runner with senpi exit semantics + +- [x] 18. Output builder in src/senpi/output-builder.ts + Recommended task executor category: quick - pure factory module + What to do / Must NOT do: `SenpiHookOutputBuilder` pure factories returning schema-valid objects: approve(), block(reason?), deny(reason?), ask(reason?), context(additionalContext), updatedInput(input), updatedToolOutput(output), forceStop(stopReason?), systemMessage(text) (event-gated note in JSDoc), success(message?), error(reason). Every output round-trips senpiHookOutputSchema. No IO, no dispatch. + Parallelization: Wave 4 | Blocked by: 15,16 | Blocks: 20 + References: src/grok/output-builder.ts:24-125 (factory surface style); vendored hooks/types.d.ts HookOutputWire. + Acceptance criteria: every factory output passes senpiHookOutputSchema; JSDoc on every export naming params/returns/consumer-visible behavior. + QA scenarios: happy - factory/schema round-trip table (Evidence /task-18-builder.log); failure - empty deny reason falls back per engine rule (documented in JSDoc + test), same evidence + Commit: Y | feat(senpi): typed hook output builder + +- [x] 19. Trust state read + pure hash parity in src/senpi/trust.ts + Recommended task executor category: unspecified-low - read-only module + golden hash + What to do / Must NOT do: Read/validate HookTrustState v1 from /hooks-state.json and /.senpi/hooks-state.json (scope global|project): {version:1, hooks:{:{enabled,trustedHash?,scope,sourcePath,matcher?,commandPreview,updatedAt}}}; pure `senpiHookTrustId(handler)` and `senpiHashCommandHook(handler, opts?)` reproducing the algorithm in VENDORED hooks/trust.js exactly (the .d.ts has signatures only; the js is already pinned by task 1 - canonical-JSON sha256 ids of form hk____; platform-dependent hash input takes an INJECTED platform option, defaulting process.platform); `readSenpiHookTrustState(path)` with fail-closed malformed handling; `isSenpiCommandHookTrusted(handler, state)`. READ-ONLY: no writes anywhere in this module. + Parallelization: Wave 4 | Blocked by: 1,15 | Blocks: 20,21 + References: vendored hooks/trust.d.ts + trust-storage.d.ts (state shape, scope rules); vendored hooks/trust.js (algorithm authority, pinned by task 1). + Acceptance criteria: golden hash parity test computes id+hash for a fixture handler with a FIXED injected platform value (cross-machine deterministic - never default process.platform in the golden) and asserts the recorded golden; malformed state file => fail-closed result object. + QA scenarios: happy - golden hash parity (Evidence /task-19-trust.log); failure - corrupted hooks-state.json yields fail-closed, not throw, same evidence + Commit: Y | feat(senpi): read-only trust state inspection and hash parity + +- [x] 20. Reference doc docs/reference/senpi-adapter.md + Recommended task executor category: writing - reference documentation + What to do / Must NOT do: Write the full adapter reference mirroring docs/reference/grok-adapter.md structure: scope (attach-only), 7-event table with gate kinds and stdout honored fields, envelope contract (aliases), stdout contract + exit codes, runner, settings validation + config sources + trust gate, session layout + processing APIs table, tree/compaction semantics (persisted-leaf rule, splice mutations, checkpoint fields), pin/drift policy (npm tarball, engineVersion), OmO-vs-senpi naming note (OmO native = branded distribution; engine is senpi), Cockpit seam note. Update docs/README.md index. No prose pinning by tests beyond machine-checkable values. + Parallelization: Wave 4/5 boundary | Blocked by: 15,16,17,18,19,24 | Blocks: — + References: docs/reference/grok-adapter.md (structure template); plans/omo-native-adapter/planning-brief.md; final exported API from tasks 15-19. + Acceptance criteria: doc references only real exported names (spot-checked by grep against barrels); docs/README.md links it; `pnpm run check` green (docs not type-checked but lint may apply). + QA scenarios: happy - link/name audit script passes (Evidence /task-20-docs.log); failure - any exported name in doc missing from barrels is caught by audit grep, same evidence + Commit: Y | docs(senpi): adapter reference + +- [x] 21. Consent-gated hook trust writer in src/senpi/trust-writer.ts + Recommended task executor category: unspecified-high - locking/atomic write safety + What to do / Must NOT do: Exported from the /senpi barrel as `writeSenpiHookTrustEntry(opts)` (todo 15's exact-list includes it) performing an EXPLICIT, caller-authorized write of one trust entry to the scoped hooks-state.json: file lock implemented INTERNALLY (mkdir/O_EXCL retry lock - NO new runtime npm dependency such as proper-lockfile), read-modify-write preserving ALL unrelated/unknown entries, atomic temp+rename, 0600 permissions, fail-closed on malformed existing state, requires an explicit `consent: true` option plus a `reason` string recorded in the entry. Never called from runner/settings/install code paths; no auto-trust. Export type documents that calling it IS the approval act. + Parallelization: Wave 5 | Blocked by: 15,19 | Blocks: 22 + References: vendored hooks/trust-storage.js (scope path logic + lock semantics to replicate internally: /hooks-state.json, /.senpi/hooks-state.json); planning-brief.md risk 3 (managed grant = consent-gated write). + Acceptance criteria: write preserves pre-existing unrelated entries byte-for-byte where untouched; concurrent-writer test (two locked writers) serializes; malformed existing state aborts with no write. + QA scenarios: happy - grant then isSenpiCommandHookTrusted true (Evidence /task-21-trust-writer.log); failure - consent:false rejects with typed error, no file mutation (mtime assert), same evidence + Commit: Y | feat(senpi): consent-gated hook trust writer + +- [x] 22. Forwarder asset variant + hooks.json registration helper + Recommended task executor category: unspecified-high - asset bundling + registration writer + What to do / Must NOT do: (a) senpi forwarder asset: a silent command-hook script (modeled on src/forwarder/hook-forwarder.ts) that reads HookInputWire stdin and POSTs to a configured endpoint, exit 0 always (observe-only; never emits gate JSON); esbuild-bundled like dist/standalone/hook-forwarder.mjs (check src/forwarder/assets.ts + package.json build step and mirror) with --target=node22 (kit runtime floor is Node >=22; do not copy the legacy node18 target). (b) `buildSenpiHooksRegistration(events, command)` producing a hooks.json document for the 7 events (command type only), plus `writeSenpiHooksConfig(path, doc)` atomic write. Must NOT write trust state or enable gates (observe-only registration). + Parallelization: Wave 5 | Blocked by: 15,17,21 | Blocks: 23 + References: src/forwarder/hook-forwarder.ts + src/forwarder/assets.ts + package.json build esbuild step; vendored config-loader.d.ts (hooks.json shape); planning-brief.md hooks surface section. + Acceptance criteria: registration document validates through validateSenpiHooksConfig; forwarder smoke: pipe fixture envelope, assert HTTP POST received by local test server, exit 0. + QA scenarios: happy - end-to-end pipe->local-server capture (Evidence /task-22-forwarder.log); failure - endpoint unreachable still exits 0 silently (observe-only guarantee), same evidence + Commit: Y | feat(senpi): observe forwarder asset and hooks registration helper + +- [x] 23. Cockpit Phase C cross-repo execution brief at plans/omo-native-adapter/cockpit-phase-c.md + Recommended task executor category: writing - cross-repo execution brief + What to do / Must NOT do: Write the execution brief for the libar-cockpit observe adapter (executed in THAT repo under its own planning; this is a spec deliverable in this repo): mirror of src/main/services/grok/* file set (grokDiscovery/grokIngest/grokNormalize/grokRuntime/grokSessionFs/grokIngestSerializer/grokIngestError/grokProcessing + daemonComposition/grokSessionProcessing) mapped to senpi equivalents; harness id decision `omo` in shared schemas; lossy block map from SenpiSessionBlock; dynamic ESM import of /senpi + /senpi/processing; session-end via watch quiescence (no SessionEnd); ADR requirement (cockpit ADR equivalent of 0003) listed as prerequisite; merge-gate note: harness-kit PR merges only after Phase C integration tested. Must NOT include cockpit code changes in this repo. + Parallelization: Wave 5 | Blocked by: 5,12,15 | Blocks: — + References: /Users/darkomijic/dev-libar/libar-cockpit/docs/hooks-contract.md (seam contract); cockpit src/main/services/grok/* (file inventory listed in planning-brief.md Phase C section); planning-brief.md Phase C. + Acceptance criteria: brief names every cockpit file to create/modify with its senpi counterpart and the DaemonSessionSummary mapping; prerequisite ADR + merge gate sections present. + QA scenarios: happy - completeness checklist in brief self-verified against cockpit file listing (Evidence /task-23-phase-c-brief.log); failure - any cockpit grok file without a senpi mapping is listed in a gaps section rather than silently dropped, same evidence + Commit: Y | docs(senpi): cockpit phase-c cross-repo execution brief + +- [x] 24. Finalize /senpi barrel with hooks library exports + Recommended task executor category: quick - extend barrel + exact-list test once hooks modules exist + What to do / Must NOT do: Extend src/senpi/index.ts to its FINAL approved surface: hook wire schemas+validators (16), execute runner + stdin/stdout helpers (17), output builder (18), trust read/pure hash (19), consent-gated writeSenpiHookTrustEntry (21). Update tests/package-exports.test.ts exact list to the final names; keep cursor/marker internals absent. No behavior changes to any module; no wildcard exports. + Parallelization: Wave 5 | Blocked by: 15,16,17,18,19,21 | Blocks: 20 + References: src/senpi/index.ts (initial surface from todo 15); tests/package-exports.test.ts exact-list pattern; approved export list in planning-brief.md hooks surface section. + Acceptance criteria: `pnpm run test:run -- tests/package-exports.test.ts` green against the final name list; smoke import of '@libar-dev/agent-harness-kit/senpi' resolves every exported symbol. + QA scenarios: happy - final export snapshot green (Evidence /task-24-barrel.log); failure - removing one approved name fails the exact-list assertion, same evidence + Commit: Y | feat(senpi): finalize public hooks barrel surface + +## Final verification wave +> Runs in parallel after ALL todos. ALL must APPROVE. Surface results and wait for the user's explicit okay before declaring complete. +- [x] F1. Plan compliance audit + Recommended task executor category: unspecified-high - structural audit scripting + Re-run the structural grammar check (column-zero `- [ ] N.`/`- [ ] F.` rows, category line on every implementation row, numbering continuity, dependency-matrix consistency derived mechanically from Depends-on edges, no cycles) over .omo/plans/omo-native-adapter.md; verify every Success-criteria statement maps to an existing artifact path. Evidence /final-f1-compliance.log. APPROVE only when every check passes with zero manual waivers. +- [x] F2. Code quality review + Recommended task executor category: unspecified-high - adversarial code review + Run `pnpm run check` + full `pnpm run test:run`; then spawn a FRESH unspecified-high adversarial reviewer over `git diff ...HEAD` limited to src/senpi/**, scripts/sync-upstream-senpi.mjs, tests/senpi-*, docs/upstream/senpi/**: verify no-explicit-any, JSDoc on every export, NodeNext .js imports, no dead code. Evidence /final-f2-quality.log. APPROVE only with zero error-severity findings unresolved. +- [x] F3. Real manual QA (agent-executed, read-only) + Recommended task executor category: deep - live-system proof + Against the REAL store ~/.omo/agent/sessions (read-only, nothing copied into the repo): pick one real session with branch traffic; run tailSenpiSession from scratch and after a marker checkpoint; independently recompute expected linear history with a throwaway python jsonl leaf-walk and diff against the kit output; assert first/last user messages and message counts match. Then pipe each fixture envelope in tests/fixtures/senpi/hook-inputs/ through executeSenpiHook via node child process and assert exit codes/outputs match vendored semantics. Evidence /final-f3-live-qa.log. APPROVE only on exact-match diffs. +- [x] F4. Scope fidelity + Recommended task executor category: unspecified-high - guardrail audit + Grep audits over src/ and tests/: zero imports matching @code-yeongyu/senpi or oh-my-openagent; zero imports of src/grok inside src/senpi; zero reads of auth.json; package.json dependencies unchanged from base (no new runtime deps); exports map has exactly ./senpi and ./senpi/processing additions (no wildcards); root barrel untouched. Evidence /final-f4-scope.log. APPROVE only when every grep returns empty/expected. + +## Commit strategy + +- Work branch: `senpi-adapter` off current default branch. This plan's commit strategy authorizes per-todo commits on that work branch (recovery boundaries), each after its QA gate passes. +- One commit per todo, message given in the todo row. No pushes; push only on explicit user request. +- MERGE GATE (owner decision): the kit PR must NOT be merged until Cockpit Phase C (brief = todo 23) is implemented and tested end-to-end in the cockpit repo against this branch. +- Never commit unredacted session data; task 14's privacy grep runs before its commit. + +## Success criteria + +1. `pnpm run check` and `pnpm run test:run` fully green with all new senpi tests. +2. `node scripts/sync-upstream-senpi.mjs --tarball <2026.8.19 tarball> --check` exits 0; drift test mutation case proves the check bites. +3. Golden integration proof: tail over a sanitized real fixture reproduces the expected linear leaf-path history including one cross-branch splice and both compaction forms. +4. Export surface exact: `/senpi` + `/senpi/processing` barrels match approved lists; cursor/marker internals absent; root barrel unchanged. +5. Hook contract parity: drift tests pass both directions against vendored artifacts; runner exit-code semantics match vendored output-parser behavior. +6. Trust safety: read-only paths cannot write; trust writer refuses without explicit consent; unrelated entries preserved. +7. Deliverables complete: reference doc indexed, cockpit Phase C brief written, no Must-NOT-have violations (final wave F4 audits). diff --git a/AGENTS.md b/AGENTS.md index dd7a3a4..ef47c0d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # Agent harness kit -`@libar-dev/agent-harness-kit` is a TypeScript library for Claude Code hooks, session export/tail CLIs, and a Grok Build adapter. Claude Code has 30 hook events. `CLAUDE.md` is a symlink to this file. Edit this file. +`@libar-dev/agent-harness-kit` is a TypeScript library for Claude Code hooks, session export/tail CLIs, and attach-only Grok Build and OmO-native (senpi) observe adapters. Claude Code has 30 hook events. `CLAUDE.md` is a symlink to this file. Edit this file. ## `any` @@ -22,6 +22,7 @@ Imports use `.js` extensions (NodeNext). | [docs/guides/configuring-settings-json.md](docs/guides/configuring-settings-json.md) | handler types, matcher, `if` / `once` / `timeout` | | [docs/guides/writing-your-first-hook.md](docs/guides/writing-your-first-hook.md) | `executeHook` module pattern | | [docs/reference/grok-adapter.md](docs/reference/grok-adapter.md) | Grok envelopes, settings, or processing | +| [docs/reference/senpi-adapter.md](docs/reference/senpi-adapter.md) | senpi hook input/output shapes, trust model, forwarder, or tail/watch APIs | | [docs/internal/tail-session.md](docs/internal/tail-session.md) | tail markers or `CLAUDE_TAIL_MARKER_ROOTS` | | [docs/upstream/hooks-reference.md](docs/upstream/hooks-reference.md) | mirrored official hook contract | | [tests/docs-round-trip.test.ts](tests/docs-round-trip.test.ts) | changing JSON examples in `docs/upstream/hooks-*.md` | @@ -50,7 +51,7 @@ Greptile reviews this public repo. After a commit: `greptile review -b main --js ## Public tree -Keep scratch out of the index: `prometheus-implementation-context.md`, `.omo/notepads/`, `.omo/senpi-task/`, `.omo/start-work/`, `.omo/run-continuation/`, `boulder.json`, root `plans/`, `.grok/`. Product law goes in `docs/` or `docs/decisions/`. +Keep scratch out of the index: `prometheus-implementation-context.md`, `.omo/notepads/`, `.omo/senpi-task/`, `.omo/start-work/`, `.omo/run-continuation/`, `boulder.json`, root `plans/`, `.grok/`. Product law goes in `docs/` or `docs/decisions/`. The `@noble/hashes` BLAKE3-parity exception is `docs/decisions/0001-noble-hashes-blake3-parity.md`. `.omo/` is live. At most one unchecked plan in `.omo/plans/`. Archive to `.plans/NN-slug.md`. Workstation copy: `~/.agents/AGENTS.md` (skill `omo-workspace-state`). @@ -60,6 +61,6 @@ Unslop every reply, commit message, PR body, and new doc. Skill: `~/.agents/skil Commits are recovery boundaries. A plan's commit strategy authorizes commits on that work branch. Otherwise ask. Push only when asked. No `git stash`. -Before a push, the remote must be `git@github.com:/.git` and `gh auth status` must report `Git operations protocol: ssh`. Ask before changing remotes or credentials. +Push over the existing remote and protocol - HTTPS + gh token works here. Treat SSH + key checks as a fallback only when HTTPS/token auth fails (the old ssh-first rule came from a Fedora machine where tokens kept breaking, not this one). Ask before changing remotes or credentials. The user owns `~/dev-admin/oh-my-openagent` and `~/.omo/omo.jsonc`. Inspect and report. Do not checkout, pull, build, install, or edit OmO unless asked. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7521546..8c319f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,121 @@ Categories per release: **Added**, **Changed**, **Deprecated**, **Removed**, **F ### Added +- Exported `byteCursorsEqual` from the Grok processing barrel so + consumers can compare per-source cursors without an internal barrel. + Contract tests pin `StaleCheckpointConflict` shape (`name`, + constructor fields, and `isStaleCheckpointConflict`) as a cross-repo + contract. + +### Changed + +- Token-lease lock protocol replaces the rename-based claim/release + mechanism at the marker, trust, and raw-transcript lock sites. + Canonical lock directories hold immutable `owner..` + token files. Expiry is age-based (`now - token.mtimeMs > staleMs`). + The trust lock is a directory (`.lock`), not a file; + pre-existing FILE-shaped locks are still captured when mtime-stale. + Pre-protocol sibling debris (`.release.*` / `.reclaim.*` / `.stale.*`) + is inert: the protocol never creates those names, and leftover debris + is left alone (no automatic cleanup). Renewal is explicit + (`Lease.renew()` / `assertHeld()`) at commit points, not a heartbeat. + `removeStaleStateLock` in `src/senpi/trust-writer.ts` is async + (`Promise`). Direct importers of that module should treat + the return as a Promise; the symbol is not on the package barrel. +- Relocated the shared JSONL cursor, incremental resume, watch scheduler, + discovery primitives, and bounded-line reader into unbarreled + `src/internal/`. Type-only `JsonlCursor` re-exports from the Grok and + Senpi processing barrels keep the public surface unchanged. +- Timing-sensitive CLI, Senpi trust-writer, and lifecycle tests use + injected clocks and fake timers instead of wall-clock waits. The + trust writer accepts an optional `clock` so lock-budget assertions + never sleep. + +### Fixed + +- Grok tail full-state rebuilds ignore the steady-state scan bounds + (`maxScanBytes`/`maxScanLines`; per-line `maxLineBytes` still applies), and a + rebuild that still cannot complete returns a non-advancing result + (additive `stateComplete: false`, checkpoint rewound to previous cursors) + instead of persisting a checkpoint past incomplete reconstructed state. +- Marker lock and Senpi trust-state lock releases and stale reclamations + claim the lock atomically (rename to a private uuid path) and verify the + ownership token there before removal: a releasing owner whose stale lease + was reclaimed no longer deletes the replacement's lock, and a reclaimer + that races a fresh replacement restores it instead of deleting it. +- Clean-consumer pack parsing survives npm < 11 running the `prepare` script during `npm pack` despite `--ignore-scripts` (npm/cli#3080): the JSON report is extracted from stdout instead of parsed from byte zero. Same fix in the packed package contract test. +- Senpi execute real-signal cases spawn the self-killing child through `exec` so the observed close is signal death on shells that fork and wait (dash). +- Marker-lock replacement case no longer asserts an inode number difference; ext4/overlayfs reuse inodes after rm+mkdir, and the nonce mismatch is the property under test. +- Compatibility probe tolerates missing git metadata in codeload/tarball installs: `candidateSha` falls back to `unknown` when `git rev-parse HEAD` fails, so `prepack` no longer fails on archive pins. +- Bound stdin to 1 MiB in Senpi execute, Grok execute, both hook + forwarders, and Claude `readStdin`. +- Bound shared JSONL cursor scans to 32 MiB and 10_000 lines per pass. + Senpi and Grok tail results expose additive `scanStatus` (`complete` or + `limited`) when a pass stops early. +- Senpi tail reports `reset: false` on a first-ever scan, treats movement + as cursor and projection position change, and skips complete blank lines + so a trailing blank is not `terminalMalformed`. +- Senpi automatic checkpoint writes return additive `checkpointStatus` + (`committed`, `unchanged`, `manual`, `failed`, or `deferred`) instead of + throwing on write failure. +- Senpi watch quiesces on position stability. Reset, mtime, size beyond + the cursor, and filesystem noise no longer count as movement or re-arm + the quiet window. +- Grok watch wakes on null-filename `fs.watch` events and supports an + optional `pollMs` backstop. +- Bound Grok discovery reads of `summary.json` (64 KiB) and `.cwd` (4 KiB). + Oversized files take the existing invalid or diagnostic path. +- Senpi listing never yields or parses an unterminated final line. The + newline commits the entry on the next listing pass. +- `listAllSenpiSessions` skips root-level `*-artifacts/` directories, + matching the nested-directory filter. +- Senpi registration-document reads reject unknown hook-event keys, + groups without handlers, empty commands, and arbitrary hooks objects. + Inspect surfaces the existing malformed-shape read-error union. + +### Security + +- Senpi hook forwarder accepts only loopback `http`/`https` URLs + (`127.0.0.1`, `localhost`, `::1`) unless + `SENPI_HOOK_FORWARD_ALLOW_REMOTE=1` is set. Redirects are refused and + stdin is capped at 1 MiB. +- Marker-root containment realpath-walks the candidate and allowed + roots so a symlink inside a root that points outward is rejected. + Write-path re-verification (`assertMarkerDirStillAllowed`) rejects a + directory swapped for an outward symlink after resolve. +- Marker-lock stale reclamation writes `owner.json` `{ nonce }` after + `mkdir`, atomically rename-claims a stale lock, and re-checks `{dev, ino}` + plus nonce after the claim. Exactly one concurrent reclaimer can remove a + captured stale lock, and a recreated lock is restored rather than deleted. + +## [0.3.0] - 2026-08-23 + +First public contract freeze. This release documents the Claude, Grok, and +Senpi library surfaces that ship in the package. It is not a desktop product +integration and does not publish itself; npm publication with provenance is a +separate owner-authorized step. + +Supported runtime: Node.js `>=22.0.0`. Development and CI also run on Node 24. +Node 20 and Windows are not supported. + +### Added + +- Added the attach-only Grok Build surface on `/grok` (hook validation, output + builder, runner, JSON/TOML settings) and `/grok/processing` (discovery, parse, + tail, watch, checkpoint commit, block reduction). +- Added the attach-only OmO-native (senpi) surface on `/senpi` (agent-home + resolution, hook wire/validation, output builder, runner, read-only trust + inspection, consent-gated trust grant/revoke, observe-only hooks.json + register/inspect/unregister) and `/senpi/processing` (discovery, listing, + parse, projection, tail, watch, checkpoint commit, native block reduction). + Mutating primitives require `{ consent: true, reason, target }` and never + run at module import. +- Added the standalone Senpi hook forwarder at + `dist/standalone/hook-forwarder-senpi.mjs` alongside the existing Claude + forwarder at `dist/standalone/hook-forwarder.mjs`. `/forwarder` exports the + pack-relative asset paths and `RUN_HOOK_WRAPPER_SH`; it does not install. +- Added a packed-tarball clean-consumer CI matrix on Node 22 and 24, plus a + Node 20 engine-mismatch job that must surface `engines.node >=22.0.0`. - Added Claude Code hook parity for optional `prompt_id`, eight Notification types, Stop/SubagentStop background-task and session-cron registries, six permission-update variants, the `manual` set-mode alias, `disableAllHooks`, @@ -30,6 +145,8 @@ Categories per release: **Added**, **Changed**, **Deprecated**, **Removed**, **F ### Changed +- Package version is exact `0.3.0`. `publishConfig.provenance` is enabled so + the owner-gated release workflow can attest the npm tarball. - Notification output is restricted to universal hook fields. - Stop and SubagentStop block outputs require a present `reason` string (empty string accepted) and remain distinct from non-error `additionalContext` @@ -50,6 +167,19 @@ Categories per release: **Added**, **Changed**, **Deprecated**, **Removed**, **F - `sessionStartContext(options)` preserves empty strings and empty `watchPaths` via presence checks rather than truthiness. +### Compatibility + +- Runtime floor remains Node.js `>=22.0.0`. It is not raised and not broadened. +- Grok and Senpi are observe/attach contracts only. This library does not spawn, + drive, Commit, or translate Claude hook scripts to those engines. +- Claude `/processing` stays the session parse/tail/export surface. Marker + helpers remain public; JSONL cursor internals and Senpi marker-schema helpers + stay unexported and resolve as `ERR_PACKAGE_PATH_NOT_EXPORTED`. +- There is no public plugin ABI and no shared cross-harness `SessionBlock`. +- This package is a library. It does not ship a Cockpit (or any other) product + integration, daemon, or UI. Cockpit is observe-only for OmO/Senpi and must + not call kit hook/trust writers or install the Senpi forwarder. + ### Fixed - Canonicalized existing hook-event working directories before project-root @@ -180,4 +310,7 @@ Development milestone for `@libar-dev/claude-code-hooks` before the first public - Exact `--format raw-records` payloads and `rawLine` bytes are gated behind the explicit `--unsafe-raw-unredacted` opt-in; without it, raw records are redacted. +[Unreleased]: https://github.com/libar-dev/agent-harness-kit/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/libar-dev/agent-harness-kit/releases/tag/v0.3.0 +[0.2.0]: https://github.com/libar-dev/agent-harness-kit/releases/tag/v0.2.0 [0.1.0]: https://github.com/libar-dev/agent-harness-kit/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 31f1d87..7b6c058 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # @libar-dev/agent-harness-kit -> **Today: a Claude Code toolkit. Future: harness-agnostic.** +> Claude Code hooks, plus attach-only Grok Build and OmO-native (senpi) observe adapters. > -> This library currently targets [Claude Code hooks](https://docs.anthropic.com/en/docs/claude-code/hooks). The long-term goal is to generalize the harness layer so the same validators, builders, and session tooling work across multiple agent platforms. Claude-specific API names (event names, CLI binaries) will remain stable. +> This library validates hook I/O and reads on-disk session files. It does not start, drive, or Commit external engines, and it is not a desktop product integration. Cockpit is observe-only for OmO/Senpi: library mutation helpers exist for explicit owners, not for Cockpit product wiring. Claude-specific API names (event names, CLI binaries) remain stable and are not translated to Grok or Senpi. [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -[![npm version](https://img.shields.io/badge/version-0.1.0-blue)](https://github.com/libar-dev/agent-harness-kit/releases) +[![npm version](https://img.shields.io/badge/version-0.3.0-blue)](https://github.com/libar-dev/agent-harness-kit/releases) [![Node ≥22](https://img.shields.io/badge/node-%3E%3D22-green)](package.json) TypeScript library for [Claude Code hooks](https://docs.anthropic.com/en/docs/claude-code/hooks) with strict types, Zod-validated inputs, and a fluent output builder for all 30 hook events. @@ -81,6 +81,12 @@ echo '{"hook_event_name":"PreToolUse","session_id":"s1","transcript_path":"/tmp/ | `@libar-dev/agent-harness-kit/post-tool-use` | Reference handlers: Prettier formatter, TypeScript checker | | `@libar-dev/agent-harness-kit/lifecycle` | Reference handlers: setup, session start/end, notifications, message display, stop, subagents, elicitation | | `@libar-dev/agent-harness-kit/processing` | Session parsing, full-history reads, structured exports, and multi-source raw transcript tailing | +| `@libar-dev/agent-harness-kit/grok` | Attach-only Grok Build hook validation, output builder, and runner | +| `@libar-dev/agent-harness-kit/grok/processing` | Grok session discovery, parse, tail, watch, and checkpoint commit | +| `@libar-dev/agent-harness-kit/senpi` | Attach-only OmO-native (senpi) hooks, trust inspect/grant/revoke, and consent-gated hooks.json register/inspect/unregister | +| `@libar-dev/agent-harness-kit/senpi/processing` | Senpi session discovery, projection, tail, watch, and checkpoint commit | +| `@libar-dev/agent-harness-kit/endpoint-discovery` | Validated hook-endpoint file contract and URL helpers | +| `@libar-dev/agent-harness-kit/forwarder` | Standalone forwarder asset paths plus the managed POSIX wrapper string |} ## Documentation @@ -99,6 +105,14 @@ echo '{"hook_event_name":"PreToolUse","session_id":"s1","transcript_path":"/tmp/ The package also attaches to Grok Build through the `@libar-dev/agent-harness-kit/grok` and `/grok/processing` subpaths: Grok-native hook validation, output building, and a runner for Grok's 15 hook events (14 wire events plus the legacy `subagent_end` alias), settings validation for JSON and TOML hook config, and discovery, parsing, and tailing of Grok's on-disk session files. Scope is attach-only; the library answers hook calls and reads session logs but never starts or drives Grok. Claude hook scripts do not run correctly under Grok; write a Grok-native entrypoint instead. See the [Grok Adapter Reference](docs/reference/grok-adapter.md) for the event list, wire contracts, and the Grok-vs-Claude incompatibility matrix. +## Senpi / OmO (third harness) + +`@libar-dev/agent-harness-kit/senpi` is an attach-only library surface. It can validate hook envelopes, inspect trust, and — only when a caller passes an explicit `{ consent: true, reason, target }` options object — write or remove `hooks.json` / `hooks-state.json`. Those writers never run at import and never default to `~/.omo`. The caller owns the target directory, the consent record, and any later uninstall. + +That library capability is not a Cockpit product integration. **Cockpit is observe-only for OmO/Senpi:** it may resolve the agent home and read session files, but it must not register hooks, grant or revoke trust, install the Senpi forwarder, or enforce a Senpi Stop gate. See the [Senpi Adapter Reference](docs/reference/senpi-adapter.md). + +Standalone assets live at `dist/standalone/hook-forwarder.mjs` (Claude) and `dist/standalone/hook-forwarder-senpi.mjs` (Senpi observe-only). `@libar-dev/agent-harness-kit/forwarder` exports those pack-relative paths plus `RUN_HOOK_WRAPPER_SH`. Shipping the bytes is not an install. + ## Development Use Node 24 for local development to match the repo's `@types/node` baseline and CI matrix. Published runtime support remains Node 22+. @@ -115,7 +129,7 @@ pnpm run build # compile src/ → dist/ (publish only) ## Status -Version `0.1.0` is the initial npm release candidate. The public API is stable but may change before the first published release. Pin to a commit hash if you depend on this from another project. +Version `0.3.0` is the public contract freeze: Claude hooks and processing, attach-only Grok and Senpi observe adapters, and standalone forwarders. Runtime support is Node.js 22 and newer. Publishing to npm with provenance is a separate owner-authorized step; see [Release 0.3.0](docs/internal/release-0.3.0.md). ## Contributing diff --git a/docs/README.md b/docs/README.md index bf1e079..692af52 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ | [Types](reference/types.md) | Full type catalogue — inputs, outputs, tools, config | | [Environment Variables](reference/environment-variables.md) | Every `CLAUDE_HOOK_*` variable with type, default, and description | | [Grok Adapter](reference/grok-adapter.md) | Grok Build envelopes, settings, and session processing (attach-only) | +| [Senpi Adapter](reference/senpi-adapter.md) | OmO-native (senpi) envelopes, consent-gated register/trust, forwarder assets, and session processing. Cockpit is observe-only. | ## Architecture & Internal @@ -30,6 +31,7 @@ | [API Update Checklist](internal/api-update-checklist.md) | Maintainer checklist for tracking Claude Code API changes | | [Export Sessions Script](internal/export-sessions.md) | How the export-sessions utility works | | [Session Tailing](internal/tail-session.md) | CLI and library APIs, marker offsets, multi-source session discovery, and live polling | +| [Release 0.3.0](internal/release-0.3.0.md) | Public contract, Node `>=22`, provenance publish steps (owner-gated) | ## Upstream Reference (Mirrored) diff --git a/docs/decisions/0001-noble-hashes-blake3-parity.md b/docs/decisions/0001-noble-hashes-blake3-parity.md new file mode 100644 index 0000000..d7ad8d2 --- /dev/null +++ b/docs/decisions/0001-noble-hashes-blake3-parity.md @@ -0,0 +1,23 @@ +# ADR 0001: @noble/hashes for grok BLAKE3 parity + +Status: accepted. + +## Context + +`package.json` `dependencies` lists `@noble/hashes` at `^2.3.0` beside `zod`. The kit otherwise refuses new runtime dependencies. + +Grok session discovery must land long working-directory names in the same filesystem slots as grok-build. `encodeGrokCwdDirname` in `src/grok/processing/discovery.ts` URL-encodes `cwd`. Encoded names that exceed 255 bytes become `-`. That digest has to match upstream `encode_cwd_dirname`. SHA-256 does not. + +`docs/upstream/grok/pin.json` notes record the choice and defer this write-up: + +> The blake3 implementation decision for session discovery (@noble/hashes) is recorded separately during execution. + +> Session discovery (src/grok/processing/discovery.ts) uses @noble/hashes for BLAKE3 (audited, ESM, zero runtime dependencies) so >255-byte CWD directory names exactly match upstream encode_cwd_dirname; SHA-256 is not compatible. + +## Decision + +Keep `@noble/hashes` as a runtime dependency. `encodeGrokCwdDirname` imports `blake3` from `@noble/hashes/blake3.js` and takes the first 16 hex characters of `blake3(cwd)`. This file is the documented exception to the no-new-runtime-dependencies culture. BLAKE3 is not replaced with SHA-256 or another digest. + +## Consequences + +Long-CWD session directories stay locatable against grok-build. Other runtime packages still need their own decision record. The pin.json notes above are fulfilled by this file. diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index b85ca91..b3633cb 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -125,6 +125,13 @@ The package exposes several sub-paths so you only import what you need: | `@libar-dev/agent-harness-kit/pre-tool-use` | Reference PreToolUse handlers | | `@libar-dev/agent-harness-kit/post-tool-use` | Reference PostToolUse handlers | | `@libar-dev/agent-harness-kit/lifecycle` | Reference lifecycle handlers | +| `@libar-dev/agent-harness-kit/processing` | Claude session parse, export, and raw transcript tailing | +| `@libar-dev/agent-harness-kit/grok` | Attach-only Grok Build hook I/O | +| `@libar-dev/agent-harness-kit/grok/processing` | Grok session discovery and tailing | +| `@libar-dev/agent-harness-kit/senpi` | Attach-only OmO-native (senpi) hook I/O and trust helpers | +| `@libar-dev/agent-harness-kit/senpi/processing` | Senpi session discovery, projection, and tailing | +| `@libar-dev/agent-harness-kit/endpoint-discovery` | Hook-endpoint file contract | +| `@libar-dev/agent-harness-kit/forwarder` | Standalone forwarder wrapper asset | ## What Next? diff --git a/docs/internal/release-0.3.0.md b/docs/internal/release-0.3.0.md new file mode 100644 index 0000000..7c9d9a8 --- /dev/null +++ b/docs/internal/release-0.3.0.md @@ -0,0 +1,80 @@ +# Release 0.3.0 + +Preparation notes for `@libar-dev/agent-harness-kit@0.3.0`. This document does +not authorize publication. + +## Version + +Publish the exact version `0.3.0`. Do not retag a different number onto these +bytes. + +## Runtime + +`engines.node` is `>=22.0.0`. CI verifies Node 22 and Node 24, including a +clean-consumer job that packs the tarball and installs it with scripts enabled. +A separate engine-mismatch job installs that tarball under Node 20 and requires +the declared engine mismatch to be surfaced. Development uses Node 24 without +raising the consumer floor. + +Do not advertise Node 20, browsers, Deno, or Windows. + +## Public contract + +Documented package subpaths: + +| Subpath | Role | +| --- | --- | +| `.` | Claude types, utils, and validation | +| `./types` `./utils` `./validation` | Claude hook I/O | +| `./pre-tool-use` `./post-tool-use` `./lifecycle` | Claude reference handlers | +| `./processing` | Claude session parse, export, and raw transcript tail | +| `./grok` `./grok/processing` | Attach-only Grok Build observe/hook surface | +| `./senpi` `./senpi/processing` | Attach-only OmO-native (senpi) observe/hook surface. Mutating register/trust helpers are consent-gated library primitives, not a Cockpit integration. | +| `./endpoint-discovery` | Hook-endpoint file contract | +| `./forwarder` | Standalone forwarder asset paths and POSIX wrapper string. Not an install. | + +Packed bytes must include compiled runtime (`.js`), declarations (`.d.ts`), +`LICENSE`, `README.md`, `CHANGELOG.md`, and the standalone forwarders: + +- `dist/standalone/hook-forwarder.mjs` +- `dist/standalone/hook-forwarder-senpi.mjs` + +Undocumented deep paths (`./processing/internal`, `./senpi/trust`, cursor +modules) must fail Node resolution with `ERR_PACKAGE_PATH_NOT_EXPORTED`. + +## Observe and processing boundaries + +Grok and Senpi are attach/observe contracts. Callers may validate hook +envelopes, answer hook stdin, and read on-disk session files. This library does +not spawn, drive, Commit, or translate Claude hook scripts to those engines. + +Claude `/processing` remains a separate session pipeline. There is no public +plugin ABI and no shared cross-harness `SessionBlock`. + +This package is a library. It does not ship a Cockpit (or any other) product +integration, daemon, or UI. Cockpit is observe-only for OmO/Senpi: it must not +register hooks, grant trust, install the Senpi forwarder, or enforce a Senpi +Stop gate. Callers of mutating primitives own consent, the target directory, +and uninstall. + +## Provenance (do not run until authorized) + +The Release workflow is inert until an `NPM_TOKEN` repository secret exists. It +never runs on push to a branch. Publication is owner-authorized only. + +When authorized, the intended path is: + +1. Confirm `package.json` `version` is exactly `0.3.0`. +2. Confirm the candidate SHA and packed file list match the release evidence. +3. Push an annotated `v0.3.0` tag, or dispatch `.github/workflows/release.yml`. +4. Let GitHub Actions run `npm publish --provenance --access public` on Node 22. + +Requirements already encoded in the workflow: + +- `permissions.id-token: write` for npm provenance +- `permissions.contents: read` +- `NPM_TOKEN` secret (workflow exits 1 when missing) +- `publishConfig.access: public` and `publishConfig.provenance: true` + +Do not publish from a laptop. Do not use `npm publish` without `--provenance`. +Do not npm-tag, git-tag, or push as part of contract preparation. diff --git a/docs/reference/senpi-adapter.md b/docs/reference/senpi-adapter.md new file mode 100644 index 0000000..66a15cd --- /dev/null +++ b/docs/reference/senpi-adapter.md @@ -0,0 +1,405 @@ +# Senpi Adapter Reference + +OmO-native (senpi engine) support in `@libar-dev/agent-harness-kit/senpi` and `@libar-dev/agent-harness-kit/senpi/processing`. + +**Sources:** [`src/senpi/`](../../src/senpi/index.ts), [`src/senpi/processing/`](../../src/senpi/processing/index.ts), vendored upstream contract files under [`docs/upstream/senpi/`](../upstream/senpi/NOTICE) + +**Scope:** attach-only. The library answers senpi hook calls and reads senpi's on-disk session files. It does not start, spawn, drive, or RPC-control senpi sessions. Observe never drives. + +## Events and gate kinds + +Senpi configuration accepts exactly 7 events. Config keys and the wire discriminator `event` use canonical PascalCase only. Unlike Grok, there are no snake_case or camelCase event-name aliases on the config side. + +| Wire value (`event`) | Gate kind | stdout honored | +| -------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| `PreToolUse` | Tool gate | Yes: `decision` (`allow`/`approve`/`ask`/`deny`), `reason`, `additionalContext`, `updatedInput` (only when allow), `systemMessage` | +| `PostToolUse` | Block gate | Yes: `decision` (`block` only), `reason`, `additionalContext`, `updatedToolOutput`, `systemMessage` | +| `UserPromptSubmit` | Block gate | Yes: `decision` (`block` only), `reason`, `additionalContext`, `systemMessage` | +| `SessionStart` | Observe | `additionalContext`, `systemMessage` (any `decision` is rejected with a warning) | +| `PreCompact` | Observe | No (gate-out: no decision, context, or `systemMessage`) | +| `PostCompact` | Observe | No (gate-out: no decision, context, or `systemMessage`) | +| `Stop` | Stop gate | Yes: `decision` (`block` or `continue`), `reason`, `additionalContext`, `continue`, `stopReason`, `systemMessage` | + +`PreToolUse` is the only Tool gate. `PostToolUse` and `UserPromptSubmit` honor only `decision: "block"`. `Stop` is the Stop gate (`continue: false` implies `decision: "block"`). `SessionStart`, `PreCompact`, and `PostCompact` are Observe: gate decisions are ignored or unsupported. + +The exported `SENPI_HOOK_EVENT_NAMES` array lists the 7 canonical names. `SENPI_UNSUPPORTED_HOOK_EVENT_NAMES` lists upstream event names this kit rejects as `unsupported_event`. `senpiHookInputSchema` validates envelopes for each supported event. + +## Envelope contract + +Envelopes are JSON objects read from stdin. Primaries are camelCase. Optional snake_case aliases exist on the same object and are normalized once at the wire boundary by `validateSenpiHookInput` / `senpiHookInputSchema` (alias fills a missing primary; an explicit primary always wins). + +| Alias (snake_case) | Primary (camelCase) | +| ------------------ | ------------------- | +| `hook_event_name` | `event` | +| `session_id` | `sessionId` | +| `tool_name` | `toolName` | +| `tool_input` | `toolInput` | +| `tool_response` | `toolOutput` | + +Schemas are `z.looseObject`, so unknown extra fields pass through. Per-event required fields (from `HOOK_INPUT_BRANCHES`) and known asymmetries: + +| Event | Required primaries | Notes | +| ------------------ | -------------------------------------------- | ------------------------------------------------------------------------- | +| `SessionStart` | `sessionId`, `cwd` | camelCase `sessionId` is required; `session_id` is only an optional alias | +| `UserPromptSubmit` | `prompt`, `cwd` | `permission_mode` appears only on this event | +| `PreToolUse` | `toolName`, `toolInput`, `cwd` | `tool_use_id` stays snake_case (no camelCase primary) | +| `PostToolUse` | `toolName`, `toolInput`, `toolOutput`, `cwd` | No `transcript_path`; `tool_response` aliases to `toolOutput` | +| `PreCompact` | `reason`, `cwd` | No `accepted` field | +| `PostCompact` | `reason`, `cwd` | Optional `accepted` boolean | +| `Stop` | `cwd` | Optional `stopReason` | + +Example `PreToolUse` envelope: + +```json +{ + "event": "PreToolUse", + "toolName": "bash", + "toolInput": { "command": "ls" }, + "cwd": "/Users/dev/project", + "session_id": "sess-123" +} +``` + +After normalization, `session_id` is copied into `sessionId` when the primary was absent. `HOOK_INPUT_BRANCHES` is the drift-pinned field manifest; schemas implement it strictly. + +## stdout contract + +Gate and block events read one JSON object from handler stdout. Output shape is sourced from the vendored output parser (not the thinner `HookOutputWire` .d.ts). Decision vocabulary is `HOOK_DECISIONS`: `approve`, `block`, `deny`, `ask`, `allow`. + +Tool gate (`PreToolUse`): + +| Field | Type | Notes | +| ------------------- | ------------------------------------ | ------------------------------------------------------------- | +| `decision` | `allow`, `approve`, `ask`, or `deny` | `block` collapses to `deny`; unknown values are dropped | +| `reason` | string, optional | Blank/non-string reasons are dropped (no default substituted) | +| `additionalContext` | string, optional | Honored when nonblank | +| `updatedInput` | unknown, optional | Applied only when the permission decision is `allow` | +| `systemMessage` | string, optional | Honored on this event | + +```json +{ "decision": "deny", "reason": "command not allowed" } +``` + +Block gates (`PostToolUse`, `UserPromptSubmit`): + +| Field | Type | Notes | +| ------------------- | ---------------- | ----------------------------------- | +| `decision` | `block` only | Any other decision yields a warning | +| `reason` | string, optional | Dropped when blank | +| `additionalContext` | string, optional | Honored when nonblank | +| `updatedToolOutput` | unknown | `PostToolUse` only | +| `systemMessage` | string, optional | Honored on these events | + +Stop gate (`Stop`): + +| Field | Type | Notes | +| ------------------- | --------------------- | ---------------------------------- | +| `decision` | `block` or `continue` | Other values warn and are dropped | +| `continue` | boolean | `false` forces `decision: "block"` | +| `stopReason` | string, optional | Paired with force-stop | +| `reason` | string, optional | Feedback on block | +| `additionalContext` | string, optional | Honored when nonblank | +| `systemMessage` | string, optional | Honored on Stop | + +```json +{ + "decision": "block", + "reason": "tasks remain open", + "additionalContext": "2 tasks incomplete" +} +``` + +```json +{ "continue": false, "stopReason": "operator requested halt" } +``` + +Universal parser fields on every event: `continue`, `stopReason`, `suppressOutput`, `systemMessage`. `systemMessage` is kept only for `PreToolUse`, `PostToolUse`, `UserPromptSubmit`, `SessionStart`, and `Stop`. On `PreCompact`/`PostCompact` it produces an `unsupported_field` warning and is dropped. Nested `hookSpecificOutput` feeds the same fields (and PreToolUse `permissionDecision` / `permissionDecisionReason` / `updatedInput`); a mismatched `hookEventName` discards only the specific-derived fields. + +### Exit codes + +Consumer-visible process exit from `executeSenpiHook`: + +| Code | Meaning | +| ---- | ----------------------------------------------------------------------------------------------------------------- | +| `0` | Handler command ran. Parsed output JSON was written to stdout (blocking outcomes ride in that JSON). | +| `1` | Validation failure: stdin timeout, malformed/schema-invalid envelope, or failed command spawn. Nothing on stdout. | + +Handler-command result rules (applied before writing stdout; runner still exits 0 when the command itself ran): + +| Child result | Parsed output | +| --------------------------------------- | ------------------------------------------------------------------------------------------- | +| exit code `2` | `{ decision: "block", reason: }` (reason omitted if blank); stdout ignored | +| other exit, empty stdout | `{}` (no-op) | +| other exit, valid JSON object stdout | universal + event-specific fields as above | +| other exit, malformed/non-object stdout | `{}` (no-op) plus `invalid_root` diagnostic on stderr | + +Output-parse diagnostics (`invalid_root`, `unsupported_field`, ...) are logged as `[senpi-hook]` lines on stderr and do not change the process exit code. + +`SenpiHookOutputBuilder` covers these shapes: `approve()`, `block(reason?)`, `deny(reason?)`, `ask(reason?)`, `context(additionalContext)`, `updatedInput(input)`, `updatedToolOutput(output)`, `forceStop(stopReason?)`, `systemMessage(text)`, plus `success(message?)` and `error(reason)`. Every output round-trips through `senpiHookOutputSchema`. Stop-gate outputs are advanced: kit examples and Cockpit wiring treat Stop as observe-only unless a later phase opts into block/continue with a written loop-safety rationale. + +## Runner + +`executeSenpiHook(handler, options?)` runs one command hook end to end: `readSenpiStdinJson(options?)` collects stdin (30-second cap, bounded byte read) and validates through `validateSenpiHookInput`, the platform-selected command runs (`commandWindows` on win32 when set), the vendored output-parser rules judge the result, and `outputSenpiJson` writes the typed output. Injectable seams cover stdin/stdout/stderr/exit/platform/`runCommand` for tests. + +`defaultRunCommand` (the default `runCommand` seam) spawns that command with `shell: true` and the inherited process environment. This module never consults hook trust state. `executeSenpiHook` is unsafe unless the caller already gated the handler. + +Consumers that handle untrusted settings must gate handlers through `isSenpiCommandHookTrusted` / `readSenpiHookTrustState` (`src/senpi/trust.ts`) before execution. Grants go through the consent-gated writer `writeSenpiHookTrustEntry` (`src/senpi/trust-writer.ts`). See [Trust gate](#trust-gate). + +```ts +const loaded = readSenpiHookTrustState(statePath); +if (!loaded.ok || !isSenpiCommandHookTrusted(handler, loaded.state)) { + throw new Error("hook is not trusted"); +} +await executeSenpiHook({ event: handler.event, config: handler.config }); +``` + +The senpi path never reads `CLAUDE_*` configuration, never imports grok or Claude modules, and never touches hook trust state. + +## Settings validation + +`validateSenpiHooksConfig(json)` validates a parsed JSON hooks object. It never throws. Return shape is `{ executableHandlers, diagnostics }` with the pinned `SenpiHookDiagnosticCode` vocabulary. + +Handlers are command only: + +```json +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "bash", + "hooks": [ + { + "type": "command", + "command": "node guard.mjs", + "timeout": 10, + "statusMessage": "running guard" + } + ] + } + ] + } +} +``` + +| Field | Notes | +| ---------------- | ------------------------------------------------------------------------------- | +| `type` | `command` only; `http`/`prompt`/`agent`/`mcp_tool` → `unsupported_handler_type` | +| `command` | Required | +| `commandWindows` | Optional win32 override | +| `timeout` | Seconds; upstream default 600 | +| `statusMessage` | Optional string | +| `matcher` | Optional group matcher | + +Config event keys must be exactly the 7 names in `SENPI_HOOK_EVENT_NAMES`. Snake_case or camelCase spellings are `unknown_event`. Keys in `SENPI_UNSUPPORTED_HOOK_EVENT_NAMES` are `unsupported_event`. Handler types in `SENPI_UNSUPPORTED_HANDLER_TYPES` are rejected with diagnostics, not executed. + +### Config sources (engine discovery) + +This library validates parsed config objects; it does not perform discovery. Upstream loads command hooks from: + +1. `/hooks.json` +2. `/.senpi/hooks.json` +3. `hooks` keys inside settings.json (global `/settings.json` and project `.senpi/settings.json`) + +Agent home resolution for consumers is `resolveSenpiAgentHome(options?)`: + +1. First non-empty of `OMO_CODING_AGENT_DIR`, `SENPI_CODING_AGENT_DIR`, `PI_CODING_AGENT_DIR` (`AGENT_DIR_ENV_NAMES`) +2. `/.omo/agent` when it contains `settings.json` (`AGENT_HOME_SENTINEL`) +3. `/.omo` when it contains `settings.json` +4. else `/.senpi/agent` + +### Trust gate + +Trust inspection is read-only and never writes: + +| Export | Purpose | +| -------------------------------- | ---------------------------------------------------------------------------------------- | +| `readSenpiHookTrustState` | Read `hooks-state.json` without writing | +| `isSenpiCommandHookTrusted` | Compare live handler hash to a stored grant | +| `senpiHookTrustId` | Pure id: `hk____` | +| `senpiHashCommandHook` | Pure content hash `sha256:` over the canonical command identity | +| `resolveSenpiHookTrustStatePath` | Resolve global `/hooks-state.json` or project `/.senpi/hooks-state.json` | +| `SENPI_HOOKS_STATE_FILENAME` | `hooks-state.json` | +| `SENPI_PROJECT_CONFIG_DIR` | `.senpi` | + +Storage paths (engine parity): global `/hooks-state.json`, project `/.senpi/hooks-state.json`. Path helpers never default to `~/.omo`. + +Mutation is explicit and opt-in. `writeSenpiHookTrustEntry` and `removeSenpiHookTrustEntry` are the grant/revoke acts themselves: callers must pass an options object with `consent: true`, a non-empty `reason`, and an explicit handler/scope/`agentHome`/`cwd` target before any filesystem access. Nothing runs at module import. Failures raise `SenpiTrustConsentError`, `SenpiTrustStateMalformedError`, or `SenpiTrustLockError`. The writer locks, preserves unknown entries, replaces atomically, and is fail-closed on malformed state. Removing the last entry (with no unknown top-level keys) deletes the state file so a grant/revoke cycle is a reversible file delta. + +The caller owns consent, the target directory, and uninstall. These helpers are library capability, not a Cockpit product integration. **Cockpit is observe-only** and must not grant or revoke trust. + +### Hooks registration + +Observe-only registration helpers never write trust and never enable gates: + +| Export | Purpose | +| ----------------------------- | -------------------------------------------------------------------------------------------- | +| `buildSenpiHooksRegistration` | Pure builder for a command-only hooks.json document | +| `resolveSenpiHooksConfigPath` | Resolve `{ filePath }` / global `/hooks.json` / project `/.senpi/hooks.json` | +| `readSenpiHooksConfig` | Inspect a target; missing file → `{ ok: true, document: null }` | +| `writeSenpiHooksConfig` | Consent-gated atomic write of that document | +| `removeSenpiHooksConfig` | Consent-gated unregister (deletes the file; missing is a no-op) | +| `SENPI_HOOKS_CONFIG_FILENAME` | `hooks.json` | +| `SenpiHooksConsentError` | Thrown before any write when consent, reason, or target is omitted | + +`writeSenpiHooksConfig` and `removeSenpiHooksConfig` require `{ consent: true, reason, target }`. Positional paths are rejected before any filesystem access. `target` is `{ filePath }`, `{ scope: "global", agentHome }`, or `{ scope: "project", cwd }`. The caller supplies the isolated home; the library never writes `~/.omo` unless that path is passed in explicitly. + +### Standalone forwarder assets + +`@libar-dev/agent-harness-kit/forwarder` exports pack-relative asset paths. It does not install anything. + +| Export | Purpose | +| --------------------------------------- | --------------------------------------------------------------- | +| `STANDALONE_HOOK_FORWARDER_ASSET` | `dist/standalone/hook-forwarder.mjs` (Claude) | +| `STANDALONE_SENPI_HOOK_FORWARDER_ASSET` | `dist/standalone/hook-forwarder-senpi.mjs` (Senpi observe-only) | +| `RUN_HOOK_WRAPPER_SH` | POSIX wrapper string for Claude endpoint-discovery consumers | + +The Senpi standalone forwarder POSTs a valid envelope to `SENPI_HOOK_FORWARD_URL` and always exits 0 with empty stdout. It never emits a gate decision. Shipping the asset is not an install. Cockpit must not install this forwarder or define a Senpi Stop policy. + +By default the URL must be `http` or `https` with host `127.0.0.1`, `localhost`, or `::1`. Set `SENPI_HOOK_FORWARD_ALLOW_REMOTE=1` to allow a remote host; any other value (including `true`) keeps the loopback default. Other schemes are rejected. Redirects are never followed. Stdin is capped at 1 MiB; that cap is the scale envelope for this path — oversized input is truncated rather than buffered without bound. + +## Session layout and processing APIs + +On-disk layout: + +``` +/sessions/<--encoded-cwd-->/ + _.jsonl + _-artifacts/ (skipped) + extensions/ (skipped) +``` + +Per-project directory names are dash-encoded cwd paths wrapped as `--...--` (every `/` becomes `-`). Encoding is ambiguous (`/a/b` and `/a-b` share a dirname), so decoding is never attempted. Listing verifies each file's header `cwd`. Session files are top-level `*.jsonl` only. + +Discovery and listing exports from `./senpi/processing`: + +| Export | Purpose | +| ---------------------------------------------- | ---------------------------------------------- | +| `getSenpiSessionsRoot(agentHome?)` | Resolve `/sessions` | +| `findSenpiSessionDirs(projectCwd, agentHome?)` | Candidate per-cwd directories (may over-match) | +| `listSenpiSessions(projectCwd, options?)` | Header-cwd-verified sessions for one project | +| `listAllSenpiSessions(options?)` | Every session under the agent home | + +`SenpiSessionInfo` fields: `path`, `id`, `cwd`, `name?`, `parentSessionPath?`, `created`, `modified`, `messageCount`, `firstMessage`. Engine parity minus `allMessagesText`. Per-file failures surface as `{ kind: 'invalid', path, error }`, never throw the whole listing. + +Parse export: + +| Export | Purpose | +| ---------------------- | ----------------------------------------------------------------------------------------------- | +| `parseSenpiEntry(raw)` | Tag-peek dispatch returning `known`, `unknown`, or `invalid`; never throws on unknown/malformed | + +Tree and projection exports: + +| Export | Purpose | +| -------------------------------------------- | ----------------------------------------------------------------------------------------- | +| `resolveSenpiLeaf(entries)` | Physical-order leaf selection over a tree index | +| `projectSenpiBranch(entries, leafId, opts?)` | Root-to-leaf active context with compaction handling | +| `computeProjectionMutation(prevKeys, next)` | Longest-common-prefix splice `{ index, deleteCount, records, removedRecordKeys }` or null | + +Tail, watch, checkpoint, and block exports: + +| Export | Purpose | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `tailSenpiSession(file, options?)` | One pass: cursor → parse → index → project → splice → reduce | +| `commitSenpiSessionCheckpoint(path, checkpoint, options?)` | Persist a revisioned marker after a successful pass | +| `watchSenpiSession(file, options?)` | Async generator; `fs.watch` is a wakeup hint, quiescence is a stable-cursor window (default 30s) | +| `reduceSenpiProjection(previous, current)` | Upserts/deletes driven by projection mutations | +| `foldSenpiBlockChanges(changes)` | Fold changes to final `SenpiSessionBlock` values | + +Those 13 value exports are the full `/senpi/processing` runtime surface. JSONL cursor internals and marker schema/filename helpers stay unexported. + +Unknown entry tags whose base `{ type, id, parentId, timestamp }` validates join the tree as unknown participants. Malformed known tags are `invalid`, never silently downgraded. + +## Tree and compaction semantics + +Sessions are branching trees, not linear logs. This library (not the consumer) linearizes each tree into the active root-to-leaf history. + +**Persisted-leaf rule.** The leaf is the last complete valid non-header entry in physical file order. Timestamps and childless-node heuristics are not used. `custom`, `label`, `session_info`, `branch_summary`, and unknown-with-valid-base entries all advance the leaf. Live in-engine `/tree` navigation that has not yet been appended to disk is invisible to file-only readers; that gap is intentional and documented as persisted-leaf semantics. + +**Compaction.** Only the latest compaction on the active path applies. + +- When `retainedTail` is present (including `[]`), it is authoritative: the compaction record plus the retained messages (stable keys `retained::`) form the checkpoint, and older prefix entries are disposition `summarized`. +- Legacy sessions without `retainedTail` use `firstKeptEntryId` range inclusion. A missing or off-path first-kept id yields a `missing_first_kept` warning and an incomplete projection. +- Off-path accepted entries are disposition `off_branch` or `summarized`. + +**Splice mutations.** Incremental tail emits at most one revisioned suffix splice per pass: + +```ts +{ + baseRevision: number; + revision: number; + index: number; + deleteCount: number; + records: SenpiProjectionRecord[]; + removedRecordKeys: string[]; +} +``` + +Consumers assert their local revision equals `baseRevision`, replace `deleteCount` records at `index` with `records`, then store `revision`. Cold rebuild (invalid or missing marker) emits a full splice from index 0. + +**Checkpoint fields** (`SenpiSessionCheckpoint` returned by tail; committed via `commitSenpiSessionCheckpoint`): + +| Field | Role | +| ---------------------- | ----------------------------------------------- | +| `sessionPathDigest` | Identity of the session file path | +| `sessionId` | Session UUID | +| `device`, `inode` | Decimal-string file identity | +| `generation` | Identity/content reset counter | +| `offset`, `lineNumber` | Byte cursor at a line boundary | +| `headDigest` | SHA-256 of the committed head window | +| `boundaryDigest` | SHA-256 at the committed boundary | +| `baseRevision` | Expected current marker revision (0 if none) | +| `leafId` | Persisted leaf id, or null | +| `projectedRecordKeys` | Ordered active keys for the next LCP comparison | + +Marker files are read through a `FileHandle` with a fixed 1 MiB bound. The implementation keeps graph and overflow accounting private and bounded: at most 2048 graph entries / 256 KiB and 2048 projected keys / 256 KiB. Pure append may use that validated private state; branch switches, suffix compaction, absent or invalid state, and other non-append cases rebuild exactly from byte zero. Rebuild work has fixed, non-configurable production bounds of four scans, 128 MiB, and 40,000 lines. When that bound is exhausted, the marker and public byte offset remain unchanged while the existing opaque checkpoint state carries private continuation progress for a later call. + +Invalidation (inode change, size below offset, header/digest change, offset not on a line boundary, `fromStart`, malformed marker) forces a cold rebuild. Automatic checkpoint mode writes only after a full successful parse+projection; manual mode returns the checkpoint for the caller to commit. + +## Upstream pin and drift policy + +Contract artifacts from npm package `@code-yeongyu/senpi` are vendored under `docs/upstream/senpi/`: session-format/settings/environment-variables docs, 21 hooks `.d.ts` files, and the implementation files required for hash/parser/trust-storage parity, plus an MIT `NOTICE`. `pin.json` records `engineVersion: "2026.8.19"`, registry integrity, per-file sha256, and notes. The npm tarball is the artifact of record. + +```bash +node scripts/sync-upstream-senpi.mjs --tarball --check +``` + +`--check` writes nothing. On mismatch it exits non-zero and prints `Senpi upstream vendor drift detected: `. On success it prints `Senpi upstream vendor is in sync.` The drift tests (`tests/senpi-upstream-drift.test.ts`) parse the vendored artifacts both directions (vendored→kit constants/schemas and kit→vendored) and include a mutation-detection case. The sync script is not wired into CI (it needs the tarball); the vendored-only drift tests do run on every push. + +## OmO vs senpi naming + +OmO native is the branded distribution end users see. The engine underneath is senpi (`@code-yeongyu/senpi`). This kit keeps engine-accurate names: + +| Layer | Identifier | +| ----------------------------- | ----------------------------------------------- | +| Product / UI copy | OmO native | +| Kit module subpaths | `./senpi`, `./senpi/processing` | +| Exported type/function prefix | `Senpi*`, `senpi*`, `SENPI_*` | +| Agent-home branded path | `~/.omo/agent` (with `~/.senpi/agent` fallback) | + +Do not rename kit APIs to `Omo*`. Downstream product layers may brand as OmO while importing the senpi subpaths. + +## Cockpit seam + +**Cockpit is observe-only for OmO/Senpi.** This package is a library. Library capability is not product integration. + +| Layer | Allowed | Forbidden | +| ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| Kit `/senpi` and `/senpi/processing` | Resolve home, list/tail/watch sessions, validate hook I/O, inspect trust, build a hooks.json document | Spawn, drive, Commit, RPC | +| Kit mutating primitives (`writeSenpiHooksConfig`, `removeSenpiHooksConfig`, `writeSenpiHookTrustEntry`, `removeSenpiHookTrustEntry`) | Explicit owner/operator tools that pass `{ consent: true, reason, target }` against a directory they own | Module-import side effects; defaulting to `~/.omo`; silent install | +| Cockpit product | Dynamic import of observe APIs; read session files for the open project | Register hooks, grant/revoke trust, install the Senpi forwarder, enforce a Senpi Stop gate, write OmO config | + +The caller of a mutating primitive owns consent, the target path, and uninstall. Cockpit must not be that caller. Cross-repo product wiring lives outside this package. + +## Senpi vs Claude / Grok (no unification) + +There is no shared `SessionBlock`, no Claude-to-senpi translator, and no cross-adapter hook-event unification. Write a senpi-native entrypoint with `executeSenpiHook`. + +| | Claude (root exports) | Grok (`./grok`) | Senpi (`./senpi`) | +| ---------------------- | --------------------------------------- | --------------------------------------- | --------------------------------------------------------- | +| Events | 30 | 14 wire + legacy alias (15 accepted) | 7 supported | +| Envelope discriminator | `hook_event_name` PascalCase value | `hookEventName` snake_case value | `event` PascalCase (snake alias `hook_event_name`) | +| Handler types | command, http, mcp_tool, prompt, agent | command, http | command only | +| PreToolUse decisions | allow, deny, ask, defer, + updatedInput | allow, deny only | allow, approve, ask, deny (+ updatedInput on allow) | +| Failure / block | exit 2 blocks | fail-open except deny/block JSON/exit 2 | child exit 2 → block JSON; runner exit 0 when command ran | +| Session root | `~/.claude/projects` | `GROK_HOME ?? ~/.grok` | `resolveSenpiAgentHome()` → `.../sessions` | +| Session shape | single JSONL | `updates.jsonl` + `events.jsonl` | single tree-shaped JSONL (v3) | diff --git a/docs/upstream/senpi/NOTICE b/docs/upstream/senpi/NOTICE new file mode 100644 index 0000000..ce381fd --- /dev/null +++ b/docs/upstream/senpi/NOTICE @@ -0,0 +1,11 @@ +Senpi upstream contract files + +The contract files in this directory are copied from the published npm package +@code-yeongyu/senpi (version recorded in pin.json): + + https://github.com/code-yeongyu/senpi + +Copyright (c) code-yeongyu and contributors. The upstream package is licensed +under the MIT License. The pin manifest records the engine version, npm +registry integrity, package-relative source paths, and per-file sha256 hashes +of the vendored bytes. diff --git a/docs/upstream/senpi/environment-variables.md b/docs/upstream/senpi/environment-variables.md new file mode 100644 index 0000000..fe81f82 --- /dev/null +++ b/docs/upstream/senpi/environment-variables.md @@ -0,0 +1,102 @@ +# Environment Variables + +Pi uses environment variables in three ways: + +- Variables such as `PI_OFFLINE` configure the Pi process. +- Pi sets process markers so child processes can identify Pi as the launching agent. +- Commands run by the LLM-callable bash tool receive `PI_*` variables describing the current session. + +Provider API-key variables are documented separately in [Providers](providers.md#environment-variables-or-auth-file). + +## Process Marker + +The CLI and RPC entry points set two process markers: + +- `AI_AGENT=pi` is a generic marker that lets tooling identify Pi as the agent that launched the process. +- `PI_CODING_AGENT=true` is Pi-specific and lets child processes detect that they run inside Pi. + +Child processes inherit both markers. They are not session-specific and are not set automatically when Pi is embedded through the SDK. + +## Bash Tool Session Environment + +Commands run by the bash tool receive the current Pi session state: + +| Variable | Description | +|----------|-------------| +| `PI_SESSION_ID` | Current session ID | +| `PI_SESSION_FILE` | Absolute path to the current session JSONL file; unset for ephemeral sessions | +| `PI_PROVIDER` | Currently selected model provider | +| `PI_MODEL` | Currently selected model ID | +| `PI_REASONING_LEVEL` | Current effective reasoning level: `off`, `minimal`, `low`, `medium`, `high`, `xhigh`, or `max` | + +The values are resolved when each command starts. Switching models or changing the reasoning level therefore affects the next bash command without restarting Pi. `PI_PROVIDER` and `PI_MODEL` identify the selected Pi model, not a different upstream model that a router may choose internally. + +When asked which model or provider is running, inspect these variables instead of inferring the answer from the system prompt: + +```bash +printf '%s/%s\n' "$PI_PROVIDER" "$PI_MODEL" +printf 'reasoning=%s session=%s\n' "$PI_REASONING_LEVEL" "$PI_SESSION_ID" +``` + +The session file can be inspected directly when the session is persistent: + +```bash +if [ -n "$PI_SESSION_FILE" ]; then + tail -n 1 "$PI_SESSION_FILE" +fi +``` + +These variables are injected into the LLM-callable bash tool. They are not injected into user-entered `!` or `!!` commands. + +### Custom Bash Tools + +Bash tools created with `createBashTool()` expose the session environment by default when registered with Pi. Injection happens before `spawnHook`, so a hook receives the variables in `ctx.env`: + +```typescript +const bashTool = createBashTool(cwd, { + spawnHook: (ctx) => ({ + ...ctx, + env: { ...ctx.env, CI: "1" }, + }), +}); +``` + +Disable session metadata independently of the spawn hook: + +```typescript +const bashTool = createBashTool(cwd, { + exposeSessionEnvironment: false, + spawnHook: (ctx) => ctx, +}); +``` + +When disabled, Pi removes inherited values for these variables so nested Pi processes do not expose stale parent-session metadata. + +## Pi Process Configuration + +These variables are read by Pi itself: + +| Variable | Description | +|----------|-------------| +| `PI_CODING_AGENT_DIR` | Override the config directory; default is `~/.pi/agent` | +| `PI_CODING_AGENT_SESSION_DIR` | Override session storage; overridden by `--session-dir` | +| `PI_PACKAGE_DIR` | Override the package directory, useful for Nix/Guix store paths | +| `PI_OFFLINE` | Disable startup network operations, including update checks, package updates, and install/update telemetry | +| `PI_SKIP_VERSION_CHECK` | Disable the `pi.dev` latest-version request | +| `PI_TELEMETRY` | Override install/update telemetry and provider attribution headers: `1`/`true`/`yes` or `0`/`false`/`no` | +| `PI_CACHE_RETENTION` | Set to `long` to opt into extended provider prompt caching where supported; direct Anthropic defaults to 5 minutes | +| `PI_SHARE_VIEWER_URL` | Override the base URL used by `/share` | +| `PI_HARDWARE_CURSOR` | Set to `1` to show the hardware cursor; see [Terminal setup](terminal-setup.md) | +| `PI_TUI_ESC_TIMEOUT` | How long to wait after a lone ESC before treating it as Escape, in milliseconds; defaults to `100` over SSH and `10` otherwise. Increase if Alt-key input is misread as Escape | +| `SENPI_RECOVER_INSPECTOR_VM_IMPORT` | Set to `1` at process start to keep the TUI running when a Node Inspector (`node inspect` / `--inspect`) eval uses dynamic `import()`, which Node rejects with `ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING`. Recovery applies only to that exact Inspector-originated rejection while an Inspector endpoint is active; all other uncaught errors remain fatal. Use `require()` or a target-side loader in Inspector evals instead | +| `VISUAL`, `EDITOR` | External editor fallback when `externalEditor` is unset | +| `HTTP_PROXY`, `HTTPS_PROXY` | Proxy outbound HTTP requests | + +### Image Generation + +| Variable | Description | +|----------|-------------| +| `PI_OPENAI_IMAGE_GEN` | Enable or disable native `image_generation` server-tool injection on OpenAI Responses models. Accepts `1`/`true`/`yes` (default) or `0`/`false`/`no`. When disabled, the client-side `generate_image` tool is used instead | +| `PI_IMAGE_GEN_PROVIDER` | Pin a specific configured gateway provider for image generation (for example `quotio-openai`). The provider must have a resolvable API key and base URL in `models.json`. When unset, the credential resolver picks the best available source automatically | + +Provider credentials such as `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, and cloud-provider configuration are listed in [Providers](providers.md#environment-variables-or-auth-file). diff --git a/docs/upstream/senpi/hooks/command-runner.d.ts b/docs/upstream/senpi/hooks/command-runner.d.ts new file mode 100644 index 0000000..166a253 --- /dev/null +++ b/docs/upstream/senpi/hooks/command-runner.d.ts @@ -0,0 +1,25 @@ +import { type HookOutputPolicy, type HookOutputSafetyMetadata } from "./safety.ts"; +import type { ExecutableHookHandler, HookInputWire } from "./types.ts"; +export type CommandHookRunOptions = { + readonly cwd: string; + readonly envPassthrough?: readonly string[]; + readonly outputPolicy?: HookOutputPolicy; + readonly signal?: AbortSignal; + readonly sourceEnv?: NodeJS.ProcessEnv; +}; +export type CommandHookRunResult = { + readonly command: string; + readonly cwd: string; + readonly stdout: string; + readonly stderr: string; + readonly exitCode: number | null; + readonly signal: NodeJS.Signals | null; + readonly timedOut: boolean; + readonly aborted: boolean; + readonly durationMs: number; + readonly outputSafety: HookOutputSafetyMetadata; + readonly timeoutSeconds: number; +}; +export declare function runCommandHook(handler: ExecutableHookHandler, input: HookInputWire, options: CommandHookRunOptions): Promise; +export declare function selectCommandForPlatform(handler: ExecutableHookHandler, platform?: NodeJS.Platform): string; +//# sourceMappingURL=command-runner.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/command.d.ts b/docs/upstream/senpi/hooks/command.d.ts new file mode 100644 index 0000000..ac56368 --- /dev/null +++ b/docs/upstream/senpi/hooks/command.d.ts @@ -0,0 +1,11 @@ +import type { ExtensionAPI, ExtensionContext } from "../../types.ts"; +import type { HookStateStorage } from "./trust-storage.ts"; +import type { HookDiagnostic, HookRuntimeState } from "./types.ts"; +type HookCommandRuntimeState = HookRuntimeState & { + readonly storage: HookStateStorage; +}; +export declare function registerHooksCommand(pi: ExtensionAPI, refreshState: (ctx: ExtensionContext) => HookCommandRuntimeState): void; +export declare function formatHookStatus(state: HookCommandRuntimeState): string; +export declare function formatHookDiagnostics(diagnostics: readonly HookDiagnostic[]): string; +export {}; +//# sourceMappingURL=command.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/config-loader.d.ts b/docs/upstream/senpi/hooks/config-loader.d.ts new file mode 100644 index 0000000..ba4a41c --- /dev/null +++ b/docs/upstream/senpi/hooks/config-loader.d.ts @@ -0,0 +1,19 @@ +import type { ParsedHookConfig } from "./types.ts"; +export type HookConfigFileSystem = { + readonly readTextFile: (path: string) => string | undefined; +}; +export type HookConfigLoaderOptions = { + readonly cwd: string; + readonly agentDir: string; + readonly fileSystem: HookConfigFileSystem; + readonly globalSettingsHooks?: unknown; + readonly projectSettingsHooks?: unknown; + readonly globalHooksPath?: string; + readonly projectHooksPath?: string; + readonly globalHookSourcePaths?: readonly string[]; + readonly projectHookSourcePaths?: readonly string[]; + readonly preSessionHookSourcePaths?: readonly string[]; + readonly runtimeHookSourcePaths?: readonly string[]; +}; +export declare function loadHookConfigSources(options: HookConfigLoaderOptions): ParsedHookConfig; +//# sourceMappingURL=config-loader.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/diagnostics.d.ts b/docs/upstream/senpi/hooks/diagnostics.d.ts new file mode 100644 index 0000000..4451d82 --- /dev/null +++ b/docs/upstream/senpi/hooks/diagnostics.d.ts @@ -0,0 +1,10 @@ +import type { HookDiagnostic, HookDiagnosticCode, HookSourceMetadata } from "./types.ts"; +export type DiagnosticDraft = { + readonly code: HookDiagnosticCode; + readonly message: string; + readonly path: string; + readonly event?: string; + readonly severity?: "error" | "warning"; +}; +export declare function diagnostic(draft: DiagnosticDraft, source: HookSourceMetadata): HookDiagnostic; +//# sourceMappingURL=diagnostics.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/dispatcher.d.ts b/docs/upstream/senpi/hooks/dispatcher.d.ts new file mode 100644 index 0000000..bf631b2 --- /dev/null +++ b/docs/upstream/senpi/hooks/dispatcher.d.ts @@ -0,0 +1,62 @@ +import { type CommandHookRunOptions, type CommandHookRunResult } from "./command-runner.ts"; +import { type ParsedHookOutput } from "./output-parser.ts"; +import { type HookOutputPolicy } from "./safety.ts"; +import { type HookTrustOptions, type HookTrustRecord } from "./trust.ts"; +import type { ExecutableHookHandler, HookDiagnostic, HookInputWire, HookSourceMetadata, HookTrustState } from "./types.ts"; +export type HookCommandRunner = (handler: ExecutableHookHandler, input: HookInputWire, options: CommandHookRunOptions) => Promise; +export type HookDispatchDecision = { + readonly kind: "none"; +} | { + readonly kind: "allow" | "block"; + readonly reason?: string; + readonly source: HookSourceMetadata; + readonly sourceCommand: string; + readonly updatedInput?: unknown; +} | { + readonly fallback: { + readonly kind: "block"; + readonly reason: string; + }; + readonly kind: "ask"; + readonly nativeRepresentable: false; + readonly reason?: string; + readonly source: HookSourceMetadata; + readonly sourceCommand: string; +}; +export type HookDispatchSkipped = { + readonly diagnostics: readonly HookDiagnostic[]; + readonly handler: ExecutableHookHandler; + readonly reason: "disabled" | "untrusted" | "unsafe"; + readonly record: HookTrustRecord; +}; +export type HookDispatchSummary = { + readonly completionIndex: number; + readonly diagnostics: readonly HookDiagnostic[]; + readonly handler: ExecutableHookHandler; + readonly output: ParsedHookOutput["output"]; + readonly run: CommandHookRunResult; +}; +export type HookDispatchResult = { + readonly decision: HookDispatchDecision; + readonly diagnostics: readonly HookDiagnostic[]; + readonly executableHandlers: readonly ExecutableHookHandler[]; + readonly matchedHandlers: readonly ExecutableHookHandler[]; + readonly skipped: readonly HookDispatchSkipped[]; + readonly summaries: readonly HookDispatchSummary[]; +}; +export type HookDispatchOptions = { + readonly cwd: string; + readonly envPassthrough?: readonly string[]; + readonly handlers: readonly ExecutableHookHandler[]; + readonly input: HookInputWire; + readonly onRunningHandlersChange?: (running: readonly ExecutableHookHandler[]) => void; + readonly outputPolicy?: HookOutputPolicy; + readonly runCommand?: HookCommandRunner; + readonly signal?: AbortSignal; + readonly sourceEnv?: NodeJS.ProcessEnv; + readonly trustOptions?: HookTrustOptions; + readonly trustState: HookTrustState; +}; +export declare function dispatchHookEvent(options: HookDispatchOptions): Promise; +export declare function runningHookHandlersStatusLabel(handlers: readonly ExecutableHookHandler[], platform?: NodeJS.Platform): string; +//# sourceMappingURL=dispatcher.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/handler.d.ts b/docs/upstream/senpi/hooks/handler.d.ts new file mode 100644 index 0000000..e857a36 --- /dev/null +++ b/docs/upstream/senpi/hooks/handler.d.ts @@ -0,0 +1,12 @@ +import type { ExecutableHookHandler, HookDiagnostic, HookSourceMetadata, SupportedHookEvent } from "./types.ts"; +type HandlerParseContext = { + readonly event: SupportedHookEvent; + readonly matcher?: string; + readonly groupIndex: number; + readonly handlerIndex: number; + readonly source: HookSourceMetadata; + readonly diagnostics: HookDiagnostic[]; +}; +export declare function parseHandler(handler: unknown, context: HandlerParseContext): ExecutableHookHandler | undefined; +export {}; +//# sourceMappingURL=handler.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/index.d.ts b/docs/upstream/senpi/hooks/index.d.ts new file mode 100644 index 0000000..dcb775a --- /dev/null +++ b/docs/upstream/senpi/hooks/index.d.ts @@ -0,0 +1,6 @@ +import type { ExtensionAPI } from "../../types.ts"; +export { parseHookConfig } from "./schema.ts"; +export type { CommandHookConfig, ExecutableHookHandler, HookDiagnostic, HookDiagnosticCode, HookInputWire, HookOutputWire, HookRuntimeState, HookSourceMetadata, HookTrustEntry, HookTrustState, ParsedHookConfig, SupportedHookEvent, UnsupportedKnownHookEvent, } from "./types.ts"; +export default function hooksExtension(pi: ExtensionAPI): void; +export { SUPPORTED_HOOK_EVENTS, UNSUPPORTED_HANDLER_TYPES, UNSUPPORTED_KNOWN_HOOK_EVENTS, } from "./types.ts"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/lifecycle-adapter.d.ts b/docs/upstream/senpi/hooks/lifecycle-adapter.d.ts new file mode 100644 index 0000000..dff89e0 --- /dev/null +++ b/docs/upstream/senpi/hooks/lifecycle-adapter.d.ts @@ -0,0 +1,31 @@ +import type { ExtensionAPI, ExtensionContext, SessionBeforeCompactEvent, SessionBeforeCompactResult, SessionCompactEvent, SessionStartEvent } from "../../types.ts"; +import { type HookDispatchResult } from "./dispatcher.ts"; +import type { ExecutableHookHandler, HookDiagnostic, HookInputWire, HookTrustState } from "./types.ts"; +type LifecycleHookEvent = "SessionStart" | "PreCompact" | "PostCompact"; +type LifecycleDispatchOptions = { + readonly cwd: string; + readonly handlers: readonly ExecutableHookHandler[]; + readonly input: HookInputWire; + readonly matcherInputs: readonly string[]; + readonly signal?: AbortSignal; + readonly trustState: HookTrustState; +}; +type LifecycleResultDetails = { + readonly cancel: boolean; + readonly contexts: readonly string[]; + readonly diagnostics: readonly HookDiagnostic[]; + readonly reason?: string; +}; +export declare function buildSessionStartHookInput(event: SessionStartEvent, ctx: ExtensionContext): HookInputWire; +export declare function buildPreCompactHookInput(event: SessionBeforeCompactEvent, ctx: ExtensionContext): HookInputWire; +export declare function buildPostCompactHookInput(event: SessionCompactEvent, ctx: ExtensionContext): HookInputWire; +export declare function dispatchLifecycleHookEvent(options: LifecycleDispatchOptions): Promise; +export declare function sessionStartResultDetails(result: HookDispatchResult | undefined): LifecycleResultDetails; +export declare function preCompactResultDetails(result: HookDispatchResult | undefined): LifecycleResultDetails; +export declare function postCompactResultDetails(result: HookDispatchResult | undefined): LifecycleResultDetails; +export declare function sessionBeforeCompactResult(details: LifecycleResultDetails): SessionBeforeCompactResult | undefined; +export declare function recordLifecycleHookResult(pi: Pick, event: LifecycleHookEvent, details: LifecycleResultDetails, options?: { + compactionRequestId?: string; +}): void; +export {}; +//# sourceMappingURL=lifecycle-adapter.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/matcher.d.ts b/docs/upstream/senpi/hooks/matcher.d.ts new file mode 100644 index 0000000..281203f --- /dev/null +++ b/docs/upstream/senpi/hooks/matcher.d.ts @@ -0,0 +1,7 @@ +import type { ExecutableHookHandler, HookDiagnostic, HookInputWire } from "./types.ts"; +export type HookMatcherResult = { + readonly handlers: readonly ExecutableHookHandler[]; + readonly diagnostics: readonly HookDiagnostic[]; +}; +export declare function matchingHookHandlers(input: HookInputWire, handlers: readonly ExecutableHookHandler[]): HookMatcherResult; +//# sourceMappingURL=matcher.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/output-bounds.d.ts b/docs/upstream/senpi/hooks/output-bounds.d.ts new file mode 100644 index 0000000..05f05e4 --- /dev/null +++ b/docs/upstream/senpi/hooks/output-bounds.d.ts @@ -0,0 +1,29 @@ +export declare const DEFAULT_STDOUT_LIMIT_BYTES: number; +export declare const DEFAULT_STDERR_LIMIT_BYTES: number; +export type HookOutputPolicy = { + readonly maxStdoutBytes?: number; + readonly maxStderrBytes?: number; + readonly spillDir?: string; +}; +export type HookStreamSafetyMetadata = { + readonly originalBytes: number; + readonly returnedBytes: number; + readonly redacted: boolean; + readonly spilled: boolean; + readonly truncated: boolean; + readonly spillPath?: string; +}; +export type HookOutputSafetyMetadata = { + readonly stdout: HookStreamSafetyMetadata; + readonly stderr: HookStreamSafetyMetadata; +}; +export type HookSafeOutput = { + readonly text: string; + readonly safety: HookStreamSafetyMetadata; +}; +export declare function applyHookOutputSafety(stream: "stderr" | "stdout", text: string, policy: HookOutputPolicy | undefined, capture?: { + readonly originalBytes: number; + readonly truncated: boolean; +}): HookSafeOutput; +export declare function redactHookTokenValues(text: string, replacement?: string): string; +//# sourceMappingURL=output-bounds.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/output-bounds.js b/docs/upstream/senpi/hooks/output-bounds.js new file mode 100644 index 0000000..cc8c682 --- /dev/null +++ b/docs/upstream/senpi/hooks/output-bounds.js @@ -0,0 +1,43 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { redactSensitiveOutput, redactSensitiveTokenValues } from "../../../sensitive-output.js"; +export const DEFAULT_STDOUT_LIMIT_BYTES = 64 * 1024; +export const DEFAULT_STDERR_LIMIT_BYTES = 64 * 1024; +export function applyHookOutputSafety(stream, text, policy, capture) { + const originalBytes = capture?.originalBytes ?? Buffer.byteLength(text); + const redactedText = redactHookOutput(text); + const redacted = redactedText !== text; + const limit = stream === "stdout" ? policy?.maxStdoutBytes : policy?.maxStderrBytes; + const maxBytes = limit ?? (stream === "stdout" ? DEFAULT_STDOUT_LIMIT_BYTES : DEFAULT_STDERR_LIMIT_BYTES); + const redactedBytes = Buffer.byteLength(redactedText); + const truncated = capture?.truncated === true || redactedBytes > maxBytes; + const spillPath = truncated ? spillRedactedOutput(stream, redactedText, policy) : undefined; + const returnedText = truncated ? Buffer.from(redactedText).subarray(0, maxBytes).toString("utf8") : redactedText; + return { + safety: { + originalBytes, + redacted, + returnedBytes: Buffer.byteLength(returnedText), + spilled: spillPath !== undefined, + ...(spillPath === undefined ? {} : { spillPath }), + truncated, + }, + text: returnedText, + }; +} +function redactHookOutput(text) { + return redactSensitiveOutput(text); +} +export function redactHookTokenValues(text, replacement = "[REDACTED]") { + return redactSensitiveTokenValues(text, replacement); +} +function spillRedactedOutput(stream, text, policy) { + const spillDir = policy?.spillDir ?? join(tmpdir(), "senpi-hook-output"); + mkdirSync(spillDir, { recursive: true }); + const path = join(spillDir, `hook-${stream}-${process.pid}-${randomUUID()}.txt`); + writeFileSync(path, text); + return path; +} +//# sourceMappingURL=output-bounds.js.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/output-parser.d.ts b/docs/upstream/senpi/hooks/output-parser.d.ts new file mode 100644 index 0000000..9378c86 --- /dev/null +++ b/docs/upstream/senpi/hooks/output-parser.d.ts @@ -0,0 +1,27 @@ +import type { HookDiagnostic, HookSourceMetadata, SupportedHookEvent } from "./types.ts"; +export type HookOutputParseInput = { + readonly event: SupportedHookEvent; + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; + readonly source: HookSourceMetadata; +}; +type HookDecision = "allow" | "approve" | "ask" | "block" | "deny"; +type MutableHookOutput = { + decision?: HookDecision; + reason?: string; + additionalContext?: string; + updatedInput?: unknown; + updatedToolOutput?: unknown; + continue?: boolean; + stopReason?: string; + suppressOutput?: boolean; + systemMessage?: string; +}; +export type ParsedHookOutput = { + readonly output: Readonly; + readonly diagnostics: readonly HookDiagnostic[]; +}; +export declare function parseHookOutput(input: HookOutputParseInput): ParsedHookOutput; +export {}; +//# sourceMappingURL=output-parser.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/output-parser.js b/docs/upstream/senpi/hooks/output-parser.js new file mode 100644 index 0000000..95acc63 --- /dev/null +++ b/docs/upstream/senpi/hooks/output-parser.js @@ -0,0 +1,168 @@ +import { diagnostic } from "./diagnostics.js"; +const SYSTEM_MESSAGE_EVENTS = new Set([ + "PreToolUse", + "PostToolUse", + "UserPromptSubmit", + "SessionStart", + "Stop", +]); +export function parseHookOutput(input) { + const state = { input, output: {}, diagnostics: [] }; + const stderr = text(input.stderr); + if (input.exitCode === 2) { + return { output: { decision: "block", ...(stderr === undefined ? {} : { reason: stderr }) }, diagnostics: [] }; + } + const stdout = input.stdout.trim(); + if (stdout.length === 0) + return parsedOutput(state); + const parsed = parseStdoutJson(stdout, state); + if (!isRecord(parsed)) + return parsedOutput(state); + parseUniversal(parsed, state); + const specific = parseSpecific(parsed.hookSpecificOutput, state); + if (specific === "mismatched" || specific === "invalid") + return parsedOutput(state); + parseEvent(parsed, specific, state); + return parsedOutput(state); +} +function parseStdoutJson(stdout, state) { + try { + const parsed = JSON.parse(stdout); + if (isRecord(parsed)) + return parsed; + add(state, "invalid_root", "stdout", "Hook stdout JSON must be an object."); + } + catch (error) { + if (!(error instanceof SyntaxError)) + throw error; + add(state, "invalid_root", "stdout", "Hook stdout must be valid JSON."); + } + return undefined; +} +function parseUniversal(parsed, state) { + if (typeof parsed.continue === "boolean") { + state.output.continue = parsed.continue; + if (state.input.event === "Stop" && parsed.continue === false) + state.output.decision = "block"; + } + copyText(parsed.stopReason, "stopReason", state); + if (typeof parsed.suppressOutput === "boolean") + state.output.suppressOutput = parsed.suppressOutput; + if (parsed.systemMessage === undefined) + return; + if (SYSTEM_MESSAGE_EVENTS.has(state.input.event)) { + copyText(parsed.systemMessage, "systemMessage", state); + return; + } + add(state, "unsupported_field", "stdout.systemMessage", "Hook systemMessage is not supported for this event.", "warning"); +} +function parseSpecific(value, state) { + if (value === undefined) + return undefined; + if (!isRecord(value)) { + add(state, "invalid_event_config", "stdout.hookSpecificOutput", "Hook hookSpecificOutput field must be an object."); + return "invalid"; + } + const eventName = value.hookEventName; + if (eventName === undefined || eventName === state.input.event) + return value; + add(state, "invalid_event_config", "stdout.hookSpecificOutput.hookEventName", `Hook output event ${String(eventName)} does not match ${state.input.event}.`); + return "mismatched"; +} +function parseEvent(parsed, specific, state) { + switch (state.input.event) { + case "PreToolUse": + parsePreToolUse(parsed, specific, state); + return; + case "PostToolUse": + blockOnlyDecision(parsed.decision, "PostToolUse", state); + copyText(parsed.reason, "reason", state); + copyText(specific?.additionalContext ?? parsed.additionalContext, "additionalContext", state); + copyUnknown(specific?.updatedToolOutput ?? parsed.updatedToolOutput, "updatedToolOutput", state); + return; + case "UserPromptSubmit": + blockOnlyDecision(parsed.decision, "UserPromptSubmit", state); + copyText(parsed.reason, "reason", state); + copyText(specific?.additionalContext ?? parsed.additionalContext, "additionalContext", state); + rejectPromptReplacement(specific, state); + return; + case "Stop": + if (parsed.decision === "block") + state.output.decision = "block"; + else if (parsed.decision !== undefined && parsed.decision !== "continue") { + add(state, "unsupported_field", "stdout.decision", "Stop only supports decision block or continue.", "warning"); + } + copyText(parsed.reason, "reason", state); + copyText(specific?.additionalContext ?? parsed.additionalContext, "additionalContext", state); + return; + case "SessionStart": + copyText(specific?.additionalContext ?? parsed.additionalContext, "additionalContext", state); + if (parsed.decision !== undefined) { + add(state, "unsupported_field", "stdout.decision", "SessionStart does not support decisions.", "warning"); + } + return; + case "PreCompact": + case "PostCompact": + return; + } +} +function parsePreToolUse(parsed, specific, state) { + const decision = preToolUseDecision(specific?.permissionDecision ?? parsed.decision); + if (decision !== undefined) + state.output.decision = decision; + copyText(specific?.permissionDecisionReason ?? parsed.reason, "reason", state); + copyText(specific?.additionalContext ?? parsed.additionalContext, "additionalContext", state); + const updatedInput = specific?.updatedInput ?? parsed.updatedInput; + if (updatedInput === undefined) + return; + if (specific?.permissionDecision === "allow") { + state.output.updatedInput = updatedInput; + return; + } + add(state, "unsupported_field", "stdout.hookSpecificOutput.updatedInput", "PreToolUse updatedInput is only applied when permissionDecision is allow.", "warning"); +} +function blockOnlyDecision(value, event, state) { + if (value === "block") + state.output.decision = "block"; + else if (value !== undefined) + add(state, "unsupported_field", "stdout.decision", `${event} only supports decision block.`, "warning"); +} +function rejectPromptReplacement(specific, state) { + for (const field of ["prompt", "updatedPrompt", "replacementPrompt"]) { + if (specific !== undefined && Object.hasOwn(specific, field)) { + add(state, "unsupported_field", `stdout.hookSpecificOutput.${field}`, "UserPromptSubmit prompt replacement is not supported.", "warning"); + } + } +} +function preToolUseDecision(value) { + if (value === "allow" || value === "approve" || value === "ask") + return value; + if (value === "deny" || value === "block") + return "deny"; + return undefined; +} +function copyText(value, field, state) { + const normalized = text(value); + if (normalized !== undefined) + state.output[field] = normalized; +} +function copyUnknown(value, field, state) { + if (value !== undefined) + state.output[field] = value; +} +function add(state, code, path, message, severity) { + state.diagnostics.push(diagnostic({ code, event: state.input.event, message, path, ...(severity === undefined ? {} : { severity }) }, state.input.source)); +} +function text(value) { + if (typeof value !== "string") + return undefined; + const trimmed = value.trim(); + return trimmed.length === 0 ? undefined : trimmed; +} +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function parsedOutput(state) { + return { output: state.output, diagnostics: state.diagnostics }; +} +//# sourceMappingURL=output-parser.js.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/plugin-loader.d.ts b/docs/upstream/senpi/hooks/plugin-loader.d.ts new file mode 100644 index 0000000..d3e5e17 --- /dev/null +++ b/docs/upstream/senpi/hooks/plugin-loader.d.ts @@ -0,0 +1,11 @@ +import { type LoadPluginHookManifestOptions, type PluginHookSourceMetadata } from "./plugin-manifest.ts"; +import type { ExecutableHookHandler, HookDiagnostic, ParsedHookConfig } from "./types.ts"; +export type { LoadPluginHookManifestOptions, PluginHookEnv, PluginHookSourceMetadata } from "./plugin-manifest.ts"; +export type PluginHookManifestLoadResult = { + readonly sources: readonly PluginHookSourceMetadata[]; + readonly parsed: ParsedHookConfig; + readonly diagnostics: readonly HookDiagnostic[]; +}; +export declare function loadPluginHookManifest(options: LoadPluginHookManifestOptions): PluginHookManifestLoadResult; +export declare function selectHookCommandForPlatform(handler: ExecutableHookHandler, platform?: NodeJS.Platform): string; +//# sourceMappingURL=plugin-loader.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/plugin-manifest.d.ts b/docs/upstream/senpi/hooks/plugin-manifest.d.ts new file mode 100644 index 0000000..02a90d5 --- /dev/null +++ b/docs/upstream/senpi/hooks/plugin-manifest.d.ts @@ -0,0 +1,47 @@ +import type { HookDiagnostic, HookDiscoveryTiming, HookSourceMetadata } from "./types.ts"; +export declare const MANIFEST_PATH = ".codex-plugin/plugin.json"; +export declare const DEFAULT_HOOK_PATH = "hooks/hooks.json"; +export type PluginHookEnv = { + readonly PLUGIN_ROOT: string; + readonly PLUGIN_DATA: string; + readonly CLAUDE_PLUGIN_ROOT: string; + readonly CLAUDE_PLUGIN_DATA: string; +}; +export type PluginHookSourceMetadata = HookSourceMetadata & { + readonly pluginEnv: PluginHookEnv; +}; +export type LoadPluginHookManifestOptions = { + readonly pluginRoot: string; + readonly displayOrder: number; + readonly discoveredAt?: HookDiscoveryTiming; + readonly dataRoot?: string; + readonly includeDefaultHooks?: boolean; +}; +export declare function buildPluginEnv(pluginRoot: string, dataRoot: string | undefined): PluginHookEnv; +export declare function pluginSource(input: { + readonly discoveredAt: HookDiscoveryTiming; + readonly displayOrder: number; + readonly env: PluginHookEnv; + readonly manifestPath: string; + readonly pluginRoot: string; + readonly sourcePath: string; +}): PluginHookSourceMetadata; +export declare function sourceForPath(options: LoadPluginHookManifestOptions, env: PluginHookEnv, sourcePath: string): PluginHookSourceMetadata; +export declare function resolveContainedPath(pluginRootInput: string, manifestPathInput: string): { + readonly ok: true; + readonly path: string; +} | { + readonly ok: false; + readonly message: string; +}; +export declare function readJsonFile(path: string, manifestPath: string, source: PluginHookSourceMetadata): { + readonly ok: true; + readonly value: unknown; +} | { + readonly ok: false; + readonly diagnostic: HookDiagnostic; +}; +export declare function fileExists(path: string): boolean; +export declare function directoryExists(path: string): boolean; +export declare function isRecord(value: unknown): value is Record; +//# sourceMappingURL=plugin-manifest.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/prompt-adapter.d.ts b/docs/upstream/senpi/hooks/prompt-adapter.d.ts new file mode 100644 index 0000000..fa99500 --- /dev/null +++ b/docs/upstream/senpi/hooks/prompt-adapter.d.ts @@ -0,0 +1,30 @@ +import type { HookDispatchResult } from "./dispatcher.ts"; +import type { HookDiagnostic, HookInputWire } from "./types.ts"; +export declare const HOOK_CUSTOM_MESSAGE_TYPE = "senpi.hook"; +export declare const USER_PROMPT_BLOCK_REASON = "UserPromptSubmit hook blocked the prompt."; +export type UserPromptHookInputOptions = { + readonly cwd: string; + readonly permissionMode: string; + readonly prompt: string; + readonly sessionId: string; + readonly transcriptPath?: string; +}; +export type PendingPromptHookContext = { + readonly additionalContext: readonly string[]; + readonly diagnostics: readonly HookDiagnostic[]; + readonly systemMessages: readonly string[]; +}; +export declare function buildUserPromptHookInput(options: UserPromptHookInputOptions): HookInputWire; +export declare function promptContextFromResult(result: HookDispatchResult): PendingPromptHookContext | undefined; +export declare function promptBlockReasonFromResult(result: HookDispatchResult): string; +export declare function formatPromptContextMessage(pending: PendingPromptHookContext): string | undefined; +export declare function appendSystemMessages(systemPrompt: string, messages: readonly string[]): string; +export declare function safeDiagnosticDetails(diagnostic: HookDiagnostic): { + readonly code: HookDiagnostic["code"]; + readonly event?: string; + readonly message: string; + readonly path: string; + readonly severity: HookDiagnostic["severity"]; + readonly sourcePath: string; +}; +//# sourceMappingURL=prompt-adapter.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/safety.d.ts b/docs/upstream/senpi/hooks/safety.d.ts new file mode 100644 index 0000000..7efdd8d --- /dev/null +++ b/docs/upstream/senpi/hooks/safety.d.ts @@ -0,0 +1,14 @@ +import type { ExecutableHookHandler, HookDiagnostic, HookInputWire } from "./types.ts"; +export { applyHookOutputSafety, DEFAULT_STDERR_LIMIT_BYTES, DEFAULT_STDOUT_LIMIT_BYTES, type HookOutputPolicy, type HookOutputSafetyMetadata, type HookSafeOutput, type HookStreamSafetyMetadata, } from "./output-bounds.ts"; +export declare const DEFAULT_HOOK_TIMEOUT_SECONDS = 600; +type HookEnvironmentOptions = { + readonly handler: ExecutableHookHandler; + readonly input: HookInputWire; + readonly sourceEnv: NodeJS.ProcessEnv; + readonly envPassthrough?: readonly string[]; +}; +export declare function resolveHookTimeoutSeconds(handler: ExecutableHookHandler): number; +export declare function isValidHookTimeoutSeconds(timeout: number): boolean; +export declare function buildHookEnvironment(options: HookEnvironmentOptions): NodeJS.ProcessEnv; +export declare function validateHookHandlerSafety(handler: ExecutableHookHandler): readonly HookDiagnostic[]; +//# sourceMappingURL=safety.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/schema.d.ts b/docs/upstream/senpi/hooks/schema.d.ts new file mode 100644 index 0000000..e17317d --- /dev/null +++ b/docs/upstream/senpi/hooks/schema.d.ts @@ -0,0 +1,3 @@ +import type { HookSourceMetadata, ParsedHookConfig } from "./types.ts"; +export declare function parseHookConfig(input: unknown, source: HookSourceMetadata): ParsedHookConfig; +//# sourceMappingURL=schema.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/stop-adapter.d.ts b/docs/upstream/senpi/hooks/stop-adapter.d.ts new file mode 100644 index 0000000..12058a7 --- /dev/null +++ b/docs/upstream/senpi/hooks/stop-adapter.d.ts @@ -0,0 +1,18 @@ +import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import type { ExtensionAPI, ExtensionContext } from "../../types.ts"; +import type { HookDispatchResult } from "./dispatcher.ts"; +import type { HookInputWire } from "./types.ts"; +export declare const STOP_STATE_CUSTOM_TYPE = "senpi.hooks.stop-state"; +export declare const STOP_DIAGNOSTICS_CUSTOM_TYPE = "senpi.hooks.stop-diagnostics"; +export declare const STOP_OUTPUT_CUSTOM_TYPE = "senpi.hooks.stop-output"; +type StopRuntime = Pick; +export declare function buildStopHookInput(event: { + readonly messages: readonly AgentMessage[]; +}, ctx: ExtensionContext): HookInputWire; +export declare function createStopTurnTracker(): { + readonly reset: () => void; + readonly turnKey: (ctx: ExtensionContext) => string; +}; +export declare function applyStopHookResult(pi: StopRuntime, ctx: ExtensionContext, result: HookDispatchResult, turnKey: string): Promise; +export {}; +//# sourceMappingURL=stop-adapter.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/tool-adapter.d.ts b/docs/upstream/senpi/hooks/tool-adapter.d.ts new file mode 100644 index 0000000..9c26733 --- /dev/null +++ b/docs/upstream/senpi/hooks/tool-adapter.d.ts @@ -0,0 +1,11 @@ +import type { ExtensionContext, ToolCallEvent, ToolCallEventResult, ToolResultEvent, ToolResultEventResult } from "../../types.ts"; +import type { HookDispatchResult } from "./dispatcher.ts"; +import type { HookInputWire } from "./types.ts"; +export declare const PRE_TOOL_BLOCK_REASON = "PreToolUse hook blocked the tool call."; +export declare const POST_TOOL_BLOCK_REASON = "PostToolUse hook flagged the tool result."; +export declare function buildPreToolUseHookInput(event: ToolCallEvent, ctx: ExtensionContext): HookInputWire; +export declare function buildPostToolUseHookInput(event: ToolResultEvent, ctx: ExtensionContext): HookInputWire; +export declare function applyPreToolUseResult(event: ToolCallEvent, result: HookDispatchResult): ToolCallEventResult | undefined; +export declare function applyPostToolUseResult(event: ToolResultEvent, result: HookDispatchResult, preToolContexts?: readonly string[]): ToolResultEventResult | undefined; +export declare function toolContextsFromResult(result: HookDispatchResult): readonly string[]; +//# sourceMappingURL=tool-adapter.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/trust-storage.d.ts b/docs/upstream/senpi/hooks/trust-storage.d.ts new file mode 100644 index 0000000..4f84822 --- /dev/null +++ b/docs/upstream/senpi/hooks/trust-storage.d.ts @@ -0,0 +1,24 @@ +import { type HookTrustStorageScope } from "./trust.ts"; +import type { HookTrustState } from "./types.ts"; +export interface HookStateStorage { + read(scope: HookTrustStorageScope): HookTrustState; + update(scope: HookTrustStorageScope, updater: (current: HookTrustState) => HookTrustState): HookTrustState; +} +export type FileHookStateStorageOptions = { + readonly agentDir?: string; + readonly cwd: string; +}; +export declare class FileHookStateStorage implements HookStateStorage { + private readonly globalStatePath; + private readonly projectStatePath; + constructor(options: FileHookStateStorageOptions); + read(scope: HookTrustStorageScope): HookTrustState; + update(scope: HookTrustStorageScope, updater: (current: HookTrustState) => HookTrustState): HookTrustState; +} +export declare class InMemoryHookStateStorage implements HookStateStorage { + private globalState; + private projectState; + read(scope: HookTrustStorageScope): HookTrustState; + update(scope: HookTrustStorageScope, updater: (current: HookTrustState) => HookTrustState): HookTrustState; +} +//# sourceMappingURL=trust-storage.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/trust-storage.js b/docs/upstream/senpi/hooks/trust-storage.js new file mode 100644 index 0000000..764d426 --- /dev/null +++ b/docs/upstream/senpi/hooks/trust-storage.js @@ -0,0 +1,102 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import lockfile from "proper-lockfile"; +import { CONFIG_DIR_NAME, getAgentDir } from "../../../../config.js"; +import { emptyHookTrustState, readHookTrustStateJson } from "./trust.js"; +export class FileHookStateStorage { + constructor(options) { + const agentDir = options.agentDir ?? getAgentDir(); + this.globalStatePath = join(agentDir, "hooks-state.json"); + this.projectStatePath = join(options.cwd, CONFIG_DIR_NAME, "hooks-state.json"); + } + read(scope) { + return withHookStateFileLock(statePathForScope(scope, this.globalStatePath, this.projectStatePath), (path) => readHookTrustStateJson(existsSync(path) ? readFileSync(path, "utf-8") : undefined)); + } + update(scope, updater) { + return withHookStateFileLock(statePathForScope(scope, this.globalStatePath, this.projectStatePath), (path) => { + const current = readHookTrustStateJson(existsSync(path) ? readFileSync(path, "utf-8") : undefined); + const next = updater(current); + writeFileSync(path, serializeHookTrustState(next), "utf-8"); + return next; + }); + } +} +export class InMemoryHookStateStorage { + constructor() { + this.globalState = emptyHookTrustState(); + this.projectState = emptyHookTrustState(); + } + read(scope) { + return scope === "global" ? this.globalState : this.projectState; + } + update(scope, updater) { + const next = updater(this.read(scope)); + if (scope === "global") { + this.globalState = next; + } + else { + this.projectState = next; + } + return next; + } +} +function statePathForScope(scope, globalStatePath, projectStatePath) { + return scope === "global" ? globalStatePath : projectStatePath; +} +function serializeHookTrustState(state) { + const sortedHooks = {}; + for (const key of Object.keys(state.hooks).sort()) { + const entry = state.hooks[key]; + if (entry !== undefined) { + sortedHooks[key] = entry; + } + } + return `${JSON.stringify({ version: 1, hooks: sortedHooks }, null, 2)}\n`; +} +function acquireHookStateLockSync(path) { + const stateDir = dirname(path); + mkdirSync(stateDir, { recursive: true }); + const maxAttempts = 10; + const delayMs = 20; + let lastError; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return lockfile.lockSync(stateDir, { realpath: false, lockfilePath: `${path}.lock` }); + } + catch (error) { + const code = errorCode(error); + if (code !== "ELOCKED" || attempt === maxAttempts) { + throw error; + } + lastError = error; + const start = Date.now(); + while (Date.now() - start < delayMs) { + Date.now(); + } + } + } + if (lastError instanceof Error) { + throw lastError; + } + throw new Error("Failed to acquire hook state lock"); +} +function errorCode(error) { + if (!isRecord(error)) { + return undefined; + } + const code = error.code; + return typeof code === "string" ? code : undefined; +} +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function withHookStateFileLock(path, fn) { + const release = acquireHookStateLockSync(path); + try { + return fn(path); + } + finally { + release(); + } +} +//# sourceMappingURL=trust-storage.js.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/trust.d.ts b/docs/upstream/senpi/hooks/trust.d.ts new file mode 100644 index 0000000..0430bd0 --- /dev/null +++ b/docs/upstream/senpi/hooks/trust.d.ts @@ -0,0 +1,34 @@ +import type { ExecutableHookHandler, HookSourceScope, HookTrustEntry, HookTrustState } from "./types.ts"; +export type HookTrustPlatform = NodeJS.Platform; +export type HookTrustStorageScope = "global" | "project"; +export type HookTrustOptions = { + readonly platform?: HookTrustPlatform; +}; +export type HookTrustStorageOptions = { + readonly projectTrusted: boolean; +}; +export type HookTrustRecord = { + readonly id: string; + readonly currentHash: string; + readonly enabled: boolean; + readonly trusted: boolean; + readonly executable: boolean; + readonly scope: HookSourceScope; + readonly sourcePath: string; + readonly matcher?: string; + readonly commandPreview: string; + readonly entry?: HookTrustEntry; +}; +export declare function emptyHookTrustState(): HookTrustState; +export declare function hookTrustId(handler: ExecutableHookHandler): string; +export declare function buildHookTrustRecord(handler: ExecutableHookHandler, options?: HookTrustOptions): HookTrustRecord; +export declare function createHookTrustEntry(handler: ExecutableHookHandler, options?: HookTrustOptions & { + readonly updatedAt?: string; +}): HookTrustEntry; +export declare function hashCommandHook(handler: ExecutableHookHandler, options?: HookTrustOptions): string; +export declare function isCommandHookTrusted(handler: ExecutableHookHandler, state: HookTrustState, options?: HookTrustOptions): boolean; +export declare function listHookTrustRecords(handlers: readonly ExecutableHookHandler[], state: HookTrustState, options?: HookTrustOptions): readonly HookTrustRecord[]; +export declare function filterExecutableTrustedHooks(handlers: readonly ExecutableHookHandler[], state: HookTrustState, options?: HookTrustOptions): readonly ExecutableHookHandler[]; +export declare function readHookTrustStateJson(input: string | undefined): HookTrustState; +export declare function hookTrustStorageScope(handler: ExecutableHookHandler, options: HookTrustStorageOptions): HookTrustStorageScope | undefined; +//# sourceMappingURL=trust.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/trust.js b/docs/upstream/senpi/hooks/trust.js new file mode 100644 index 0000000..f9c82f9 --- /dev/null +++ b/docs/upstream/senpi/hooks/trust.js @@ -0,0 +1,194 @@ +import { createHash } from "node:crypto"; +import { DEFAULT_HOOK_TIMEOUT_SECONDS, isValidHookTimeoutSeconds } from "./safety.js"; +const HOOK_STATE_VERSION = 1; +export function emptyHookTrustState() { + return { version: HOOK_STATE_VERSION, hooks: {} }; +} +export function hookTrustId(handler) { + return `hk_${sourceKeyHash(handler.source)}_${handler.event}_${handler.groupIndex}_${handler.handlerIndex}`; +} +export function buildHookTrustRecord(handler, options = {}) { + const id = hookTrustId(handler); + const currentHash = hashCommandHook(handler, options); + const commandPreview = selectedCommand(handler, options.platform ?? process.platform); + return { + id, + currentHash, + enabled: true, + trusted: false, + executable: false, + scope: handler.source.scope, + sourcePath: handler.source.sourcePath, + ...(handler.matcher === undefined ? {} : { matcher: handler.matcher }), + commandPreview, + }; +} +export function createHookTrustEntry(handler, options = {}) { + const commandPreview = selectedCommand(handler, options.platform ?? process.platform); + return { + enabled: true, + trustedHash: hashCommandHook(handler, options), + scope: handler.source.scope, + sourcePath: handler.source.sourcePath, + ...(handler.matcher === undefined ? {} : { matcher: handler.matcher }), + commandPreview, + updatedAt: options.updatedAt ?? new Date().toISOString(), + }; +} +export function hashCommandHook(handler, options = {}) { + const platform = options.platform ?? process.platform; + const hook = { + async: false, + command: handler.config.command, + commandWindows: handler.config.commandWindows, + platformCommand: selectedCommand(handler, platform), + statusMessage: handler.config.statusMessage, + timeout: normalizedTimeout(handler.config.timeout), + type: "command", + }; + const identity = { + event: handler.event, + hook, + matcher: handler.matcher, + sourceKeyHash: sourceKeyHash(handler.source), + }; + return `sha256:${sha256Hex(JSON.stringify(canonicalJson(identity)))}`; +} +export function isCommandHookTrusted(handler, state, options = {}) { + const record = buildStatefulHookTrustRecord(handler, state, options); + return record.executable; +} +export function listHookTrustRecords(handlers, state, options = {}) { + return handlers.map((handler) => buildStatefulHookTrustRecord(handler, state, options)); +} +export function filterExecutableTrustedHooks(handlers, state, options = {}) { + return handlers.filter((handler) => buildStatefulHookTrustRecord(handler, state, options).executable); +} +export function readHookTrustStateJson(input) { + if (input === undefined || input.trim() === "") { + return emptyHookTrustState(); + } + try { + return parseHookTrustState(JSON.parse(input)); + } + catch (error) { + if (error instanceof Error) { + return emptyHookTrustState(); + } + return emptyHookTrustState(); + } +} +export function hookTrustStorageScope(handler, options) { + if (handler.source.scope === "project") { + return options.projectTrusted ? "project" : undefined; + } + return "global"; +} +function buildStatefulHookTrustRecord(handler, state, options) { + const base = buildHookTrustRecord(handler, options); + const entry = state.hooks[base.id]; + const enabled = entry?.enabled ?? true; + const trusted = entry?.trustedHash === base.currentHash; + return { + ...base, + enabled, + trusted, + executable: enabled && trusted, + ...(entry === undefined ? {} : { entry }), + }; +} +function parseHookTrustState(input) { + if (!isRecord(input) || input.version !== HOOK_STATE_VERSION || !isRecord(input.hooks)) { + return emptyHookTrustState(); + } + const hooks = {}; + for (const [id, entry] of Object.entries(input.hooks)) { + const parsed = parseHookTrustEntry(entry); + if (parsed !== undefined) { + hooks[id] = parsed; + } + } + return { version: HOOK_STATE_VERSION, hooks }; +} +function parseHookTrustEntry(input) { + if (!isRecord(input)) { + return undefined; + } + const enabled = input.enabled; + const trustedHash = input.trustedHash; + const scope = input.scope; + const sourcePath = input.sourcePath; + const matcher = input.matcher; + const commandPreview = input.commandPreview; + const updatedAt = input.updatedAt; + if (typeof enabled !== "boolean" || + (trustedHash !== undefined && typeof trustedHash !== "string") || + !isHookSourceScope(scope) || + typeof sourcePath !== "string" || + (matcher !== undefined && typeof matcher !== "string") || + typeof commandPreview !== "string" || + typeof updatedAt !== "string") { + return undefined; + } + return { + enabled, + ...(trustedHash === undefined ? {} : { trustedHash }), + scope, + sourcePath, + ...(matcher === undefined ? {} : { matcher }), + commandPreview, + updatedAt, + }; +} +function selectedCommand(handler, platform) { + if (platform === "win32" && handler.config.commandWindows !== undefined) { + return handler.config.commandWindows; + } + return handler.config.command; +} +function normalizedTimeout(timeout) { + if (timeout === undefined) { + return DEFAULT_HOOK_TIMEOUT_SECONDS; + } + if (!isValidHookTimeoutSeconds(timeout)) { + throw new Error("Invalid command hook timeout reached trust hashing."); + } + return timeout; +} +function sourceKeyHash(source) { + return sha256Hex([source.scope, source.sourcePath, source.pluginRoot ?? "", source.manifestPath ?? ""].join("\0")).slice(0, 12); +} +function sha256Hex(value) { + return createHash("sha256").update(value).digest("hex"); +} +function canonicalJson(value) { + if (Array.isArray(value)) { + return value.map(canonicalJson); + } + if (!isJsonRecord(value)) { + return value; + } + const result = {}; + for (const key of Object.keys(value).sort()) { + const child = value[key]; + if (child !== undefined) { + result[key] = canonicalJson(child); + } + } + return result; +} +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function isJsonRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function isHookSourceScope(value) { + return (value === "global" || + value === "project" || + value === "plugin" || + value === "runtime" || + value === "cli" || + value === "managed"); +} +//# sourceMappingURL=trust.js.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/types.d.ts b/docs/upstream/senpi/hooks/types.d.ts new file mode 100644 index 0000000..ac1a26f --- /dev/null +++ b/docs/upstream/senpi/hooks/types.d.ts @@ -0,0 +1,134 @@ +export declare const SUPPORTED_HOOK_EVENTS: readonly ["PreToolUse", "PostToolUse", "UserPromptSubmit", "SessionStart", "PreCompact", "PostCompact", "Stop"]; +export declare const UNSUPPORTED_KNOWN_HOOK_EVENTS: readonly ["PermissionRequest", "PermissionDenied", "SubagentStart", "SubagentStop", "Notification", "Setup", "UserPromptExpansion", "PostToolUseFailure", "PostToolBatch", "TaskCreated", "TaskCompleted", "StopFailure", "TeammateIdle", "InstructionsLoaded", "ConfigChange", "CwdChanged", "FileChanged", "WorktreeCreate", "WorktreeRemove", "MessageDisplay", "SessionEnd", "Elicitation", "ElicitationResult"]; +export declare const UNSUPPORTED_HANDLER_TYPES: readonly ["prompt", "agent", "http", "mcp_tool"]; +export type SupportedHookEvent = (typeof SUPPORTED_HOOK_EVENTS)[number]; +export type UnsupportedKnownHookEvent = (typeof UNSUPPORTED_KNOWN_HOOK_EVENTS)[number]; +export type HookSourceScope = "global" | "project" | "plugin" | "runtime" | "cli" | "managed"; +export type HookDiscoveryTiming = "pre-session" | "runtime"; +export type HookSourceMetadata = { + readonly scope: HookSourceScope; + readonly sourcePath: string; + readonly displayOrder: number; + readonly discoveredAt: HookDiscoveryTiming; + readonly pluginRoot?: string; + readonly manifestPath?: string; +}; +export type CommandHookConfig = { + readonly type: "command"; + readonly command: string; + readonly commandWindows?: string; + readonly timeout?: number; + readonly statusMessage?: string; +}; +export type ExecutableHookHandler = { + readonly event: SupportedHookEvent; + readonly matcher?: string; + readonly groupIndex: number; + readonly handlerIndex: number; + readonly config: CommandHookConfig; + readonly source: HookSourceMetadata; +}; +export type HookDiagnosticCode = "invalid_root" | "invalid_hooks" | "invalid_event_config" | "invalid_matcher" | "invalid_handler_group" | "invalid_handler_list" | "invalid_handler" | "invalid_command" | "invalid_command_windows" | "invalid_command_target" | "missing_command_target" | "invalid_timeout" | "invalid_status_message" | "unknown_event" | "unsupported_event" | "unsupported_field" | "unsupported_handler_type" | "unsupported_async_handler" | "unsupported_command_variant"; +export type HookDiagnostic = { + readonly code: HookDiagnosticCode; + readonly severity: "error" | "warning"; + readonly message: string; + readonly path: string; + readonly source: HookSourceMetadata; + readonly event?: string; +}; +export type ParsedHookConfig = { + readonly executableHandlers: readonly ExecutableHookHandler[]; + readonly diagnostics: readonly HookDiagnostic[]; +}; +export type HookTrustEntry = { + readonly enabled: boolean; + readonly trustedHash?: string; + readonly scope: HookSourceScope; + readonly sourcePath: string; + readonly matcher?: string; + readonly commandPreview: string; + readonly updatedAt: string; +}; +export type HookTrustState = { + readonly version: 1; + readonly hooks: Readonly>; +}; +export type HookRuntimeState = { + readonly parsed: ParsedHookConfig; + readonly trust: HookTrustState; +}; +export type HookInputWire = { + readonly event: "SessionStart"; + readonly sessionId: string; + readonly cwd: string; + readonly hook_event_name?: "SessionStart"; + readonly reason?: string; + readonly session_id?: string; + readonly transcript_path?: string; +} | { + readonly event: "UserPromptSubmit"; + readonly prompt: string; + readonly cwd: string; + readonly session_id?: string; + readonly permission_mode?: string; + readonly transcript_path?: string; +} | { + readonly event: "PreToolUse"; + readonly toolName: string; + readonly toolInput: unknown; + readonly cwd: string; + readonly session_id?: string; + readonly hook_event_name?: "PreToolUse"; + readonly tool_name?: string; + readonly tool_input?: unknown; + readonly tool_use_id?: string; +} | { + readonly event: "PostToolUse"; + readonly toolName: string; + readonly toolInput: unknown; + readonly toolOutput: unknown; + readonly cwd: string; + readonly session_id?: string; + readonly hook_event_name?: "PostToolUse"; + readonly tool_name?: string; + readonly tool_input?: unknown; + readonly tool_response?: unknown; + readonly tool_use_id?: string; +} | { + readonly event: "PreCompact"; + readonly reason: string; + readonly cwd: string; + readonly custom_instructions?: string; + readonly hook_event_name?: "PreCompact"; + readonly request_id?: string; + readonly session_id?: string; + readonly transcript_path?: string; + readonly will_retry?: boolean; +} | { + readonly event: "PostCompact"; + readonly reason: string; + readonly cwd: string; + readonly accepted?: boolean; + readonly hook_event_name?: "PostCompact"; + readonly request_id?: string; + readonly session_id?: string; + readonly transcript_path?: string; + readonly will_retry?: boolean; +} | { + readonly event: "Stop"; + readonly stopReason?: string; + readonly cwd: string; + readonly hook_event_name?: "Stop"; + readonly session_id?: string; + readonly transcript_path?: string; +}; +export type HookOutputWire = { + readonly decision?: "approve" | "block" | "deny" | "ask"; + readonly reason?: string; + readonly additionalContext?: string; + readonly updatedInput?: unknown; + readonly updatedToolOutput?: unknown; + readonly continue?: boolean; +}; +//# sourceMappingURL=types.d.ts.map \ No newline at end of file diff --git a/docs/upstream/senpi/hooks/types.js b/docs/upstream/senpi/hooks/types.js new file mode 100644 index 0000000..ec32f17 --- /dev/null +++ b/docs/upstream/senpi/hooks/types.js @@ -0,0 +1,36 @@ +export const SUPPORTED_HOOK_EVENTS = [ + "PreToolUse", + "PostToolUse", + "UserPromptSubmit", + "SessionStart", + "PreCompact", + "PostCompact", + "Stop", +]; +export const UNSUPPORTED_KNOWN_HOOK_EVENTS = [ + "PermissionRequest", + "PermissionDenied", + "SubagentStart", + "SubagentStop", + "Notification", + "Setup", + "UserPromptExpansion", + "PostToolUseFailure", + "PostToolBatch", + "TaskCreated", + "TaskCompleted", + "StopFailure", + "TeammateIdle", + "InstructionsLoaded", + "ConfigChange", + "CwdChanged", + "FileChanged", + "WorktreeCreate", + "WorktreeRemove", + "MessageDisplay", + "SessionEnd", + "Elicitation", + "ElicitationResult", +]; +export const UNSUPPORTED_HANDLER_TYPES = ["prompt", "agent", "http", "mcp_tool"]; +//# sourceMappingURL=types.js.map \ No newline at end of file diff --git a/docs/upstream/senpi/pin.json b/docs/upstream/senpi/pin.json new file mode 100644 index 0000000..68b047b --- /dev/null +++ b/docs/upstream/senpi/pin.json @@ -0,0 +1,131 @@ +{ + "engineVersion": "2026.8.19", + "pinnedAt": "2026-08-21", + "source": "npm", + "registryIntegrity": "sha512-Uo+oI3DeYNn43ehmaLBFn9zDB+TojNtOecrTTrjE9iouxhIvMgee6yiMPveVIU5e5rk6sX3T96POOSTR/9eODQ==", + "files": { + "session-format.md": { + "upstreamPath": "docs/session-format.md", + "sha256": "628785e46242ee18ebae05263a0b412446330a97d9ea4c206791d95e0918e077" + }, + "settings.md": { + "upstreamPath": "docs/settings.md", + "sha256": "fcf50b66e794cfadebe30e0f9dbd9ee7ed9022df157d8c5bccf876c3867b4e54" + }, + "environment-variables.md": { + "upstreamPath": "docs/environment-variables.md", + "sha256": "f39e6e4ba0697d2fd349d5b61ec188f1456fe61017506089f27996b11ffebc64" + }, + "hooks/types.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/types.d.ts", + "sha256": "055c19cb992d0eee0ad8de2c27eec40b0dd513b9db05ddad5669bd5d4cce988e" + }, + "hooks/index.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/index.d.ts", + "sha256": "88fb5dbe99b67884cfb634ca0fd6cd1b33b516505da5704ddc5d83f63e14f6a3" + }, + "hooks/schema.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/schema.d.ts", + "sha256": "55e27b2ad71bfe3540028f31687584b65d2178f47fddbe36ad48214a64f571f0" + }, + "hooks/trust.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/trust.d.ts", + "sha256": "9fab928fae267556ee3856a8a754d0cacb6e34848b86a96aeebd0d592ead95a8" + }, + "hooks/trust-storage.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/trust-storage.d.ts", + "sha256": "8cdb99403b355ef85237a40d15e1a8a6fd074f36dbc792efa12c2c67660c9dcc" + }, + "hooks/config-loader.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/config-loader.d.ts", + "sha256": "c8f3e342eb7a67f2e589a4bdb1d82d8326ee3910a4295f80da3934d67c3661e6" + }, + "hooks/command-runner.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/command-runner.d.ts", + "sha256": "1261e91c0dbfb3c1802aa8c8d074d36ed77e77bcf7f0897747f1dc55b4f801dc" + }, + "hooks/dispatcher.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/dispatcher.d.ts", + "sha256": "f516782cfd3d4f9b3ce5eed98a5c1418afa87159ba80d42194c879d0fd050f85" + }, + "hooks/output-parser.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/output-parser.d.ts", + "sha256": "122f7782f475c2c8c65af964ab7d92f891c7687b037a59a562007b5c285f3ec2" + }, + "hooks/output-bounds.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/output-bounds.d.ts", + "sha256": "58484814214b8e57164c20ebde8f7a1f47af6b148bc29245bb778f7a1085e084" + }, + "hooks/safety.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/safety.d.ts", + "sha256": "e1af2b6f554c314027fb3106e4fb5e5545ad71b8a7d07abe2c8650ee39b8edc0" + }, + "hooks/matcher.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/matcher.d.ts", + "sha256": "b077d8fe2a62638458442232319da0af0b54360b6351b6b1b8eac51e9ec2df11" + }, + "hooks/command.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/command.d.ts", + "sha256": "f4c24842d007282c21148e78f105717953f1aa854608c9ebac18879fa4e1fbca" + }, + "hooks/handler.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/handler.d.ts", + "sha256": "049945d65c3bf65622deeb16023836b1f5f06de7f34ead98177c25cfb992e337" + }, + "hooks/diagnostics.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/diagnostics.d.ts", + "sha256": "30923d1ff2c50b99c1f01d9458986319bd286f3b26b3709e6df17bf4fe6615ee" + }, + "hooks/plugin-loader.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/plugin-loader.d.ts", + "sha256": "b2b3f4db03c207ad3bb7b5a4d3f050afabf8d4f208443f854f04cfd64703f5b4" + }, + "hooks/plugin-manifest.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/plugin-manifest.d.ts", + "sha256": "b24cab50b2f8da596cdf2e556ed13adc4b7f1215fb4481350e31967f21d5493b" + }, + "hooks/lifecycle-adapter.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/lifecycle-adapter.d.ts", + "sha256": "71e6e5cb586ea11a81ba897f631ba1c93d13d8ad315284524a93037fabe01806" + }, + "hooks/tool-adapter.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/tool-adapter.d.ts", + "sha256": "77cfc522f8c2022a06223abde77d2408b4c56dbcc01b8670f740511e615e2c7c" + }, + "hooks/stop-adapter.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/stop-adapter.d.ts", + "sha256": "3e1164dddb12dbb64e43fb4c117162aaf54b811506b1f754d6b49f0df7067817" + }, + "hooks/prompt-adapter.d.ts": { + "upstreamPath": "dist/core/extensions/builtin/hooks/prompt-adapter.d.ts", + "sha256": "e2843c96e3c7919c9ef8bdd52fb844cbd0e9937d30c85e0ff27ce57f3dc85a38" + }, + "hooks/trust.js": { + "upstreamPath": "dist/core/extensions/builtin/hooks/trust.js", + "sha256": "396160f2a40ceed4e6f19e94309946de1eadbb88b91965f522cf73c2822dd477" + }, + "hooks/output-parser.js": { + "upstreamPath": "dist/core/extensions/builtin/hooks/output-parser.js", + "sha256": "a4b9d7080c751ee8ea87a4427bec9b56a6d5ccaf2e1b2f73956a288bf4076636" + }, + "hooks/trust-storage.js": { + "upstreamPath": "dist/core/extensions/builtin/hooks/trust-storage.js", + "sha256": "a7ad655a4d0c5a48119aa8dc36d7c8ab9e0ae4bcc13b4798c8c35f205d42fa1d" + }, + "hooks/output-bounds.js": { + "upstreamPath": "dist/core/extensions/builtin/hooks/output-bounds.js", + "sha256": "d9261a519e02173172b42a23b98d2d8b6df21154628f040229f5675e2dfca2d0" + }, + "hooks/types.js": { + "upstreamPath": "dist/core/extensions/builtin/hooks/types.js", + "sha256": "7bbb639e976a932f03eeec5aede5d7a733ac896deb39157f123a5af855fc6e03" + } + }, + "fixtureRedump": "copy small redacted *.jsonl from ~/.omo/agent/sessions// into tests/fixtures/senpi/", + "notes": [ + "Artifact of record is the npm tarball for @code-yeongyu/senpi@2026.8.19; registryIntegrity is npm view dist.integrity (sha512).", + "Vendored set is complete at pin time: 3 docs, 21 hooks .d.ts, and 5 hooks .js implementations required because .d.ts signatures alone are insufficient (hashCommandHook, output parsing, trust storage lock paths, output bounds, types constants).", + "No runtime import of @code-yeongyu/senpi; drift checks re-extract from a provided local tarball only (no network at check time).", + "Upstream package license is MIT (code-yeongyu/senpi)." + ] +} diff --git a/docs/upstream/senpi/session-format.md b/docs/upstream/senpi/session-format.md new file mode 100644 index 0000000..1c3f669 --- /dev/null +++ b/docs/upstream/senpi/session-format.md @@ -0,0 +1,442 @@ +# Session File Format + +Sessions are stored as JSONL (JSON Lines) files. Each line is a JSON object with a `type` field. Session entries form a tree structure via `id`/`parentId` fields, enabling in-place branching without creating new files. + +## File Location + +``` +~/.senpi/agent/sessions/----/_.jsonl +``` + +Where `` is the working directory with `/` replaced by `-`. + +## Deleting Sessions + +Sessions can be removed by deleting their `.jsonl` files under `~/.senpi/agent/sessions/`. + +Senpi also supports deleting sessions interactively from `/resume` (select a session and press `Ctrl+D`, then confirm). When available, senpi uses the `trash` CLI to avoid permanent deletion. + +## Session Version + +Sessions have a version field in the header: + +- **Version 1**: Linear entry sequence (legacy, auto-migrated on load) +- **Version 2**: Tree structure with `id`/`parentId` linking +- **Version 3**: Renamed `hookMessage` role to `custom` (extensions unification) + +Existing sessions are automatically migrated to the current version (v3) when loaded. + +## Source Files + +Source on GitHub ([pi-mono](https://github.com/earendil-works/pi-mono)): +- [`packages/coding-agent/src/core/session-manager.ts`](https://github.com/earendil-works/pi-mono/blob/main/packages/coding-agent/src/core/session-manager.ts) - Session entry types and SessionManager +- [`packages/coding-agent/src/core/messages.ts`](https://github.com/earendil-works/pi-mono/blob/main/packages/coding-agent/src/core/messages.ts) - Extended message types (BashExecutionMessage, CustomMessage, etc.) +- [`packages/ai/src/types.ts`](https://github.com/earendil-works/pi-mono/blob/main/packages/ai/src/types.ts) - Base message types (UserMessage, AssistantMessage, ToolResultMessage) +- [`packages/agent/src/types.ts`](https://github.com/earendil-works/pi-mono/blob/main/packages/agent/src/types.ts) - AgentMessage union type + +For TypeScript definitions in your project, inspect `node_modules/@code-yeongyu/senpi/dist/` and `node_modules/@earendil-works/pi-ai/dist/`. + +## Message Types + +Session entries contain `AgentMessage` objects. Understanding these types is essential for parsing sessions and writing extensions. + +### Content Blocks + +Messages contain arrays of typed content blocks: + +```typescript +interface TextContent { + type: "text"; + text: string; +} + +interface ImageContent { + type: "image"; + data: string; // base64 encoded + mimeType: string; // e.g., "image/jpeg", "image/png" +} + +interface ThinkingContent { + type: "thinking"; + thinking: string; + startedAt?: number; // Unix epoch milliseconds + endedAt?: number; // Unix epoch milliseconds +} + +`startedAt` and `endedAt` are optional epoch milliseconds stamped by the agent loop at stream-event receipt on a best-effort basis. They are absent on pre-feature sessions and on messages not produced through the agent loop. Renderers must treat their absence as "no timing available". + +interface ToolCall { + type: "toolCall"; + id: string; + name: string; + arguments: Record; +} +``` + +### Base Message Types (from pi-ai) + +```typescript +interface UserMessage { + role: "user"; + content: string | (TextContent | ImageContent)[]; + timestamp: number; // Unix ms +} + +interface AssistantMessage { + role: "assistant"; + content: (TextContent | ThinkingContent | ToolCall)[]; + api: string; + provider: string; + model: string; + usage: Usage; + stopReason: "stop" | "length" | "toolUse" | "error" | "aborted"; + errorMessage?: string; + timestamp: number; +} + +interface ToolResultMessage { + role: "toolResult"; + toolCallId: string; + toolName: string; + content: (TextContent | ImageContent)[]; + details?: any; // Tool-specific metadata + usage?: Usage; // Nested LLM work performed by the tool + isError: boolean; + timestamp: number; +} + +interface Usage { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + totalTokens: number; + cost: { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + total: number; + }; +} +``` + +The exported pi-ai `StopReason` type also includes `"pending"`, but that value is reserved for partial messages in streaming events. Terminal `done`/`error` messages replace it with a completion reason before Senpi persists the assistant message, so `"pending"` should never appear in session JSONL. + +### Extended Message Types (from @code-yeongyu/senpi) + +```typescript +interface BashExecutionMessage { + role: "bashExecution"; + command: string; + output: string; + exitCode: number | undefined; + cancelled: boolean; + truncated: boolean; + fullOutputPath?: string; + excludeFromContext?: boolean; // true for !! prefix commands + timestamp: number; +} + +interface CustomMessage { + role: "custom"; + customType: string; // Extension identifier + content: string | (TextContent | ImageContent)[]; + display: boolean; // Show in TUI + details?: any; // Extension-specific metadata + timestamp: number; +} + +interface BranchSummaryMessage { + role: "branchSummary"; + summary: string; + fromId: string; // Entry we branched from + timestamp: number; +} + +interface CompactionSummaryMessage { + role: "compactionSummary"; + summary: string; + tokensBefore: number; + timestamp: number; +} +``` + +### AgentMessage Union + +```typescript +type AgentMessage = + | UserMessage + | AssistantMessage + | ToolResultMessage + | BashExecutionMessage + | CustomMessage + | BranchSummaryMessage + | CompactionSummaryMessage; +``` + +## Entry Base + +All entries (except `SessionHeader`) extend `SessionEntryBase`: + +```typescript +interface SessionEntryBase { + type: string; + id: string; // 8-char hex ID + parentId: string | null; // Parent entry ID (null for first entry) + timestamp: string; // ISO timestamp +} +``` + +## Entry Types + +### SessionHeader + +First line of the file. Metadata only, not part of the tree (no `id`/`parentId`). + +```json +{"type":"session","version":3,"id":"uuid","timestamp":"2024-12-03T14:00:00.000Z","cwd":"/path/to/project"} +``` + +For sessions with a parent (created via `/fork`, `/clone`, or `newSession({ parentSession })`): + +```json +{"type":"session","version":3,"id":"uuid","timestamp":"2024-12-03T14:00:00.000Z","cwd":"/path/to/project","parentSession":"/path/to/original/session.jsonl"} +``` + +### SessionMessageEntry + +A message in the conversation. The `message` field contains an `AgentMessage`. + +```json +{"type":"message","id":"a1b2c3d4","parentId":"prev1234","timestamp":"2024-12-03T14:00:01.000Z","message":{"role":"user","content":"Hello"}} +{"type":"message","id":"b2c3d4e5","parentId":"a1b2c3d4","timestamp":"2024-12-03T14:00:02.000Z","message":{"role":"assistant","content":[{"type":"text","text":"Hi!"}],"provider":"anthropic","model":"claude-sonnet-4-5","usage":{...},"stopReason":"stop"}} +{"type":"message","id":"c3d4e5f6","parentId":"b2c3d4e5","timestamp":"2024-12-03T14:00:03.000Z","message":{"role":"toolResult","toolCallId":"call_123","toolName":"bash","content":[{"type":"text","text":"output"}],"isError":false}} +``` + +### ModelChangeEntry + +Emitted when the user switches models mid-session. + +```json +{"type":"model_change","id":"d4e5f6g7","parentId":"c3d4e5f6","timestamp":"2024-12-03T14:05:00.000Z","provider":"openai","modelId":"gpt-4o"} +``` + +### ThinkingLevelChangeEntry + +Emitted when the user changes the thinking/reasoning level. + +```json +{"type":"thinking_level_change","id":"e5f6g7h8","parentId":"d4e5f6g7","timestamp":"2024-12-03T14:06:00.000Z","thinkingLevel":"high"} +``` + +### CompactionEntry + +Created when context is compacted. Stores a summary of earlier messages. + +```json +{"type":"compaction","id":"f6g7h8i9","parentId":"e5f6g7h8","timestamp":"2024-12-03T14:10:00.000Z","summary":"User discussed X, Y, Z...","firstKeptEntryId":"c3d4e5f6","tokensBefore":50000} +``` + +Newer harness-generated compactions embed the retained post-compaction context directly on the entry, instead of `firstKeptEntryId`: + +```json +{"type":"compaction","id":"f6g7h8i9","parentId":"e5f6g7h8","timestamp":"2024-12-03T14:10:00.000Z","summary":"User discussed X, Y, Z...","tokensBefore":50000,"retainedTail":[{"role":"user","content":"latest request"},{"role":"assistant","content":[{"type":"text","text":"latest reply"}],"provider":"anthropic","model":"claude-sonnet-4-5","usage":{...},"stopReason":"stop"}]} +``` + +Optional fields: +- `usage`: LLM usage from generating the summary; included in session token and cost totals +- `retainedTail`: Materialized `AgentMessage[]` kept after compaction. This is optional only for backward compatibility with older sessions. Newer harness-generated compactions include it so we can rebuild context from this checkpoint without walking older entries before the compaction entry. +- `details`: Implementation-specific data (e.g., `{ readFiles: string[], modifiedFiles: string[] }` for default, or custom data for extensions) +- `fromHook`: `true` if generated by an extension, `false`/`undefined` if senpi-generated (legacy field name) +- `firstKeptEntryId`: for compatibility with old entry format. + +### BranchSummaryEntry + +Created when switching branches via `/tree` with an LLM generated summary of the left branch up to the common ancestor. Captures context from the abandoned path. + +```json +{"type":"branch_summary","id":"g7h8i9j0","parentId":"a1b2c3d4","timestamp":"2024-12-03T14:15:00.000Z","fromId":"f6g7h8i9","summary":"Branch explored approach A..."} +``` + +Optional fields: +- `usage`: LLM usage from generating the summary; included in session token and cost totals +- `details`: File tracking data (`{ readFiles: string[], modifiedFiles: string[] }`) for default, or custom data for extensions +- `fromHook`: `true` if generated by an extension, `false`/`undefined` if senpi-generated (legacy field name) + +### CustomEntry + +Extension state persistence. Does NOT participate in LLM context. + +```json +{"type":"custom","id":"h8i9j0k1","parentId":"g7h8i9j0","timestamp":"2024-12-03T14:20:00.000Z","customType":"my-extension","data":{"count":42}} +``` + +Use `customType` to identify your extension's entries on reload. Interactive mode can render custom entries via `pi.registerEntryRenderer(customType, renderer)`, but they still do not participate in LLM context. + +### CustomMessageEntry + +Extension-injected messages that DO participate in LLM context. + +```json +{"type":"custom_message","id":"i9j0k1l2","parentId":"h8i9j0k1","timestamp":"2024-12-03T14:25:00.000Z","customType":"my-extension","content":"Injected context...","display":true} +``` + +Fields: +- `content`: String or `(TextContent | ImageContent)[]` (same as UserMessage) +- `display`: `true` = show in TUI with distinct styling, `false` = hidden +- `details`: Optional extension-specific metadata (not sent to LLM) + +### LabelEntry + +User-defined bookmark/marker on an entry. + +```json +{"type":"label","id":"j0k1l2m3","parentId":"i9j0k1l2","timestamp":"2024-12-03T14:30:00.000Z","targetId":"a1b2c3d4","label":"checkpoint-1"} +``` + +Set `label` to `undefined` to clear a label. + +### SessionInfoEntry + +Session metadata (e.g., user-defined display name). Set via `/name`, `--name` / `-n`, or `pi.setSessionName()` in extensions. + +```json +{"type":"session_info","id":"k1l2m3n4","parentId":"j0k1l2m3","timestamp":"2024-12-03T14:35:00.000Z","name":"Refactor auth module"} +``` + +The session name is displayed in the session selector (`/resume`) instead of the first message when set. + +## Tree Structure + +Entries form a tree: +- First entry has `parentId: null` +- Each subsequent entry points to its parent via `parentId` +- Branching creates new children from an earlier entry +- The "leaf" is the current position in the tree + +``` +[user msg] ─── [assistant] ─── [user msg] ─── [assistant] ─┬─ [user msg] ← current leaf + │ + └─ [branch_summary] ─── [user msg] ← alternate branch +``` + +## Context Building + +`buildContextEntries()` walks from the current leaf to the root, producing the active entry list while honoring compaction: + +1. Collects all entries on the path +2. If a `CompactionEntry` is on the path: + - Includes the compaction entry first + - If `retainedTail` is present, it acts as a self-contained checkpoint and entries after the compaction are included + - Otherwise entries from `firstKeptEntryId` to the compaction are included + - Then entries after compaction are included +3. Preserves non-message entries in the selected range so interactive mode can render them + +`buildSessionContext()` builds on that entry list to produce the message list for the LLM: + +1. Extracts current model and thinking level settings from the full path +2. Converts selected entries to messages: + - `message` -> stored `AgentMessage` + - `compaction` -> `compactionSummary` plus `retainedTail` when present + - `branch_summary` -> `branchSummary` + - `custom_message` -> `CustomMessage` + - `custom` -> no context message + +This makes newer compactions act like self-contained checkpoints. `retainedTail` is optional only so older sessions that only store `firstKeptEntryId` continue to load correctly. + +## Parsing Example + +```typescript +import { readFileSync } from "fs"; + +const lines = readFileSync("session.jsonl", "utf8").trim().split("\n"); + +for (const line of lines) { + const entry = JSON.parse(line); + + switch (entry.type) { + case "session": + console.log(`Session v${entry.version ?? 1}: ${entry.id}`); + break; + case "message": + console.log(`[${entry.id}] ${entry.message.role}: ${JSON.stringify(entry.message.content)}`); + break; + case "compaction": + console.log(`[${entry.id}] Compaction: ${entry.tokensBefore} tokens summarized`); + break; + case "branch_summary": + console.log(`[${entry.id}] Branch from ${entry.fromId}`); + break; + case "custom": + console.log(`[${entry.id}] Custom (${entry.customType}): ${JSON.stringify(entry.data)}`); + break; + case "custom_message": + console.log(`[${entry.id}] Extension message (${entry.customType}): ${entry.content}`); + break; + case "label": + console.log(`[${entry.id}] Label "${entry.label}" on ${entry.targetId}`); + break; + case "model_change": + console.log(`[${entry.id}] Model: ${entry.provider}/${entry.modelId}`); + break; + case "thinking_level_change": + console.log(`[${entry.id}] Thinking: ${entry.thinkingLevel}`); + break; + } +} +``` + +## SessionManager API + +Key methods for working with sessions programmatically. + +### Static Creation Methods +- `SessionManager.create(cwd, sessionDir?)` - New session +- `SessionManager.open(path, sessionDir?)` - Open existing session file +- `SessionManager.continueRecent(cwd, sessionDir?)` - Continue most recent or create new +- `SessionManager.inMemory(cwd?)` - No file persistence +- `SessionManager.forkFrom(sourcePath, targetCwd, sessionDir?)` - Fork session from another project + +### Static Listing Methods +- `SessionManager.list(cwd, sessionDir?, onProgress?)` - List sessions for a directory +- `SessionManager.listAll(onProgress?)` - List all sessions across all projects + +### Instance Methods - Session Management +- `newSession(options?)` - Start a new session (options: `{ parentSession?: string }`) +- `setSessionFile(path)` - Switch to a different session file +- `createBranchedSession(leafId)` - Extract branch to new session file + +### Instance Methods - Appending (all return entry ID) +- `appendMessage(message)` - Add message +- `appendThinkingLevelChange(level)` - Record thinking change +- `appendModelChange(provider, modelId)` - Record model change +- `appendCompaction(summary, firstKeptEntryId, tokensBefore, details?, fromHook?)` - Add compaction +- `appendCustomEntry(customType, data?)` - Extension state (not in context) +- `appendSessionInfo(name)` - Set session display name +- `appendCustomMessageEntry(customType, content, display, details?)` - Extension message (in context) +- `appendLabelChange(targetId, label)` - Set/clear label + +### Instance Methods - Tree Navigation +- `getLeafId()` - Current position +- `getLeafEntry()` - Get current leaf entry +- `getEntry(id)` - Get entry by ID +- `getBranch(fromId?)` - Walk from entry to root +- `getTree()` - Get full tree structure +- `getChildren(parentId)` - Get direct children +- `getLabel(id)` - Get label for entry +- `branch(entryId)` - Move leaf to earlier entry +- `resetLeaf()` - Reset leaf to null (before any entries) +- `branchWithSummary(entryId, summary, details?, fromHook?)` - Branch with context summary + +### Instance Methods - Context & Info +- `buildContextEntries()` - Get active branch entries with compaction applied +- `buildSessionContext()` - Get messages, thinkingLevel, and model for LLM +- `getEntries()` - All entries (excluding header) +- `getHeader()` - Session header metadata +- `getSessionName()` - Get display name from latest session_info entry +- `getCwd()` - Working directory +- `getSessionDir()` - Session storage directory +- `getSessionId()` - Session UUID +- `getSessionFile()` - Session file path (undefined for in-memory) +- `isPersisted()` - Whether session is saved to disk diff --git a/docs/upstream/senpi/settings.md b/docs/upstream/senpi/settings.md new file mode 100644 index 0000000..422d113 --- /dev/null +++ b/docs/upstream/senpi/settings.md @@ -0,0 +1,598 @@ +# Settings + +Senpi uses JSON settings files with project settings overriding global settings. + +| Location | Scope | +|----------|-------| +| `~/.senpi/agent/settings.json` | Global (all projects) | +| `.senpi/settings.json` | Project (current directory) | + +Edit directly or use `/settings` for common options. + +## Project Trust + +On interactive startup, senpi asks before trusting a project folder that contains project-local settings, resources, or project `.agents/skills` and has no saved decision for the folder or a parent folder in `~/.senpi/agent/trust.json`. Trusting a project allows senpi to load `.senpi/settings.json` and `.senpi` resources, install missing project packages, and execute project extensions. + +Non-interactive modes (`-p`, `--mode json`, and `--mode rpc`) do not show a trust prompt. Without an applicable saved trust decision, they use `defaultProjectTrust` from global settings: `ask` (default) and `never` ignore those project resources, while `always` trusts them. Pass `--approve`/`-a` or `--no-approve`/`-na` to override project trust for one run. + +If no extension or saved decision applies, `defaultProjectTrust` controls the fallback behavior. Set it to `"ask"`, `"always"`, or `"never"` in `~/.senpi/agent/settings.json`, or change it with `/settings`. + +`senpi config` and package commands use the same project trust flow, except `senpi update` never prompts. Pass `--approve` to trust project-local settings for one command or `--no-approve` to ignore them. + +Use `/trust` in interactive mode to save a project trust decision for future sessions, including trust for the immediate parent folder. It writes `~/.senpi/agent/trust.json` only; the current session is not reloaded, so restart senpi for changes to take effect. + +## Permissions + +Senpi includes a built-in permission system for tool calls. It evaluates a preset first, then applies explicit rules from global settings, project settings, and CLI flags. The last matching rule wins. + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `permissionPreset` | string | `"full-access"` | Permission preset: `"full-access"`, `"workspace"`, `"read-only"`, or `"ask"` | +| `permission` | object | - | Explicit permission rules that override the selected preset | + +Presets: + +| Preset | Behavior | +|--------|----------| +| `full-access` | Allow all permission checks without prompting | +| `workspace` | Allow `read`, `list`, `grep`, `edit`, and `bash`; ask for `external_directory` | +| `read-only` | Allow `read`, `list`, and `grep`; ask for `edit`, `bash`, and `external_directory` | +| `ask` | Restore prompt-on-unknown behavior | + +Example: + +```json +{ + "permissionPreset": "workspace", + "permission": { + "bash": { + "rm *": "deny" + }, + "edit": { + "secrets/*": "ask" + } + } +} +``` + +Flat rules apply to all patterns for that permission: + +```json +{ + "permissionPreset": "read-only", + "permission": { + "bash": "deny" + } +} +``` + +CLI overrides have the highest precedence: + +```bash +senpi --permission-preset ask +senpi --permission-preset workspace --permission "bash:rm *=deny" +``` + +Permission rules are a confirmation policy, not a sandbox. Senpi, extensions, package installs, and child processes still run with the host process permissions. + +## All Settings + +### Model & Thinking + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `defaultProvider` | string | - | Default provider (e.g., `"anthropic"`, `"openai"`) | +| `defaultModel` | string | - | Default model ID | +| `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`, `"max"` | +| `modelThinkingLevels` | object | - | Per-model reasoning effort memory (`"provider/id": "level"`) | +| `modelLastOnThinkingLevels` | object | - | Per-model last non-off reasoning level, used by `/reasoning on` to restore the previous effort | +| `modelServiceTiers` | object | - | Per-model service tier memory (`"provider/id": "auto" \| "priority"`) | +| `promptPreset` | string | `"auto"` | Force a system prompt preset: `"auto"`, `"kimi-k2-6"`, `"kimi-k2-7"`, `"kimi-k3"`, `"glm-5.2"`, `"glm-5.3"`, `"grok-4.5"`, `"grok-4.6"`, `"claude-fable-5"`, `"claude-opus-5"`, `"claude-opus-4-5"`, `"claude-opus-4-6"`, `"claude-opus-4-7"`, `"claude-opus-4-8"`, `"deepseek-v4-flash"`, `"deepseek-v4-flash-0731"`, `"deepseek-v4-pro"`, `"gpt-5"`, `"gpt-5.2"`, `"gpt-5.3-codex"`, `"gpt-5.4"`, `"gpt-5.5"`, or `"gpt-5.6"` | +| `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output | +| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses and compaction or branch-summary usage | +| `thinkingBudgets` | object | - | Custom token budgets per thinking level. Anthropic, Google, and Bedrock use these natively. OpenAI-compatible models use them when `compat.thinkingTokenBudgetField` (or `supportsThinkingTokenBudget`) is set. | + +#### promptPreset + +Use `promptPreset` when a provider's model ID does not auto-detect to the preset you want, or when you want to force one preset for a project. + +```json +{ + "promptPreset": "kimi-k2-6" +} +``` + +Project settings in `.senpi/settings.json` override global settings in `~/.senpi/agent/settings.json`. +When this value is anything other than `"auto"`, it overrides any model-level `promptPreset` configured in `models.json`. + +#### thinkingBudgets + +```json +{ + "thinkingBudgets": { + "minimal": 1024, + "low": 4096, + "medium": 10240, + "high": 32768 + } +} +``` + +### UI & Display + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `theme` | string | `"dark"` | Theme name (`"dark"`, `"light"`, or custom) | +| `externalEditor` | string | `$VISUAL`, then `$EDITOR`, then Notepad on Windows or `nano` elsewhere | Command for Ctrl+G external editor; takes precedence over environment variables | +| `quietStartup` | boolean | `false` | Hide startup header | +| `tips` | boolean | `true` | Show the rotating startup and working-status tip lines | +| `tipsHistory` | object | - | Internal record of which tips were shown last (managed automatically) | +| `defaultProjectTrust` | string | `"ask"` | Fallback project trust behavior: `"ask"`, `"always"`, or `"never"`. Global setting only | +| `collapseChangelog` | boolean | `false` | Show condensed changelog after updates | +| `enableInstallTelemetry` | boolean | `true` | Send an anonymous install/update version ping after first install or changelog-detected updates. This does not control update checks | +| `enableAnalytics` | boolean | `false` | Opt-in analytics data sharing. Currently only asked for during the experimental first-time setup (`PI_EXPERIMENTAL=1`) | +| `trackingId` | string | - | Analytics tracking identifier, generated when `enableAnalytics` is turned on | +| `doubleEscapeAction` | string | `"tree"` | Action for double-escape: `"tree"`, `"fork"`, or `"none"` | +| `treeFilterMode` | string | `"default"` | Default filter for `/tree`: `"default"`, `"no-tools"`, `"user-only"`, `"labeled-only"`, `"all"` | +| `editorPaddingX` | number | `0` | Horizontal padding for input editor (0-3) | +| `outputPad` | number | `1` | Horizontal padding for user messages, assistant messages, and thinking (0 or 1) | +| `autocompleteMaxVisible` | number | `5` | Max visible items in autocomplete dropdown (3-20) | +| `showHardwareCursor` | boolean | `false` | Show the terminal cursor while TUI positions it for IME support | +| `tuiMode` | string | `"regular"` | Interactive TUI mode: `"regular"` or experimental `"fullscreen"`. Changes from `/settings` apply immediately; `--tui-mode` overrides this setting at startup | +| `fullscreenExitOutput` | string | `"transcript"` | Fullscreen exit output: `"transcript"` prints the final transcript and resume hint, while `"resume-hint"` restores the previous screen and prints only the resume hint. Has no effect in regular TUI mode | +| `fullscreenScrollbar` | string | `"auto"` | Fullscreen transcript scrollbar: `"auto"` shows it temporarily while scrolling, `"always"` reserves the rightmost column and keeps it visible, and `"hidden"` hides it. Has no effect in regular TUI mode | + +For VS Code, include `--wait` so senpi resumes after the editor exits: + +```json +{ + "externalEditor": "code --wait" +} +``` + +### Telemetry and update checks + +`enableInstallTelemetry` only controls the anonymous install/update ping to `https://pi.dev/api/report-install`. Opting out of telemetry does not disable update checks; senpi can still fetch the latest published `@code-yeongyu/senpi` version from the npm registry (`registry.npmjs.org`). + +Set `PI_SKIP_VERSION_CHECK=1` to disable the senpi version update check. Use `--offline` or `PI_OFFLINE=1` to disable all startup network operations described here, including update checks, package update checks, and install/update telemetry. + +### Network + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `httpProxy` | string | - | HTTP proxy URL applied as `HTTP_PROXY` and `HTTPS_PROXY`. Global setting only. | + +```json +{ + "httpProxy": "http://127.0.0.1:7890" +} +``` + +### Warnings + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `warnings.anthropicExtraUsage` | boolean | `true` | Show a warning when Anthropic subscription auth may use paid extra usage | + +```json +{ + "warnings": { + "anthropicExtraUsage": false + } +} +``` + +### Compaction + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `compaction.enabled` | boolean | `true` | Enable auto-compaction | +| `compaction.reserveTokens` | number | `16384` | Tokens reserved for LLM response | +| `compaction.keepRecentTokens` | number | `20000` | Recent tokens to keep (not summarized) | + +```json +{ + "compaction": { + "enabled": true, + "reserveTokens": 16384, + "keepRecentTokens": 20000 + } +} +``` + +### Branch Summary + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `branchSummary.reserveTokens` | number | `16384` | Tokens reserved for branch summarization | +| `branchSummary.skipPrompt` | boolean | `false` | Skip "Summarize branch?" prompt on `/tree` navigation (defaults to no summary) | + +### Retry + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `retry.enabled` | boolean | `true` | Enable automatic agent-level retry on transient errors | +| `retry.maxRetries` | number | `3` | Maximum agent-level retry attempts | +| `retry.baseDelayMs` | number | `2000` | Base delay for agent-level exponential backoff (2s, 4s, 8s) | +| `retry.modelFallback` | boolean | `true` | Let eligible retry failures advance through configured per-model fallback chains | +| `retry.fallbackChains` | `Record` | `{}` | Ordered exact model-selector to fallback-selector chains | +| `retry.fallbackRevertPolicy` | `"cooldown-expiry"` \| `"never"` | `"cooldown-expiry"` | Automatic primary-model restoration policy | +| `retry.abortServerSideFallback` | boolean | `true` | Abort a turn when the provider substitutes a different model after a classifier decline | +| `retry.provider.timeoutMs` | number | `300000` | Provider/SDK request timeout and stream idle timeout in milliseconds | +| `retry.provider.streamStartTimeoutMs` | number | `90000` | Maximum wait for the first provider stream event; `0` disables | +| `retry.provider.streamRetryTimeoutMs` | number | `30000` | First-request liveness cap after a known provider stream/transport timeout; `0` disables the cap | +| `retry.provider.maxRetries` | number | `0` | Provider/SDK retry attempts | +| `retry.provider.maxRetryDelayMs` | number | `60000` | Max server-requested delay honored on the same model before the fallback chain engages (60s) | + +A server-requested retry delay at or below `retry.provider.maxRetryDelayMs` is honored on the same model. A longer delay means the model is unavailable rather than busy, so Senpi engages the configured fallback chain instead of waiting, suppressing the primary for the requested duration; the turn fails with an informative error only when no chain candidate can take over. + +After an exact provider stream/transport timeout, `retry.provider.streamRetryTimeoutMs` caps the retry's first +provider request and defers queued user input from that request. The cap applies only to stream guards that are +already enabled, never turns a disabled guard back on, and restores configured timeouts for later requests. + +Keep `retry.provider.maxRetries` at `0` unless provider-level retries are explicitly needed. Setting it above `0` can make SDK/provider retries handle out-of-usage-limit errors before senpi sees them, which may block the agent until the provider quota resets in some circumstances. + +```json +{ + "retry": { + "enabled": true, + "maxRetries": 3, + "baseDelayMs": 2000, + "provider": { + "timeoutMs": 3600000, + "streamStartTimeoutMs": 90000, + "streamRetryTimeoutMs": 30000, + "maxRetries": 0, + "maxRetryDelayMs": 60000 + } + } +} +``` + +#### Model fallback chains + +`retry.fallbackChains` maps a primary-model selector to an ordered list of fallback selectors. A selector is `provider/model` with an optional `:thinking-level` suffix, or a bare `model` id that applies to every provider serving that model family. Bare selectors expand against the models you actually have: providers holding an OAuth credential are preferred, then a fixed precedence order, and OpenRouter is never chosen by expansion. Senpi ships a bare default chain for `claude-fable-5`, so Fable 5 keeps a fallback chain whichever provider serves it; set that key to `[]` to opt out entirely, or set one `provider/claude-fable-5` key to override just that provider. For example, this switches Fable 5 to Kimi K3 at `max` thinking when an eligible failure occurs: + +```json +{ + "retry": { + "modelFallback": true, + "fallbackChains": { + "anthropic/claude-fable-5": ["ccapi/kimi-k3:max"] + }, + "fallbackRevertPolicy": "cooldown-expiry" + } +} +``` + +A chain is only for the exact primary model it names: selector lookup first considers an exact thinking-level selector, then the same `provider/model` without its thinking suffix. Wildcard selectors, role keys such as `default`, and other catch-all chains are not supported. + +A fallback entry with `:thinking-level` requests that level on the target model; a bare entry inherits the current thinking level. Either value is clamped to the target model's supported levels. When an unpinned fallback later returns to the primary, it restores the original thinking level unless you changed it while using the fallback. + +`/fallback` writes these settings to the global settings file. Project settings are still merged when read; because `fallbackChains` is a nested map, a project `retry.fallbackChains` replaces the global map rather than merging individual chain keys. + +#### Fallback behavior and diagnostics + +With `retry.enabled` and `retry.modelFallback` enabled, Senpi can switch from a transient or eligible hard provider failure to the next configured candidate. Transient failures (timeouts, overload, 429, 5xx, transport drops) first retry the same model on the existing exponential backoff; the chain engages only after `retry.maxRetries` attempts are spent, and each fallback candidate starts with a fresh retry budget. Hard failures (quota, auth, model-not-found) and classifier refusals still switch immediately. The switch continues the current turn without changing the existing conversation prefix, preserving prompt-cache inputs; fallback lifecycle events are never added to model context. Returning to a primary model happens only at a turn boundary, never while a response is streaming. Selector cooldowns are error-derived, and a provider retry-after hint always wins: quota and billing failures park a model for 30 minutes, rate limits for 30 seconds, overload for 45 seconds plus jitter, 5xx for 20 seconds, and timeout or connection/transport failures for 60 seconds; unmatched failures default to five minutes. A fully failing chain costs up to `1 + (chainLength + 1) * maxRetries` provider calls plus per-rung backoff before the turn fails; with `maxRetries: 0` every failure switches immediately, costing `1 + chainLength` calls. + +Billing-class failures — Anthropic's 400 *credit balance is too low*, OpenAI's 429 `insufficient_quota`, and other credit/quota exhaustion responses — never recover by retrying the same account, so a configured chain candidate receives a **pinned** fallback switch, exactly like a refusal-pinned fallback: it never auto-reverts and later turns keep running on the replacement model instead of returning to the exhausted account after the 30-minute billing cooldown. + +Anthropic streaming refusals are identified from typed `stopDetails`. A configured candidate receives an immediate **pinned** fallback switch with a user-visible fallback notice: Senpi does not retry the refusing model and a pinned fallback never auto-reverts. Set `retry.fallbackRevertPolicy` to `"cooldown-expiry"` (the default) to return an unpinned fallback to its primary after the primary's cooldown expires, or `"never"` to keep the fallback until you change models. + +#### Provider-substituted models + +Anthropic's server-side fallback betas can retry a classifier-declined request on a substitute model *inside the same response*, marking the handoff with a `fallback` content block; a gateway may enable this on your behalf. Honoring that response means paying for a model you did not select, and after the first handoff Anthropic routes later turns of the conversation straight to the substitute with no marker at all — reported only as a `fallback_message` entry in `usage.iterations`. + +With `retry.abortServerSideFallback` enabled (the default), Senpi treats either signal as a decline: it aborts the request as soon as the signal arrives, discards the substitute's partial output, and re-enters the turn as a classifier refusal so your own `retry.fallbackChains` chooses the replacement model. The transcript shows `Server fallback -> aborted`, naming `/fallback` when no chain is configured for the current model. + +Two caveats. Aborting minimizes but cannot eliminate cost: output already streamed before the abort is billed, and because per-attempt usage never arrives on an aborted stream, the turn carries a `billing_incomplete_after_client_abort` diagnostic instead of a precise cost. A served-model string that merely differs from the requested one never triggers an abort, because gateways and Bedrock-style endpoints legitimately rewrite model ids. + +Set it to `false` to keep the substituted response instead. If a gateway in front of Senpi injects the fallback itself, disabling the injection there avoids launching the substitute at all and is cheaper than aborting it client-side. + +Fallback decisions are process-local. A `senpi-task` or subagent child process reads its own settings and maintains its own in-memory suppression state; it does not affect its parent process. Disable fallback for one run without changing settings with `--no-model-fallback` or `SENPI_NO_FALLBACK=1`. + +For diagnostics, Senpi writes sanitized NDJSON records for candidate skips, cooldowns, switches, reverts, manual clears, and validation warnings to `/logs/fallback.log`. The file is mode `0600` and rotates at 5 MB (`fallback.log.1`). + +### Message Delivery + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `steeringMode` | string | `"one-at-a-time"` | How steering messages are sent: `"all"` or `"one-at-a-time"` | +| `followUpMode` | string | `"one-at-a-time"` | How follow-up messages are sent: `"all"` or `"one-at-a-time"` | +| `transport` | string | `"auto"` | Preferred transport for providers that support multiple transports: `"sse"`, `"websocket"`, `"websocket-cached"`, or `"auto"` | +| `httpIdleTimeoutMs` | number | `300000` | HTTP header/body idle timeout in milliseconds, also used by providers with explicit stream idle timeouts. Set to `0` to disable. | +| `websocketConnectTimeoutMs` | number | `15000` | WebSocket connect/open handshake timeout in milliseconds for providers that support WebSocket transports. Set to `0` to disable. | + +### OpenAI + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `openai.serviceTier` | string | - | Injects OpenAI Responses `service_tier`: `"auto"`, `"flex"`, or `"priority"` | + +```json +{ + "openai": { + "serviceTier": "priority" + } +} +``` + +When unset, senpi leaves provider payloads unchanged. This setting currently applies only to the built-in OpenAI Responses provider path. + +### Providers + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `claudeSdkOauthProvider.enabled` | boolean | `false` | Enable the ambient (host-CLI-derived) lane of `claude-sdk-oauth`. Env override: `SENPI_CLAUDE_SDK_OAUTH_ENABLED`. Explicit senpi-side logins (stored OAuth accounts in `auth.json`, `CLAUDE_CODE_OAUTH_TOKEN` / `CLAUDE_CODE_OAUTH_TOKEN_` env accounts) keep the provider available with this unset | +| `cursorCliOauthProvider.enabled` | boolean | `false` | Enable the `cursor-cli-oauth` fallback lane and automatic native credential bootstrap. Env override: `SENPI_CURSOR_CLI_OAUTH_ENABLED` | + +Both ambient-auth providers are explicit opt-in: a vendor CLI being logged in on the machine is not consent to spend that subscription. Before these gates existed, a logged-in Claude Code or `cursor-agent` CLI made the lane available with no senpi-side action, so subscription usage could flow through a provider you never configured. Env overrides follow the usual precedence (`env > project settings > global settings > default`). See [providers.md](providers.md) for the full lane documentation. + +```json +{ + "claudeSdkOauthProvider": { + "enabled": true + }, + "cursorCliOauthProvider": { + "enabled": true + } +} +``` + +### Terminal & Images + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `terminal.showImages` | boolean | `true` | Show images in terminal (if supported) | +| `terminal.imageWidthCells` | number | `60` | Preferred inline image width in terminal cells | +| `terminal.clearOnShrink` | boolean | `false` | Clear empty rows when content shrinks (can cause flicker) | +| `images.autoResize` | boolean | `true` | Resize images to 2000x2000 max. Applies to `@file` attachments, `read`, and images returned by tools | +| `images.blockImages` | boolean | `false` | Block all images from being sent to LLM | + +### Prompt Cache + +Sizes how long foreground tools may block on the active model's prompt-cache lifetime, so a long +`bash` call never straddles cache expiry and forces a full re-read. When the model's cache TTL is +unknown (e.g. Google models) or caching is off, no budget applies and timeout behavior is unchanged. + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `promptCache.cacheAwareTimeouts` | boolean | `true` | Cap foreground tool waits at the model's prompt-cache TTL minus the safety buffer; `false` restores the fixed legacy ceilings | +| `promptCache.safetyBufferSeconds` | number | `30` | Headroom subtracted from the cache TTL (a 5m TTL yields a 270s ceiling). If it consumes the whole TTL, no budget applies | + +A foreground `bash` command still running at the budget is handed to a live background session +instead of being killed; its explicit `timeout` remains the kill deadline. See +`terminal.timeoutAction` to switch that hand-off back to a kill. + +### Shell + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `shellPath` | string | - | Custom shell path (e.g., for Cygwin on Windows); supports a leading `~` for the home directory | +| `shellCommandPrefix` | string | - | Prefix for every bash command (e.g., `"shopt -s expand_aliases"`) | +| `npmCommand` | string[] | - | Command argv used for npm package lookup/install operations (e.g., `["mise", "exec", "node@20", "--", "npm"]`) | + +Windows paths in JSON must use forward slashes or escaped backslashes: + +```json +{ + "shellPath": "C:/Program Files/Git/bin/bash.exe" +} +``` + +```json +{ + "shellPath": "C:\\Program Files\\Git\\bin\\bash.exe" +} +``` + +```json +{ + "npmCommand": ["mise", "exec", "node@20", "--", "npm"] +} +``` + +`npmCommand` is used for all npm package-manager operations, including installs, uninstalls, and dependency installs inside git packages. User-scoped npm packages install under `~/.senpi/agent/npm/`; project-scoped npm packages install under `.senpi/npm/`. Use argv-style entries exactly as the process should be launched. When `npmCommand` is configured, git package dependency installs use plain `install` to avoid npm-specific flags in wrappers or alternate package managers. + +### Tools + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `defaultTools` | string[] | - | Built-in tools enabled initially. When omitted, Pi uses its standard defaults | + +`defaultTools` selects the built-in tools enabled at startup. Extension and SDK custom tools remain enabled: + +```json +{ + "defaultTools": ["bash", "edit", "write"] +} +``` + +An empty array starts with no built-in tools while preserving extension and SDK custom tools. `--tools` replaces this behavior with a strict allowlist for all tools, `--no-tools` disables all tools, and `--no-builtin-tools` disables the built-in defaults. `--exclude-tools` filters the resulting list. A project `defaultTools` array replaces the global array. + +### Sessions + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `sessionDir` | string | - | Directory where session files are stored. Accepts absolute or relative paths, plus `~`. | + +```json +{ "sessionDir": ".senpi/sessions" } +``` + +When multiple sources specify a session directory, precedence is `--session-dir`, `SENPI_CODING_AGENT_SESSION_DIR`, then `sessionDir` in settings.json. + +### Model Cycling + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `favoriteModels` | string[] | - | Favorite model patterns for Ctrl+P cycling (same format as `--models` CLI flag) | +| `enabledModels` | string[] | - | Legacy global model-catalog narrowing patterns (same format as `--models` CLI flag) | + +```json +{ + "enabledModels": ["anthropic/*", "openai/*"], + "favoriteModels": ["anthropic/claude-*", "openai/gpt-5.4"] +} +``` + +`enabledModels` changes which models appear in the catalog, startup selection, and `/model` narrowing. `favoriteModels` is separate and only controls Ctrl+P cycling. + +#### Per-model memory + +`modelThinkingLevels`, `modelLastOnThinkingLevels`, and `modelServiceTiers` are maps from `"provider/id"` to a level or tier value. They're managed automatically: switching models, using `/reasoning`, `/efforts`, or `/fast` writes the appropriate key. You rarely need to edit them by hand, but the shape looks like this: + +```json +{ + "modelThinkingLevels": { + "openai-codex/gpt-5.6-sol": "xhigh", + "anthropic/claude-fable-5": "high" + }, + "modelServiceTiers": { + "openai-codex/gpt-5.6-sol": "priority" + } +} +``` + +A `-fast` catalog variant (like `gpt-5.6-sol-fast`) and its base model share one entry, so you can't give them conflicting tiers. + +#### Favorite model decorators + +Favorite model patterns accept optional decorator suffixes for reasoning level and service tier: + +``` +provider/model-id # bare pattern +provider/model-id:high # pin reasoning to high +provider/model-id:priority # pin service tier to priority +provider/model-id:priority:high # pin both tier and level +claude-*:xhigh # glob with level pin +``` + +Decorators survive favorite toggling. A `:level` pin takes precedence over the per-model memory for reasoning, and a `:priority` pin takes precedence for the service tier. Under a pin, `/fast off` notifies that fast mode is fixed by the active model selection. + +#### Thinking level precedence + +When a model becomes active, its reasoning level is resolved in this order: + +1. An explicit or ephemeral session-scoped level (e.g. turn-scope `set_thinking_level`) +2. A favorite pattern `:level` pin +3. The per-model `modelThinkingLevels` memory +4. `defaultThinkingLevel` +5. `"medium"` (the hardcoded fallback) + +The resolved level is always clamped to what the model actually supports. + +#### Service tier precedence + +The service tier on outgoing requests is resolved as: + +1. A scoped/favorite `:priority` pin +2. The model catalog's `compat.serviceTier` +3. `openai.serviceTier` (the global OpenAI setting) + +The per-model `modelServiceTiers` memory is not part of that resolution: it applies to OpenAI Codex +models only, through fast mode. It acts as the session-start default for `/fast` (a remembered +`"priority"` starts the session fast) and as an explicit `"auto"` opt-out of a catalog-inherited +priority tier, which keeps `service_tier` off the wire. Under a `:priority` pin the memory has no +effect, because the pin outranks it. + +### Markdown + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `markdown.codeBlockIndent` | string | `" "` | Indentation for code blocks | +| `markdown.mermaid` | string | `"streaming"` | Mermaid rendering mode: `"off"`, `"final"`, or `"streaming"` | + +### Resources + +These settings define where to load extensions, skills, prompts, and themes from. + +Paths in `~/.senpi/agent/settings.json` resolve relative to `~/.senpi/agent`. Paths in `.senpi/settings.json` resolve relative to `.senpi`. Absolute paths and `~` are supported. + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `packages` | array | `[]` | npm/git packages to load resources from | +| `enabledBuiltinExtensions` | string[] | all builtins | Optional allowlist of builtin extension ids to load | +| `disabledBuiltinExtensions` | string[] | `[]` | Builtin extension ids to skip; overrides `enabledBuiltinExtensions` | +| `extensions` | string[] | `[]` | Local extension file paths or directories | +| `skills` | string[] | `[]` | Local skill file paths or directories | +| `prompts` | string[] | `[]` | Local prompt template paths or directories | +| `themes` | string[] | `[]` | Local theme file paths or directories | +| `enableSkillCommands` | boolean | `true` | Register skills as `/skill:name` commands | + +Arrays support glob patterns and exclusions. Use `!pattern` to exclude. Use `+path` to force-include an exact path and `-path` to force-exclude an exact path. + +#### packages + +String form loads all resources from a package: + +```json +{ + "packages": ["pi-skills", "@org/my-extension"] +} +``` + +Object form filters which resources to load: + +```json +{ + "packages": [ + { + "source": "pi-skills", + "skills": ["brave-search", "transcribe"], + "extensions": [] + } + ] +} +``` + +See [packages.md](packages.md) for package management details. + +## Example + +```json +{ + "defaultProvider": "anthropic", + "defaultModel": "claude-sonnet-4-20250514", + "defaultThinkingLevel": "medium", + "theme": "dark", + "compaction": { + "enabled": true, + "reserveTokens": 16384, + "keepRecentTokens": 20000 + }, + "retry": { + "enabled": true, + "maxRetries": 3 + }, + "enabledModels": ["claude-*", "gpt-4o"], + "favoriteModels": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o"], + "warnings": { + "anthropicExtraUsage": true + }, + "packages": ["pi-skills"] +} +``` + +## Project Overrides + +Project settings (`.senpi/settings.json`) override global settings. Nested objects are merged: + +```json +// ~/.senpi/agent/settings.json (global) +{ + "theme": "dark", + "compaction": { "enabled": true, "reserveTokens": 16384 } +} + +// .senpi/settings.json (project) +{ + "compaction": { "reserveTokens": 8192 } +} + +// Result +{ + "theme": "dark", + "compaction": { "enabled": true, "reserveTokens": 8192 } +} +``` diff --git a/package.json b/package.json index dfd710a..3eaa0f3 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "@libar-dev/agent-harness-kit", - "version": "0.2.0", + "version": "0.3.0", "packageManager": "pnpm@10.4.1", "type": "module", - "description": "Agent harness toolkit — Claude Code hooks today, harness-agnostic future. Typed session export, tail tooling, and 30 hook event handlers with Zod validation.", + "description": "Agent harness toolkit for Claude Code hooks plus attach-only Grok Build and OmO-native (senpi) observe adapters. Typed session processing, standalone hook forwarders, and Zod-validated hook I/O.", "sideEffects": false, "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -28,6 +28,14 @@ "import": "./dist/grok/processing/index.js", "types": "./dist/grok/processing/index.d.ts" }, + "./senpi": { + "import": "./dist/senpi/index.js", + "types": "./dist/senpi/index.d.ts" + }, + "./senpi/processing": { + "import": "./dist/senpi/processing/index.js", + "types": "./dist/senpi/processing/index.d.ts" + }, "./validation": { "import": "./dist/validation/index.js", "types": "./dist/validation/index.d.ts" @@ -75,7 +83,7 @@ } }, "scripts": { - "build": "tsc --project tsconfig.build.json && esbuild src/forwarder/hook-forwarder.ts --bundle --platform=node --format=esm --target=node18 --outfile=dist/standalone/hook-forwarder.mjs && node scripts/chmod-bins.mjs", + "build": "tsc --project tsconfig.build.json && esbuild src/forwarder/hook-forwarder.ts --bundle --platform=node --format=esm --target=node18 --outfile=dist/standalone/hook-forwarder.mjs && esbuild src/forwarder/hook-forwarder-senpi.ts --bundle --platform=node --format=esm --target=node22 --outfile=dist/standalone/hook-forwarder-senpi.mjs && node scripts/chmod-bins.mjs", "build:emergency": "tsc --project tsconfig.emergency.json && node scripts/chmod-bins.mjs", "dev": "tsx watch src/index.ts", "docs:sync-upstream": "node scripts/sync-upstream-docs.mjs", @@ -87,6 +95,10 @@ "fix": "pnpm run lint:fix && pnpm run type-check", "fix:all": "prettier --write 'src/**/*.{ts,tsx,js,jsx,json,md}' || echo 'Prettier encountered syntax errors but continuing...' && pnpm run lint:fix || echo 'ESLint fixes completed with potential errors'", "validate": "pnpm run check", + "senpi:smoke": "tsx scripts/senpi-live-smoke.mts", + "senpi:compatibility": "tsx scripts/senpi-compatibility-probe.mts", + "consumer:matrix": "node scripts/clean-consumer.mjs --matrix", + "consumer:happy": "node scripts/clean-consumer.mjs --mode happy", "validate:dev": "pnpm run check", "validate:ci": "pnpm run check:ci", "test": "vitest", @@ -110,7 +122,7 @@ "clean": "rm -rf dist", "prepare": "pnpm run build", "reset": "pnpm run clean && pnpm run fix && pnpm run build", - "prepack": "pnpm run clean && pnpm run test:run && pnpm run check && pnpm run build" + "prepack": "pnpm run clean && pnpm run check && pnpm run build && pnpm run test:run" }, "dependencies": { "@noble/hashes": "^2.3.0", @@ -142,6 +154,8 @@ ], "keywords": [ "claude-code", + "grok", + "senpi", "hooks", "typescript", "ai", @@ -159,7 +173,8 @@ }, "homepage": "https://github.com/libar-dev/agent-harness-kit#readme", "publishConfig": { - "access": "public" + "access": "public", + "provenance": true }, "pnpm": { "onlyBuiltDependencies": [ diff --git a/plans/foamy-questing-catmull.md b/plans/foamy-questing-catmull.md new file mode 100644 index 0000000..84c401c --- /dev/null +++ b/plans/foamy-questing-catmull.md @@ -0,0 +1,85 @@ +# Context + +The checked-in Claude Code documentation mirrors were last refreshed in June 2026, while `docs/internal/api-update-checklist.md` is a closed February 2026 migration log that still describes a 14-event API. The library has since grown to 30 hook events and additional handler/output surfaces, so the mirrors, exported TypeScript contracts, Zod schemas, bundled handlers, tests, and project-authored documentation need a fresh authoritative audit. + +The intended result is parity with the current Claude Code APIs that this hooks library actually exposes: hook event I/O, hook-specific settings/handlers, hook-provided environment variables, modeled tool inputs, output builders/reference handlers, and existing transcript-processing contracts. Full unrelated Claude Code settings, general CLI flag parsing, and headless stream protocols remain audit-only unless they affect an existing exported API. + +## Implementation Plan + +1. **Run the task through a dynamic Workflow controller.** + - Use Workflow phases for baseline/sync, parallel gap analysis, sequential implementation, verification, and final adversarial review. + - Dynamically add focused audit/repair agents only for categories changed by the refreshed docs. + - Keep all workflow findings in the harness; do not create or commit scratch reports, plans, or agent-context files. + - Do not commit or push unless separately requested. + +2. **Guard the baseline and refresh all upstream mirrors.** + - Confirm the working tree has no unexpected changes before mutation. + - Run `pnpm run docs:sync-upstream`, using `scripts/sync-upstream-docs.mjs` unchanged unless it fails to retrieve the official Markdown endpoints. + - Validate the complete five-file set before analysis: + - `docs/upstream/hooks-reference.md` + - `docs/upstream/hooks-guide.md` + - `docs/upstream/settings.md` + - `docs/upstream/cli-reference.md` + - `docs/upstream/headless.md` + - Treat a partial fetch as invalid; resolve and rerun the full sync before continuing. + +3. **Build a post-sync delta matrix and enforce scope.** + - Compare the refreshed docs with their previous git versions and with current exported contracts. + - Audit in parallel by category: hook I/O, hook settings/handler restrictions, modeled tool inputs, hook environment variables, output builders/reference handlers, transcript/headless overlap, and project-doc drift. + - Classify each finding as implementation, documentation-only, audit-only, or out of scope. + - Implement only claims substantiated by the refreshed official docs. Initial queries include common fields such as `prompt_id`; Notification literals/output semantics; Stop/SubagentStop background task, cron, and context fields; current Agent/ExitPlanMode tool inputs; `disableAllHooks`; event-specific handler support; hook-provided variables such as `CLAUDE_PLUGIN_DATA`, `CLAUDE_EFFORT`, and `CLAUDE_CODE_BRIDGE_SESSION_ID`; and timeout/matcher behavior. + - Do not model the entire settings schema, every Claude CLI flag, or non-persisted headless stream events without an existing library surface. + +4. **Update public manual contracts first.** + - Modify `src/types/index.ts` for every confirmed hook/tool/settings/environment delta. + - Preserve optionality for version-gated fields so older valid Claude Code payloads remain accepted. + - Reuse shared exported interfaces for repeated structures rather than duplicating inline object types. + - Preserve broad public `HookHandler` and `MatcherGroup` primitives while adding event-aware helper types if the current docs restrict handler kinds per event. + - Preserve existing exported symbols when correcting their shape, and add API-contract JSDoc to every new or changed public declaration. + +5. **Mirror the contract exactly in runtime validation.** + - Update `src/validation/schemas.ts` in lockstep with the manual interfaces, including event collections, tool schemas, hook settings, and output schemas. + - Retain default Zod strip behavior for hook/tool/settings contracts; do not add broad `.passthrough()` because it breaks the manual/Zod structural contract. Keep loose schemas only where transcript forward preservation is intentional. + - If event-specific handler restrictions are confirmed, keep standalone `hookHandlerSchema`/`matcherGroupSchema` broad but make `hooksConfigSchema` validate handlers against the selected event. + - Review `src/validation/validators.ts`, `src/validation/index.ts`, and `src/index.ts` for changed validators, inferred types, guards, and exports; rely on existing schema dispatch where no dedicated code is needed. + +6. **Restore builder and reference-handler parity.** + - Update `src/utils/output-builder.ts` only for confirmed output contracts, using distinct helpers for semantically different outputs rather than misleading casts or reused event names. + - Audit `src/lifecycle/stop-handler.ts`, `src/lifecycle/subagent-stop.ts`, `src/lifecycle/notification-handler.ts`, and any other directly affected handler. + - Keep compatibility aliases where practical, document their actual behavior, and avoid unrelated lifecycle refactors. + +7. **Strengthen regression coverage.** + - Update `tests/test-utils.ts`, `tests/validation.test.ts`, `tests/output-builder.test.ts`, lifecycle/hook tests, and package-export tests for all confirmed deltas and compatibility cases. + - Add focused bidirectional `expectTypeOf` checks between manual interfaces and `z.infer` contracts for changed public shapes. + - Enhance `tests/docs-round-trip.test.ts` so it: + - validates hook input examples, + - validates hook-specific output examples with section context, + - validates hook-related settings snippets from `settings.md`, + - compares the official lifecycle event inventory with `hookEventNameSchema.options`, and + - retains explicit, self-invalidating skip rules only for intentionally abbreviated or non-JSON examples. + - Do not validate unrelated full-settings examples or add stream-only records to persisted transcript types without evidence. + +8. **Refresh project documentation and the maintainer checklist.** + - Audit and update representative public docs: `docs/guides/configuring-settings-json.md`, `docs/reference/{hook-events,types,validators,output-builder,environment-variables}.md`, `README.md`, `CLAUDE.md`, and `CHANGELOG.md` when the public API changes. + - Fix known stale claims such as the settings guide’s 28-event inventory and ensure all event/tool counts are consistent with the canonical schemas. + - Replace `docs/internal/api-update-checklist.md` with a current repeatable audit document containing: + - audit date and sync command/source URLs, + - explicit in-scope and excluded surfaces, + - canonical event/tool/handler inventory, + - confirmed delta/status table, + - compatibility decisions, + - verification results, and + - the next-audit procedure: sync → diff → classify → implement → verify. + - Rely on git history for the obsolete February phase narrative rather than retaining it as the active checklist. + +## Verification + +1. Run targeted tests for docs round-trip, validation/type parity, output builders, affected lifecycle handlers, processing contracts if touched, and package exports. +2. Run the full gates: + - `pnpm run test:run` + - `pnpm run type-check` + - `pnpm run lint` + - `pnpm run build` +3. Inspect generated declarations under `dist/types` and `dist/validation` after the build; do not hand-edit `dist`. +4. Run `git diff --check` and search project-authored docs for stale event/tool counts or removed contract claims. +5. Use a final workflow review to verify that every implemented field is supported by the refreshed official docs, every changed manual type matches its Zod schema, and no unrelated full-settings/CLI/headless scope or ephemeral workflow artifacts entered the diff. diff --git a/plans/grok-adapter/brief.md b/plans/grok-adapter/brief.md new file mode 100644 index 0000000..48305b8 --- /dev/null +++ b/plans/grok-adapter/brief.md @@ -0,0 +1,245 @@ +# Grok Build adapter — planning brief + +**Branch:** `feat/grok-adapter` (from `origin/main` @ `6a08ff3`) +**Audience:** a planning session with no prior conversation context (read-only; do not clone or write) +**Status:** investigation complete; no implementation yet +**Do not merge this file.** Planning notes stay out of the public package. Delete before merge, or keep only if we promote a subset into `docs/`. + +### Local Grok Build checkout (read this first) + +A shallow clone is already on disk. **Do not clone again. Open these paths.** + +| | | +|---|---| +| Checkout | `/tmp/grok-build` | +| Remote | `https://github.com/xai-org/grok-build` | +| Git `HEAD` | `e5fd4816d43260c15ba785f103990c1ed6cea230` (shallow, 2026-08-13, “Synced from monorepo”) | +| `SOURCE_REV` | `ea094a8c369475f97c85540d01730baec0dce5d6` (upstream monorepo SHA) | +| Clone shape | `git clone --depth 1` — no extra history | + +If `/tmp/grok-build` is missing (reboot, `/tmp` wipe), recreate with: + +```bash +git clone --depth 1 https://github.com/xai-org/grok-build.git /tmp/grok-build +``` + +Then re-record `HEAD` and `SOURCE_REV` before planning. + +--- + +## 1. Decision to confirm first + +This kit attaches to a harness the user already started. It does not spawn the agent. + +| Product | Who starts the agent | What we do | +|---|---|---| +| This kit (Claude today) | User’s CLI / IDE / Desktop | Hooks + transcript tail | +| [t3code](https://github.com/pingdotgg/t3code) | t3code, via `@anthropic-ai/claude-agent-sdk` `query()` | Owns the session, streams UI | +| Claude Agent SDK | Our process | In-process callbacks on *our* `query()` | + +The Agent SDK cannot join a live interactive Claude/Grok TUI. Resume starts a new process with old history. Official live-join for Claude is first-party Remote Control only. + +Grok is the second **attach** harness, not a t3code clone and not “use the Agent SDK.” Reconstructing Claude hook I/O was the right call. For Grok we should **not** reconstruct from the user guide. The implementation is public. + +--- + +## 2. Why Grok is viable as harness two + +1. Same attach shape as Claude: command/HTTP hooks, JSON stdin/stdout, settings files, fail-open except explicit deny. +2. Session files on disk under `~/.grok/sessions///`, append-only JSONL. +3. Source of truth exists: [xai-org/grok-build](https://github.com/xai-org/grok-build) (Apache-2.0, periodic monorepo sync). +4. Grok already loads `~/.claude/settings.json` hooks when `[compat.claude] hooks = true` (default). Claude *scripts* still will not work unchanged (envelope and output differ). + +--- + +## 3. Source of truth (use these, in this order) + +### 3.1 Implementation — authoritative + +Read the checkout at `/tmp/grok-build`. GitHub is only the origin; do not fetch in planning mode. + +The public tree can lag the installed `grok` binary. This snapshot is `HEAD` `e5fd481` / `SOURCE_REV` `ea094a8`. + +| Need | Absolute path | +|---|---| +| Hook stdin envelope, events, payloads | `/tmp/grok-build/crates/codegen/xai-grok-hooks/src/event.rs` | +| Hook crate overview (stale “four events” comment) | `/tmp/grok-build/crates/codegen/xai-grok-hooks/src/lib.rs` | +| Config / discovery / matcher / dispatch | `/tmp/grok-build/crates/codegen/xai-grok-hooks/src/config.rs`, `discovery.rs`, `dispatcher.rs`, `matcher.rs` | +| Allow/deny and Stop outcomes | `/tmp/grok-build/crates/codegen/xai-grok-hooks/src/result.rs` | +| HTTP / command runner | `/tmp/grok-build/crates/codegen/xai-grok-hooks/src/runner/` | +| Handler types (`command`, `http` only) | `/tmp/grok-build/crates/codegen/xai-hooks-plugins-types/src/lib.rs` | +| `events.jsonl` line schema | `/tmp/grok-build/crates/codegen/xai-grok-session-events/src/types.rs` (`Event`, `schema_version = "1.0"`) | +| Session export from `updates.jsonl` | `/tmp/grok-build/crates/codegen/xai-grok-shell/src/session/export.rs` | +| Persist/replay wire tags | `/tmp/grok-build/crates/codegen/xai-grok-shell/src/session/wire_tags.rs` | +| Replay `updates.jsonl` | `/tmp/grok-build/crates/codegen/xai-grok-shell/src/session/helpers/replay.rs` | +| On-disk envelope helper | `/tmp/grok-build/crates/codegen/xai-grok-pager-pty-harness/src/leader.rs` | +| ACP JSON-RPC types | `/tmp/grok-build/crates/codegen/xai-acp-lib/src/` | + +`event.rs` is the real event list. Trust serde structs, not crate overview comments. + +Do not: depend on the Rust crate from npm, vendor the whole tree, or send PRs (external contributions are rejected). Port types into Zod. Attribute if we copy substantial code (Apache-2.0). + +### 3.2 User guide — behavior notes only + +Prefer the checkout copy so planning does not depend on `~/.grok` being present: + +| File | Use for | +|---|---| +| `/tmp/grok-build/crates/codegen/xai-grok-pager/docs/user-guide/10-hooks.md` | Discovery paths, trust, fail-open, Claude/Cursor aliases, Stop 8-continuation cap | +| `/tmp/grok-build/crates/codegen/xai-grok-pager/docs/user-guide/17-sessions.md` | Directory layout, `grok sessions list/search`, resume flags | +| `/tmp/grok-build/crates/codegen/xai-grok-pager/docs/user-guide/14-headless-mode.md` | `streaming-json` / `streaming-messages-json` if we *spawn* `grok -p` | +| `/tmp/grok-build/crates/codegen/xai-grok-pager/docs/user-guide/15-agent-mode.md` | ACP if we *drive* a session we own | +| `/tmp/grok-build/crates/codegen/xai-grok-pager/docs/user-guide/05-configuration.md` | `[compat.claude] hooks` | + +Installed mirror (same content if this machine’s Grok is current): `~/.grok/docs/user-guide/`. + +The user guide is incomplete as a contract: one PreToolUse example, “stdout ignored” for passive events, HTTP deny body unspecified, `events.jsonl` omitted, `updates.jsonl` wrapper omitted. Confirm every guide claim against the crates. + +### 3.3 Live fixtures — lock the wire + +Sample this machine: `~/.grok/sessions/%2FUsers%2Fdarkomijic%2Fdev-libar%2Flibar-agent-harness-kit//` + +Observed beyond the guide: + +- `updates.jsonl` is `{ timestamp, method: "session/update", params: { sessionId, update } }` +- `sessionUpdate` includes undocumented `user_message_chunk` plus large `_meta` +- `events.jsonl` exists (`turn_started`, `loop_started`, `phase_changed`, `first_token`, …) +- Extra files: `hunk_records.jsonl`, `prompt_context.json`, `announcement_state.json`, `resources_state.json` + +Keep a small fixture set in tests. Re-dump when `SOURCE_REV` or `grok --version` moves. + +--- + +## 4. Hook contract (Grok vs Claude) + +### 4.1 Events Grok actually fires + +From `HookEventName` in `event.rs`: + +SessionStart, UserPromptSubmit, PreToolUse, PostToolUse, PostToolUseFailure, PermissionDenied, Stop, StopFailure, Notification, SubagentStart, SubagentStop (alias SubagentEnd), PreCompact, PostCompact, SessionEnd. + +**Not in Grok (do not port 1:1):** Setup, UserPromptExpansion, PermissionRequest, PostToolBatch, MessageDisplay, TaskCreated, TaskCompleted, TeammateIdle, InstructionsLoaded, ConfigChange, CwdChanged, FileChanged, WorktreeCreate, WorktreeRemove, Elicitation, ElicitationResult, DirectoryAdded. + +### 4.2 Wire vs Claude + +| | Claude (this kit today) | Grok | +|---|---|---| +| Envelope keys | snake_case (`hook_event_name`) | camelCase (`hookEventName`); serde struct fields are snake_case | +| Event value on stdin | `PreToolUse` | `pre_tool_use` (`#[serde(rename_all = "snake_case")]`) | +| Config / settings keys | PascalCase | PascalCase; aliases accept camelCase and Cursor names | +| Tool I/O | `tool_input`, `tool_response` | `toolInput`, `toolResult` | +| Tool names | `Bash`, `Write`, `Edit` | `run_terminal_command`, `search_replace`, …; matchers alias Claude names | +| PreToolUse output | `hookSpecificOutput.permissionDecision`: allow/deny/ask/defer + `updatedInput` | `{ "decision": "allow" \| "deny", "reason" }` only | +| Stop output | block / additionalContext / continue | Same vocabulary; extra session-end Stop with `reason != "end_turn"` (observe-only) | +| Passive events | Many outputs honored | GateKind::Observe — stdout recorded, decisions ignored | +| Handlers | command, http, mcp_tool, prompt, agent | command, http | +| Fail policy | exit 2 blocks | Fail-open unless explicit deny / Stop JSON / exit 2 with no JSON | +| Timeouts | 600s typical command | 5s default; 600s Stop/SubagentStop | +| `transcript_path` | required on base input | optional `transcriptPath` on envelope (in source; omitted from user guide) | + +`HookDecision` in `result.rs`: `Allow` | `Deny { reason, hook_name }`. +`StopHookOutcome`: `block_reason`, `additional_context`, `force_stop` (`continue: false` + `stopReason`). + +### 4.3 What this means for our API + +Do **not** reuse `PreToolUseInput` / `HookOutputBuilder.permission('ask' | 'defer')` on the Grok path. + +Build a Grok-native layer: + +- `GrokHookEventName`, `GrokHookEventEnvelope`, per-event payloads from `HookPayload` +- Zod from those structs +- `GrokHookOutputBuilder` with allow/deny + Stop block/context/force-stop only +- Shared `executeHook`-style runner that can parse either envelope **or** a Grok-only entry +- Map both harnesses into a small shared “session block” model for export/tail UI — do not unify hook types + +Claude `settings.json` command hooks may *load* in Grok. They will mis-read fields and emit the wrong deny JSON unless we ship a thin translator or a second script. + +--- + +## 5. Session parse / stream + +### 5.1 On-disk (attach) + +``` +~/.grok/sessions/// + summary.json # index + updates.jsonl # conversation (ACP session/update) — resume source + chat_history.jsonl # model-facing messages + events.jsonl # turn phases (undocumented in the guide; typed in session-events) + plan.json, rewind_points.jsonl, signals.json, subagents/ +``` + +Cwd encoding is URL-encoding (`%2FUsers%2F...`), not Claude’s dash encoding. `GROK_HOME` overrides `~/.grok`. + +Suggested ingest: + +- **Conversation / export:** `updates.jsonl` (need the JSON-RPC wrapper + `sessionUpdate` union) +- **Live “what is the agent doing”:** `events.jsonl` (`Event` in `types.rs`) +- **Optional raw model view:** `chat_history.jsonl` (no field-level guide; dump fixtures) + +There is no official tail CLI. We watch files the same way as `claude-session-tail`. + +### 5.2 Spawned stream (only if we own the process) + +`grok -p --output-format streaming-json` is well specified (type-tagged ACP-derived lines). `streaming-messages-json` is Claude-like with listed fidelity holes. Approvals are not in that stream; they go through `grok agent` (ACP). + +Out of scope for v1 attach unless the planning session chooses a cockpit-owned Grok process. + +--- + +## 6. Suggested scope for v1 + +**In** + +1. Grok hook types + Zod + output builder + `executeHook` variant, generated/ported from `event.rs` / `result.rs`. +2. Settings/config validation for Grok JSON + TOML hook objects (`command` / `http`). +3. Session discovery (`~/.grok/sessions`, `GROK_HOME`) + parse/tail of `updates.jsonl` and `events.jsonl`. +4. Upstream pin: script or `docs/upstream/grok/` copies of `event.rs` + `types.rs` at a recorded SHA; test fails on drift. +5. Docs: Grok vs Claude incompatibilities; do not claim 30-event parity. + +**Out of v1** + +- Agent SDK / ACP as the hook transport +- Reusing Claude `HookOutputBuilder` methods that Grok ignores +- mcp_tool / prompt / agent handlers +- Driving `grok` like t3code +- 17 missing Claude events + +**Later** + +- HTTP hook response contract for a cockpit forwarder (confirm against `dispatcher.rs`) +- Optional Claude→Grok stdin/stdout translator so one script works in both +- Headless `streaming-json` consumer if we spawn Grok + +--- + +## 7. Planning-session questions + +1. Confirm attach-only for v1 (hooks + JSONL), not a t3code-style driver. +2. One package with `grok/` exports vs a second package? +3. Shared session-block model now, or Grok processing isolated until a cockpit needs one UI? +4. Ship a Claude-compat translator, or document “write a Grok script”? +5. Pin strategy: git submodule (dev only), vendored rust snippets, or fetch script in CI? +6. Which `grok` / `SOURCE_REV` is the compatibility floor? +7. Cockpit forwarder for Grok in v1, or hooks library + tail only? + +--- + +## 8. Risks + +- Public repo lags the binary → pin SHA *and* record `grok --version`. +- Stale comments in `xai-grok-hooks` `lib.rs`. +- `updates.jsonl` `_meta` is large and unstable; parse the ACP `sessionUpdate` discriminant, ignore unknown `_meta`. +- Fail-open: a crashing guard does not block. Policy hooks must return explicit deny. +- Existing Claude reference hooks will not work on Grok without a translator. + +--- + +## 9. Investigation inputs (this brief is derived from) + +- Claude Agent SDK overview + hooks + TypeScript reference + sessions + Remote Control docs +- This kit: `README.md`, `src/types`, `src/forwarder`, `src/processing`, `docs/reference/hook-events.md` +- t3code: `docs/internals/overview.md`, `providers.md`, `ClaudeDriver.ts`, `ClaudeAdapter.ts` (`query()`, `canUseTool`, `settingSources`) +- Grok user guide: `10-hooks.md`, `17-sessions.md`, `14-headless-mode.md`, `15-agent-mode.md`, `05-configuration.md` +- grok-build: `xai-grok-hooks` (`event.rs`, `result.rs`, `lib.rs`), `xai-hooks-plugins-types`, `xai-grok-session-events` (`types.rs`) +- Live session dump under `~/.grok/sessions/%2FUsers%2Fdarkomijic%2Fdev-libar%2Flibar-agent-harness-kit/` diff --git a/plans/omo-native-adapter/cockpit-phase-c.md b/plans/omo-native-adapter/cockpit-phase-c.md new file mode 100644 index 0000000..dea1bed --- /dev/null +++ b/plans/omo-native-adapter/cockpit-phase-c.md @@ -0,0 +1,430 @@ +# Cockpit Phase C - OmO native observe adapter (execution brief) + +Status: SPEC · 2026-08-22 · Companion to [planning-brief.md](planning-brief.md) +Audience: implementers in `libar-cockpit` (not this repo) +Scope of this file: cross-repo execution brief only. No cockpit code lands here. + +This brief is the mirror plan for the Grok observe slice already shipped in +cockpit under `src/main/services/grok/*` and +`src/main/services/daemon/daemonComposition/grokSessionProcessing.ts`. Phase C +adds a parallel attach-only adapter for OmO native sessions (engine: senpi, +store: `~/.omo/agent`), consuming +`@libar-dev/agent-harness-kit/senpi` and +`@libar-dev/agent-harness-kit/senpi/processing` from the `senpi-adapter` +branch of this kit. + +--- + +## Harness id decision + +**Shared schema harness id: `omo`.** + +Locked by product owner in `planning-brief.md`: + +| Layer | Identifier | Notes | +|---|---|---| +| Product name | OmO native | UI copy, docs, owner language | +| Kit module / package subpaths | `senpi`, `/senpi`, `/senpi/processing` | Engine-accurate; never renamed in the kit | +| Cockpit service directory | `src/main/services/omo/` | Product-facing layer uses `omo`, not `senpi` | +| `SessionHarness` enum value | `'omo'` | Add next to `'claude' \| 'grok'` | +| Canonical session id | `omo:` | Via `toCanonicalSessionId('omo', nativeId)` | +| Marker subdirectory | `/omo` | Parallel to `/grok` | + +Do not mix `omo` and `senpi` inside one layer. Kit APIs keep the `Senpi*` / +`senpi` prefix. Cockpit files, harness tags, canonical ids, and markers use +`omo`. + +Required schema edits in cockpit (not optional): + +- `src/shared/schemas/daemon.ts`: extend + `SessionHarness = z.enum(['claude', 'grok', 'omo'])` +- `toCanonicalSessionId`: add `case 'omo': return \`omo:${nativeId}\`` +- Every `switch (session.harness)` exhaustiveness site gains an `'omo'` arm + (spawn and drive keep rejecting non-claude, same as grok) + +--- + +## Prerequisite (blocking) + +**Cockpit ADR equivalent of ADR 0003 must land before any Phase C wiring.** + +Reference shape: `libar-cockpit/docs/decisions/0003-grok-observe-adapter.md`. + +The new ADR (suggested path +`libar-cockpit/docs/decisions/000N-omo-observe-adapter.md`) must lock at +minimum: + +1. OmO native joins Cockpit as **observe-only external sessions**. +2. Claude remains the only drive plane (spawn, PTY, Commit, managed permission + prompts). +3. Kit surface consumed is attach-only + (`/senpi` + `/senpi/processing`). No engine spawn, no RPC, no resume. +4. Identity: `harness: 'omo'`, canonical ids `omo:`. +5. Block mapping is lossy onto existing `TranscriptBlock` (no lossless senpi + union in the renderer). +6. Out of scope for this ADR: OmO spawn/PTY, managed hooks trust write, Stop + gate forwarding, shared kit `SessionBlock` unification, Kimi. + +Implementation PRs in cockpit must cite the accepted ADR. Wiring without it +is out of process. + +--- + +## Merge gate + +Owner decision, recorded in `.omo/plans/omo-native-adapter.md` commit +strategy and in `planning-brief.md` Phase C: + +> The harness-kit PR on branch `senpi-adapter` must **not** merge until +> Cockpit Phase C is implemented and tested end-to-end in the cockpit repo +> against this branch. + +Acceptance for the gate (cockpit side): + +1. Pin cockpit's `@libar-dev/agent-harness-kit` dependency at a commit on + `senpi-adapter` that exports `/senpi` and `/senpi/processing`. +2. Real-daemon E2E (mirror of `tests/main/daemon/grok-e2e-observe.spec.ts`) + covers discover, open, live catch-up, search, tag, and reset/rematerialize + against a fixture under a temp agent home shaped like `~/.omo/agent`. +3. Session-end path proves `watchSenpiSession` `quiescent` marks the session + non-live / ended without any `SessionEnd` hook. +4. Ledger rows added under a new "OmO observe slice" section in + `docs/feature-reality-ledger.md`; `docs/hooks-contract.md` documents the + `/senpi` + `/senpi/processing` dynamic imports. + +Harness-kit merge is blocked on that evidence, not on this brief alone. + +--- + +## Dynamic ESM import strategy + +Cockpit main is CJS. The kit is ESM-only (`"type": "module"`, no `require` +condition). Same permanent seam as Claude `/processing` and Grok +`/grok/processing` (see `docs/hooks-contract.md`). + +| Subpath | Load site | Named exports cockpit consumes | +|---|---|---| +| `@libar-dev/agent-harness-kit/senpi` | `omoRuntime.ts` (home resolution only, if not covered by listing options) | `resolveSenpiAgentHome`, `AGENT_DIR_ENV_NAMES` | +| `@libar-dev/agent-harness-kit/senpi/processing` | `omoProcessing.ts` default loader; also direct dynamic import in `omoDiscovery.ts` / `omoRuntime.ts` where Grok does the same for list/encode | `listSenpiSessions`, `listAllSenpiSessions`, `findSenpiSessionDirs`, `getSenpiSessionsRoot`, `encodeSenpiCwdDirname`, `tailSenpiSession`, `watchSenpiSession`, `commitSenpiSessionCheckpoint`, `foldSenpiBlockChanges`, `reduceSenpiProjection` | + +Rules: + +- **No static `import` from either subpath** in main-process production code + unless cockpit later chooses a bundler path that makes static ESM safe. + Default: `await import('...')` only. +- Centralize the processing load behind `loadOmoProcessing()` (mirror of + `loadGrokProcessing`) with `setOmoProcessingLoaderForTests` for unit tests. +- Surface load failures as `OmoIngestError` code `adapter-load-failed`. +- Do not add a static dependency edge in TypeScript `import type` from value + space that forces resolution at load time; `import type` of kit types is + fine and erases at emit. +- Pin stays a git commit SHA in cockpit `package.json`, same pattern as today. + +Update `libar-cockpit/docs/hooks-contract.md` when wiring lands: add a +"/senpi and /senpi/processing (contracted)" table next to the Grok table. + +--- + +## Session-end detection (no SessionEnd) + +Senpi's supported hook events are exactly seven: +`PreToolUse`, `PostToolUse`, `UserPromptSubmit`, `SessionStart`, +`PreCompact`, `PostCompact`, `Stop`. There is **no `SessionEnd`**. + +Claude path today ends sessions via `SessionEnd` in +`sessionLifecycle/hookIngress.ts`. That path does not exist for OmO. + +**Primary end signal: `watchSenpiSession` quiescence.** + +| Signal | Source | Cockpit action | +|---|---|---| +| `watchSenpiSession` yields `{ type: 'quiescent' }` | kit `/senpi/processing`, default `quiescenceMs = 30000` | Mark observed session `status: 'ended'`, stop or back off the observer, emit list invalidation | +| Discovery `isLive` | `nowMs - mtimeMs < COCKPIT_LIVE_WINDOW_MS` on the session `.jsonl` | Catalog liveness only; does not replace the end signal | +| `Stop` hook (optional later) | observe-only forwarder, if registered in Phase D | Weak activity signal only; must not drive gate/block in Phase C | + +Implementation notes for cockpit: + +1. Live catch-up can keep the existing poll observer pattern in + `sessionLifecycle/observation.ts` (stat the single `.jsonl` path; senpi is + one file, so the Claude `stat(session.jsonlPath)` arm works once + `harness === 'omo'` is not forced through `statGrokSessionSources`). +2. Session-end must additionally run (or subscribe to) `watchSenpiSession` and + treat one `quiescent` yield as end. Do not invent a fake `SessionEnd` + envelope. +3. Default window: kit `quiescenceMs` 30000. Tune only with fixture evidence; + inject `clock` in tests (no real sleeps). +4. Missing file: watch retains the checkpoint and does not reset until a + replacement is observed. Cockpit must not clear SQLite history on transient + ENOENT during that window. +5. Features that currently key off Claude `SessionEnd` (status flip, list + event reason strings) degrade to the quiescent path for `harness === 'omo'`. + They must not break. + +--- + +## Lossy block map: `SenpiSessionBlock` → `TranscriptBlock` / `DaemonSessionSummary` + +### A. `SenpiSessionInfo` → `DaemonSessionSummary` + +Source: kit `listSenpiSessions` / `listAllSenpiSessions` → `ValidSenpiSession.info`. + +| `DaemonSessionSummary` field | Source | Notes | +|---|---|---| +| `projectCwd` | discovery call arg | Header cwd already verified by kit listing | +| `sessionId` | `toCanonicalSessionId('omo', info.id)` | `omo:` | +| `jsonlPath` | `info.path` | Absolute path of the session `.jsonl` file (not a directory) | +| `startedAt` | `info.created.toISOString()` | Header timestamp | +| `mtimeMs` | `info.modified.getTime()` | File mtime from listing | +| `isLive` | `nowMs - mtimeMs < COCKPIT_LIVE_WINDOW_MS` | Same window constant as Grok/Claude | +| `firstUserText` | `info.firstMessage` | Kit already extracts first user text | +| `spawnedBy` | `'external'` | Always; OmO is never cockpit-spawned in Phase C | +| `controllerStatus` | `null` | No PTY | +| `harness` | `'omo'` | | +| `title` | `info.name ?? null` | Latest `session_info` entry display name | + +**Fields dropped from `SenpiSessionInfo` (not on `DaemonSessionSummary`):** + +- `cwd` (redundant after project filter; do not surface as a summary column) +- `messageCount` +- `parentSessionPath` (fork lineage; optional later as system/info on open, not catalog) + +Invalid listings (`kind: 'invalid'`) are logged and skipped, same as Grok +invalid sessions. + +### B. `SenpiSessionBlock` → `TranscriptBlock` + +Source blocks come from kit `tailSenpiSession` changes, optionally folded via +`foldSenpiBlockChanges` on rematerialize. Normalize in `omoNormalize.ts`. +Canonical block ids: prefix with `omo:` when absent (mirror +`toCanonicalBlockId` in `grokNormalize.ts`). + +| Senpi input | Transcript output | Mapping rule | +|---|---|---| +| `message` + role `user` + content `text` | `user_text` | `content = text.text` | +| `message` + role `user` + content `image` | `image` | `data`, `mediaType` from `mimeType` when it matches the allowed enum; else `system/info` fallback | +| `message` + role `assistant` + content `text` | `assistant_text` | `content = text.text` | +| `message` + role `assistant` + content `thinking` | `thinking` | `content = thinking.thinking` | +| `message` + role `assistant` + content `toolCall` | `tool_use` | `toolUseId = toolCall.id`, `toolName = toolCall.name`, `input = toolCall.arguments`, `summary = toolCall.name` | +| `message` + role `toolResult` | `tool_result` | `toolUseId =` from message (via content/role fields retained on the block through reduction; use `entryId` only as id seed, not as tool id). Prefer reading tool linkage from the native message before reduction if the block alone is insufficient; Phase C normalize may need the folded message role fields already on `SenpiMessageBlock` (`isError`) plus text join of content. `toolName` when known. `structuredResult` left unset unless a single structured payload is cheap to attach. | +| `message` + role `bashExecution` | `system` / `kind: 'info'` | Title `omo.bashExecution`; body = bounded JSON of command/output/exit | +| `message` + role `branchSummary` | `system` / `kind: 'info'` | Title `omo.branchSummary` | +| `message` + role `compactionSummary` | `system` / `kind: 'info'` | Title `omo.compactionSummary` | +| `message` + role `custom` | `system` / `kind: 'info'` | Title `omo.custom.`; high-volume extension traffic (`senpi.todo-state`, `omo-senpi:wake`, …) stays inspectable, never dropped silently | +| `metadata` (any `entryType`) | `system` / `kind: 'info'` | Title `omo.meta.`; body = bounded JSON of `payload` (+ `customType` when set) | +| `SenpiBlockChange` type `delete` | `SessionMutation` type `delete` | `blockId = toCanonicalBlockId(id)` | +| Tail diagnostics (if any) | `system` / `kind: 'info'` | Collapse per kind, same throttle pattern as Grok diagnostics | + +**Fields always dropped on the way into `TranscriptBlock` (lossy by design):** + +From `SenpiSessionBlockBase`: + +- `parentId` +- `origin` (`entry` \| `retained_tail`) +- `branch` (`active` \| `off_branch` \| `summarized`) +- `entryType` (except as the system/info title suffix for metadata) +- `entryId` (except as the native id seed before `omo:` prefixing) +- `entryTimestamp` when `messageTimestamp` is used (pick message time; fall back to entry time; never keep both) +- `usage` (`SenpiUsage` input/output/cache/cost) +- `customType` as a first-class Transcript field (survives only inside system/info title or body) +- `isError` on non-`tool_result` mappings (assistant errors become system/info or a flag only on tool_result) + +From content payloads: + +- `thinking.startedAt`, `thinking.endedAt` +- Assistant wire fields not on the block surface after reduction: `api`, + `provider`, `model`, `stopReason`, `errorMessage` (unless folded into a + system/info companion for the error case) +- `toolResult.details`, `toolResult.usage` +- `bashExecution.command`, `exitCode`, `cancelled`, `truncated`, + `fullOutputPath`, `excludeFromContext` as typed fields (only inside + bounded JSON body) +- `custom.display`, `custom.details` +- `branchSummary.fromId`, `compactionSummary.tokensBefore` as typed fields +- Off-path projection records not reduced into active blocks (kit leaves + those on `SenpiProjectionResult.offPath`; cockpit Phase C does not render + them unless a later decision says otherwise) + +Bounds: reuse Grok's `SYSTEM_INFO_MAX_CHARS = 2048` for system/info bodies. +Collapse repeated diagnostic/customType keys the way +`collapseDiagnostics` does for Grok. + +No shared kit `SessionBlock`. No attempt to round-trip OmO blocks through the +Claude JSONL normalizer. + +--- + +## File-by-file mapping + +Paths on the left are current cockpit files. Paths on the right are the +cockpit files to **create** (new `omo/` tree) or **modify** (shared wiring). +Kit symbols are the harness-kit exports each file should call. + +### New service tree (mirror of `src/main/services/grok/`) + +| Cockpit Grok file | Cockpit OmO file to create | What changes | +|---|---|---| +| `src/main/services/grok/grokDiscovery.ts` | `src/main/services/omo/omoDiscovery.ts` | `discoverOmoSessions({ projectCwd, options })` dynamically imports `listSenpiSessions` from `/senpi/processing`, passes `agentHome` from `resolveOmoHome`, maps valid `SenpiSessionInfo` → `DaemonSessionSummary` with `harness: 'omo'`, logs+skips invalid rows | +| `src/main/services/grok/grokIngest.ts` | `src/main/services/omo/omoIngest.ts` | `ingestOmoSession` / `previewOmoSessionHistory` call `tailSenpiSession` + optional `foldSenpiBlockChanges`, normalize via `omoNormalize`, persist through existing `sessionMutationStream` helpers, commit via `commitSenpiSessionCheckpoint`; `jsonlPath` is the `.jsonl` file path | +| `src/main/services/grok/grokIngestError.ts` | `src/main/services/omo/omoIngestError.ts` | Same code union (`adapter-load-failed`, `source-read-failed`, `persistence-failed`, `checkpoint-commit-failed`) under `OmoIngestError` / `asOmoIngestError` | +| `src/main/services/grok/grokIngestSerializer.ts` | `src/main/services/omo/omoIngestSerializer.ts` | Per-canonical-session serial queue + reentrancy guard; copy structure, rename types only (`createOmoIngestSerializer`) | +| `src/main/services/grok/grokNormalize.ts` | `src/main/services/omo/omoNormalize.ts` | Implement the lossy map in section "Lossy block map"; input types from `/senpi/processing` (`SenpiBlockChange`, `SenpiSessionBlock`, tail diagnostics if exposed) | +| `src/main/services/grok/grokProcessing.ts` | `src/main/services/omo/omoProcessing.ts` | `loadOmoProcessing()` → `import('@libar-dev/agent-harness-kit/senpi/processing')`; test loader override; wrap failures as `adapter-load-failed` | +| `src/main/services/grok/grokRuntime.ts` | `src/main/services/omo/omoRuntime.ts` | `getOmoSessionMarkersDir` → `join(getSessionMarkersDir(options), 'omo')`; `resolveOmoHome` prefers kit `resolveSenpiAgentHome({ env, homeDir })` (honors `OMO_CODING_AGENT_DIR` / `SENPI_CODING_AGENT_DIR` / `PI_CODING_AGENT_DIR`) rather than inventing a parallel home algorithm; `resolveOmoSessionScopes` uses `getSenpiSessionsRoot` + `encodeSenpiCwdDirname` and returns `{ projectCwd, sessionsRoot }` for catalog watch | +| `src/main/services/grok/grokSessionFs.ts` | `src/main/services/omo/omoSessionFs.ts` | `isOmoSessionFile(path)` (`.jsonl` file whose first parseable header is senpi session v3, or cheaper: file under agent sessions root with `.jsonl` suffix + readable header); `statOmoSessionSource(path)` returns `{ mtimeMs, size }` for the single file (no `events.jsonl`) | + +### New daemon composition processor + +| Cockpit Grok file | Cockpit OmO file to create | What changes | +|---|---|---| +| `src/main/services/daemon/daemonComposition/grokSessionProcessing.ts` | `src/main/services/daemon/daemonComposition/omoSessionProcessing.ts` | `createOmoSessionProcessor`: `loadHistory` / `reindexSession` call `previewOmoSessionHistory` / `ingestOmoSession` through `OmoIngestSerializer`; persist `harness: 'omo'` on `upsertSession` | + +### Shared wiring to modify (not silent drops) + +These are not under `services/grok/` but already branch on Grok and must grow +an OmO arm. Listed so the completeness pass cannot miss them. + +| Cockpit file | Change | +|---|---| +| `src/shared/schemas/daemon.ts` | `SessionHarness` + `toCanonicalSessionId` for `'omo'` | +| `src/main/services/daemon/daemonComposition/contracts.ts` | Hold `omoIngestSerializer` next to `grokIngestSerializer` | +| `src/main/services/daemon/daemonComposition/createComposition.ts` | Construct `createOmoIngestSerializer`, wire `discoverOmoSessions` + `resolveOmoSessionScopes` into search + catalog watcher deps | +| `src/main/services/daemon/daemonComposition/createDaemon.ts` | `createOmoSessionProcessor` and pass into history/reindex | +| `src/main/services/daemon/daemonComposition/sessionHistory.ts` | Route `harness === 'omo'` or `isOmoSessionFile(jsonlPath)` to `omoSessionProcessor` | +| `src/main/services/daemon/daemonComposition/projectReindex.ts` | `case 'omo':` → `omoSessionProcessor.reindexSession` | +| `src/main/services/daemon/sessionLifecycle/contracts.ts` | Optional `omoIngestSerializer` dep | +| `src/main/services/daemon/sessionLifecycle/ingest.ts` | `catchUpOmoSession` via `ingestOmoSession` when `harness === 'omo'` | +| `src/main/services/daemon/sessionLifecycle/observation.ts` | Stat path: single-file stat for `omo` (not grok multi-file); session-end via `watchSenpiSession` quiescent (see above) | +| `src/main/services/daemon/sessionLifecycle/observationErrors.ts` | Treat `OmoIngestError` like `GrokIngestError` for recoverable retry | +| `src/main/services/daemon/sessionLifecycle/spawnControl.ts` | `case 'omo':` reject with `unsupported-harness` (mirror grok) | +| `src/main/services/daemon/sessionLifecycle/authorization.ts` | Non-claude drive rejection already covers omo once the enum widens; confirm exhaustiveness | +| `src/main/services/daemon/sessionCatalogWatcher/contracts.ts` | `resolveOmoSessionScopes` / `discoverOmoSessions` optional deps | +| `src/main/services/daemon/sessionCatalogWatcher/scopeInitializer.ts` | Register omo sessionsRoot scopes next to grok | +| `src/main/services/daemon/searchService/contracts.ts` | `discoverOmoSessions` optional dep; include omo summaries in discovery context | +| `src/main/services/daemon/searchService/discovery.ts` | Merge omo summaries into discovered set | +| `src/main/services/daemon/searchService/sessionCatalog.ts` | Catalog lookup includes omo summaries (same merge pattern as grok) | +| `docs/hooks-contract.md` | Document `/senpi` + `/senpi/processing` dynamic import seam | +| `docs/feature-reality-ledger.md` | New OmO observe slice rows after E2E | +| `docs/decisions/000N-omo-observe-adapter.md` | Prerequisite ADR (see above) | + +### Tests to add in cockpit (guidance, not exhaustive) + +| Grok test fixture | OmO counterpart | +|---|---| +| `tests/main/daemon/grok-discovery.spec.ts` | `omo-discovery.spec.ts` | +| `tests/main/daemon/grok-ingest.spec.ts` | `omo-ingest.spec.ts` | +| `tests/main/daemon/grok-e2e-observe.spec.ts` | `omo-e2e-observe.spec.ts` (merge-gate evidence) | +| `tests/main/daemon/grok-reset-checkpoint.spec.ts` | `omo-reset-checkpoint.spec.ts` (suffix-splice rematerialize) | +| `docs/qa/grok-observe-qa.md` | `docs/qa/omo-observe-qa.md` | + +Use kit fixtures under harness-kit `tests/fixtures/senpi/` (sanitized) or a +cockpit-local copy; never commit unredacted `~/.omo/agent` transcripts. + +--- + +## Kit export surface this brief assumes + +Landed (or landing on `senpi-adapter`) public names cockpit may call: + +**`@libar-dev/agent-harness-kit/senpi`** (from `src/senpi/index.ts`): +`resolveSenpiAgentHome`, `AGENT_DIR_ENV_NAMES`, `AGENT_HOME_SENTINEL`, +hooks-config validators, session entry types. Phase C observe needs home +resolution primarily; hooks registration is Phase D. + +**`@libar-dev/agent-harness-kit/senpi/processing`** (from +`src/senpi/processing/index.ts`): +`parseSenpiEntry`, `findSenpiSessionDirs`, `getSenpiSessionsRoot`, +`listSenpiSessions`, `listAllSenpiSessions`, `projectSenpiBranch`, +`resolveSenpiLeaf`, `tailSenpiSession`, `watchSenpiSession`, +`commitSenpiSessionCheckpoint`, `foldSenpiBlockChanges`, +`reduceSenpiProjection`, plus block/listing types +(`SenpiSessionBlock`, `SenpiBlockChange`, `SenpiSessionInfo`, …). + +Cursor and checkpoint marker internals stay unexported. Cockpit must not +reach into kit private paths. + +`encodeSenpiCwdDirname` is exported from the processing module source +(`discovery.ts`) and is part of the processing surface cockpit runtime uses +for scope roots (same role as Grok's `encodeGrokCwdDirname`). If the barrel +ever omits it, Phase C unblocks by exporting it from +`src/senpi/processing/index.ts` in the kit before cockpit wiring merges. + +--- + +## Explicit non-goals (Phase C) + +- No OmO/senpi spawn, PTY, Commit, or CLI resume from cockpit +- No managed hooks trust write, forwarder install, or Stop-gate block policy + (Phase D / later cockpit work) +- No translation of senpi hook envelopes into the Claude hook sink +- No reading `auth.json` or `omo-senpi/` telemetry state +- No multi-home auto-merge (second stores only via env / explicit home option) +- No shared `SessionBlock` unification across claude/grok/omo in the kit +- No code changes in harness-kit for Phase C beyond what Phases A/B/D already + plan; this brief is cockpit work specified from the kit repo + +--- + +## Implementation order (cockpit) + +1. Land prerequisite ADR (`000N-omo-observe-adapter`). +2. Schema: `SessionHarness` + `toCanonicalSessionId`. +3. Create `src/main/services/omo/*` tree (processing load → runtime → fs → + normalize → ingest → discovery → serializer → errors). +4. Create `omoSessionProcessing.ts`; wire composition, history, reindex, + search, catalog, lifecycle ingest/observe/errors/spawn reject. +5. Session-end: `watchSenpiSession` quiescent path. +6. Unit tests, then real-daemon E2E, then QA doc + ledger + hooks-contract + update. +7. Point cockpit pin at the `senpi-adapter` commit under test; only then is + the harness-kit merge gate eligible to open. + +--- + +## GAPS + +Items intentionally not given a 1:1 new file, with reason. Nothing under +`services/grok/` is omitted from the mapping table above. + +| Item | Reason | +|---|---| +| Grok two-file layout (`updates.jsonl` + `events.jsonl`) helpers inside `grokSessionFs` / ingest `eventsMissing` | Senpi is a single `.jsonl`; no events file. `eventsMissing` has no OmO analogue (always false / omit field). | +| Grok `includeActivities: true` tail option | Senpi tail options differ (`SenpiSessionTailOptions`); no activities stream. Map only real senpi options (`markerDir`, `allowedMarkerRoots`, `fromStart`, checkpoint mode). | +| Claude `SessionEnd` hook ingress | No senpi SessionEnd. Covered by quiescence section, not a new hook handler. | +| Grok `/grok` hooks subpath (`executeGrokHook`, …) | Phase C is observe-only. Senpi hooks library is Phase B/D in the kit; cockpit managed registration is out of Phase C scope. | +| Direct reuse of `GrokIngestSerializer` without rename | Forbidden. Separate serializer instance avoids cross-harness queue coupling and keeps stop/shutdown independent. | +| Renderer/UI chrome beyond existing external-session views | Out of scope for this brief; existing external session UI should accept `harness: 'omo'` once summaries flow. Any dedicated badge copy is a separate cockpit UI task. | + +--- + +## Completeness checklist + +Every file currently under +`libar-cockpit/src/main/services/grok/` plus +`daemonComposition/grokSessionProcessing.ts` must appear in the mapping +table. Run at brief-authoring time: + +```bash +# From harness-kit repo, against a local cockpit checkout. +GROK_DIR=/Users/darkomijic/dev-libar/libar-cockpit/src/main/services/grok +BRIEF=plans/omo-native-adapter/cockpit-phase-c.md + +echo "== grok service files ==" +for f in "$GROK_DIR"/*.ts; do + base=$(basename "$f") + if rg -q "$base" "$BRIEF"; then + echo "MATCH $base" + else + echo "MISS $base" + fi +done + +echo "== grokSessionProcessing ==" +base=grokSessionProcessing.ts +if rg -q "$base" "$BRIEF"; then echo "MATCH $base"; else echo "MISS $base"; fi +``` + +Expected: every line `MATCH`. Any `MISS` must move into GAPS with a written +reason before this brief is accepted. + +Authoring-time audit output is recorded in +`.omo/evidence/task-23-phase-c-brief.log`. diff --git a/plans/omo-native-adapter/planning-brief.md b/plans/omo-native-adapter/planning-brief.md new file mode 100644 index 0000000..32f47bb --- /dev/null +++ b/plans/omo-native-adapter/planning-brief.md @@ -0,0 +1,137 @@ +# Planning Brief — OmO Native adapter (senpi attach/observe + hooks) + +Status: PROPOSED · 2026-08-21 · Companion to [investigation.md](investigation.md) +Scope: `@libar-dev/agent-harness-kit` only. Cockpit (Phase C) is the integration gate, not work done here. + +## Decisions locked by product owner + +| Question | Decision | +|---|---| +| Product name | **OmO native** (core team's term). Kit module stays engine-accurate: `src/senpi/`, subpaths `/senpi`, `/senpi/processing`. Cockpit-facing harness id: `omo`. No `omo-native`/`senpi` mixing within one layer. | +| MVP scope in this repo | Phases **A, B, D** (observe, hooks library, managed-hooks groundwork). **No PR merge in harness-kit until Phase C (Cockpit observe integration) is implemented and tested against this branch.** | +| Session listing | Must list **all sessions in all projects** from the common user-level store, and support opening sessions from any folder. This is the engine's native model — see "All-projects listing" below. No "extra roots" concept needed for MVP. | +| Hooks | **Essential, not optional.** Minimum: monitoring of active sessions via hooks. Full Cockpit hook surface documented in this brief, delivered later. | + +## Contract source of truth + +Engine pin: `@code-yeongyu/senpi` **2026.8.19** (the version OmO pins in `packages/omo-senpi/package.json`). OmO is a branded distribution; the engine owns storage, hooks, and session format. Kit never depends on `oh-my-openagent` at runtime. + +Vendor under `docs/upstream/senpi/` with `pin.json` (version + per-file sha256) and `scripts/sync-upstream-senpi.mjs`: + +- `docs/session-format.md`, `docs/settings.md`, `docs/environment-variables.md` (from the npm tarball; `docs/` ships in the published package) +- `dist/core/extensions/builtin/hooks/*.d.ts` — the hooks wire contract is **undocumented** in senpi's docs index; dist types are the only spec +- Drift tests assert our Zod schemas and event lists against the vendored types, mirroring `tests/grok-upstream-drift.test.ts` + +## Resolved risks (previously open — now investigated in the pinned dist) + +### 1. Tree tail + compaction (highest risk) — design + +Session JSONL is a v3 tree (`id`/`parentId`, header `type:"session" version:3`). The kit owns linearization; consumers stay linear like the Claude path. + +- `parseSenpiEntry(raw)` → `known | unknown | invalid` (Grok policy: unknown tags preserved, never fatal). +- `tailSenpiSession(file, {markerDir, allowedMarkerRoots, ...})` → incremental read since byte marker, then **leaf-path projection**: build parent map from all entries, resolve current leaf (last appended entry id is the engine's leaf; entries only append — branching re-parents new entries, never rewrites old lines), emit records ordered root→leaf along the active path. Entries not on the leaf path are emitted as `off-branch` metadata, not conversation blocks. +- Compaction: a `compaction` entry with `retainedTail` is a self-contained checkpoint — projection emits the compaction record and, for consumers that want full context, the retained tail; `firstKeptEntryId` (legacy) truncates the projected prefix. +- `parentSession` forks: header field only; cross-file lineage exposed as metadata (`parentSessionPath`), never followed during tail. +- Live-append safety: appends are line-atomic JSONL; marker format and inode-reset handling reuse the existing `jsonl-cursor` pattern from `src/grok/processing`. +- Fixtures: real transcripts from `~/.omo/agent/sessions/` (116 MB, dense trees, live `senpi.hooks.stop-state` / `omo-senpi:wake` traffic) copied into `tests/fixtures/senpi/` as sanitized subsets. + +### 2. Custom-message density — neutral parse, lossy map + +Real sessions carry heavy extension traffic: `senpi.todo-state` (×101), `senpi.hooks.stop-state` (×114), `goal-cache-warmup` (×117), `omo-senpi:wake` (×173), memory bindings, checkpoints. The kit parses and folds these **neutrally**: `custom`/`custom_message` entries become typed records with `customType` preserved and payload validated loosely (`unknown` data). Cockpit maps lossily to UI blocks exactly as it does for Grok's `system/info` fallback. No OmO product-surface awareness in kit v1. Volume guard: folding must be O(n) with no per-entry allocations retained for `custom` entries beyond the record itself. + +### 3. Hook trust — mechanics now known, managed grant is a designed operation + +- Trust state lives in `/hooks-state.json` (global) and `/.senpi/hooks-state.json` (project), file-locked, `HookTrustState v1`: per-hook `enabled`, `trustedHash` (`hashCommandHook(handler)`), `commandPreview`, `updatedAt`. +- Untrusted or disabled handlers are **skipped with diagnostics** (`reason: "disabled" | "untrusted" | "unsafe"`), never run. +- Managed-install solution (Phase D): the kit computes the exact trust id + hash (`hookTrustId`, `hashCommandHook` are pure functions over handler config + platform) and writes the trust entry **only as an explicit, user-approved operation** (Cockpit consent UI or documented manual step). Writing `hooks-state.json` programmatically is semantically the user approving the hook — the kit treats it as such and never does it silently. This unblocks headless managed hooks without weakening the gate. + +### 4. No `SessionEnd` — watch + quiescence + +Supported events are exactly 7: `PreToolUse`, `PostToolUse`, `UserPromptSubmit`, `SessionStart`, `PreCompact`, `PostCompact`, `Stop`. Session-end detection for observe consumers: + +- Primary: `watchSenpiSession` mtime quiescence on the JSONL (same pattern as Grok observe), plus `session_shutdown`-equivalent absence of new entries. +- Weak signal: a `Stop` hook with no follow-up within a bounded window. +- Documented explicitly as a divergence from the Claude lifecycle; Cockpit's SessionEnd-dependent features degrade, not break. + +### 5. Stop loop / `continue` — semantics read, forwarding rules fixed + +From `output-parser.js` / `stop-adapter.js` / the `loop-guard` builtin: + +- Exit code **2** ⇒ `{decision: "block", reason: }` regardless of stdout. +- Otherwise stdout JSON; `continue: false` on a Stop hook ⇒ `decision: "block"`; a Stop block without follow-up context produces an engine-side warning, and exit-2 blockers are excluded from user-facing messaging. +- The engine's `loop-guard` builtin owns the first veto so repeated calls never re-run hooks, and the Stop adapter persists turn state (`senpi.hooks.stop-state` custom entries) — loop protection exists upstream. +- Rule for v1: **Stop hooks are observe-only in kit examples and Cockpit wiring.** Forwarding Stop with block/continue semantics is Phase D+ and requires the trust story plus a written loop-safety rationale. The kit's output builder exposes the full `HookOutputWire` vocabulary (it is the wire contract), but documentation marks Stop-gate outputs as advanced. + +### Exit-code / output semantics (complete, for the runner) + +| Input | Behavior | +|---|---| +| exit 2 | block; reason = stderr | +| exit other, empty stdout | no-op | +| exit other, stdout JSON | parse universal fields (`continue`, `stopReason`, `suppressOutput`, `systemMessage`) + event-specific fields; `hookSpecificOutput` supported; malformed JSON or non-object ⇒ diagnostic, no-op | +| `systemMessage` | honored for `PreToolUse`, `PostToolUse`, `UserPromptSubmit`, `SessionStart`, `Stop` only | +| timeout / abort | killed process tree, `exitCode: null`, `timedOut`/`aborted` flags; bounded stdout/stderr captures | + +### All-projects listing (owner question resolved) + +"Extra roots" was about non-default storage locations — irrelevant for MVP. The engine's native model already does what Cockpit needs: + +- Every session for every project lives under the **single user-level home** (`/sessions/<--encoded-cwd-->/`), resolved by `resolveAgentHome` semantics: `OMO_CODING_AGENT_DIR`/`SENPI_CODING_AGENT_DIR`/`PI_CODING_AGENT_DIR` env → `~/.omo/agent` (sentinel `settings.json`) → legacy flat `~/.omo` → `~/.senpi/agent`. +- Engine parity API: `SessionManager.listAll()` → `SessionInfo { path, id, cwd, name?, parentSessionPath?, created, modified, messageCount, firstMessage, allMessagesText }`. +- Kit exports `listAllSenpiSessions(agentHome?)` with the same shape (own implementation, no engine import) plus `listSenpiSessions(projectCwd)` for per-project filtering. Opening sessions "from other folders" is just opening any file in the store — senpi itself supports `--session `; no special handling needed. +- Discovery filters: top-level `*.jsonl` only; skip `*-artifacts/` dirs and per-project `extensions/` subdirs; **verify header `cwd`** (dash-encoding is ambiguous: `a/b` vs `a-b`). + +## Hooks surface (Phase B) — what "essential monitoring" means + +Minimum viable hooks support for Cockpit-style monitoring: + +1. `validateSenpiHooksConfig(json)` — parity with engine `parseHookConfig`: 7 events, `command` handlers only (http/prompt/agent/mcp_tool rejected with diagnostics), matchers, timeouts, `commandWindows`, the full `HookDiagnosticCode` vocabulary, config sources `/hooks.json` + `/.senpi/hooks.json` + `hooks` keys in settings.json. +2. `senpiHookInputSchema` / `senpiHookOutputSchema` — Zod over `HookInputWire`/`HookOutputWire`, alias-tolerant (camelCase primaries, Claude-style snake_case aliases: `hook_event_name`, `session_id`, `transcript_path`, `tool_name`, `tool_input`, `tool_response`). +3. `executeSenpiHook(handler)` runner — stdin JSON, exit-2 block, stdout-JSON parse, bounded captures; matches `runCommandHook` semantics. +4. `SenpiHookOutputBuilder` — `decision approve|block|deny|ask`, `reason`, `additionalContext`, `updatedInput`, `updatedToolOutput`, `continue`, `stopReason`, `suppressOutput`, `systemMessage` (event-gated). +5. Trust read/inspect API: `readSenpiHookTrustState(path)`, `computeSenpiHookTrust(handler)` — read-only in Phase B; writing trust entries is Phase D's consent-gated operation. + +**Full Cockpit hook surface (documented now, delivered later):** managed registration of a forwarding command hook (kit forwarder asset → Cockpit daemon endpoint, same pattern as Claude), trust consent flow, Stop-gate policy per risk 5, and the extension-API surface (`ExtensionAPI` in-process events) explicitly **out of scope** — that is an OmO-plugin concern, not a kit attach concern. + +## Phases + +### Phase A — observe adapter (this repo, first vertical slice) + +- `src/senpi/`: `types.ts` (entry/message/block Zod unions), `home.ts` (`resolveSenpiAgentHome`), `processing/` (discovery, `listSenpiSessions`, `listAllSenpiSessions`, `parseSenpiEntry`, leaf-linear `tailSenpiSession`, `watchSenpiSession`, checkpoint commit, fold to block changes) +- Vendored pin artifacts + drift tests +- Fixtures from real `~/.omo/agent` transcripts (sanitized) +- Accept: `pnpm run check` + `pnpm run test:run` green; tail of a real fixture produces the same linear history the engine's `/resume` shows for that session (spot-verified) + +### Phase B — hooks library (this repo) + +- Config validator, wire schemas, runner, output builder, trust read/inspect +- Accept: contract tests against vendored dist types; round-trip tests; drift tests + +### Phase C — Cockpit integration (gate for merging here) + +- Cockpit observe adapter mirroring `src/main/services/grok/*`, harness id `omo`, lossy block map +- **This repo's PR stays unmerged until Phase C is implemented and tested end-to-end against this branch** (owner decision) + +### Phase D — managed hooks groundwork (this repo, planning context) + +- Consent-gated trust writer (`hooks-state.json`), forwarder asset variant for senpi wire, registration helper writing `hooks.json` +- Stop-gate forwarding policy doc; SessionEnd quiescence recipe for consumers + +## Explicit non-goals + +- No Claude `/processing` routing or shared `SessionBlock` unification (same refusal as Grok) +- No HTTP/prompt/agent/MCP hook handlers (engine rejects them) +- No reading `auth.json`; `omo-senpi/` telemetry state is not session contract +- No drive/RPC/spawn ("OmO native" means attach-observe; the engine's `rpc.md`/`app-server.md` control planes are out of scope) +- No OmO plugin/extension-API surface in the kit +- No multi-home auto-merge (a second store like `/Volumes/.../.omo` is reachable only via env/option, never merged by default) + +## Open items (non-blocking for A/B) + +1. Cockpit ADR equivalent of 0003 for OmO-native observe (Cockpit repo, before Phase C wiring). +2. Exact quiescence window + debounce for session-end detection (tune against live fixtures in Phase A tests). +3. Whether Cockpit wants `Stop` as a session-activity signal in C (observe-only is safe; gating is D). + +## Bottom line + +Phase A is implementable immediately from this brief: the wire contract is fully read (session format, hooks types, trust mechanics, exit codes, listAll), the pin target is fixed at `2026.8.19`, and the one genuinely hard component — leaf-linear tail with compaction — has a concrete design. The main discipline is keeping hooks at Phase-B scope and refusing to let "OmO native" drift into a control plane. diff --git a/scripts/clean-consumer.mjs b/scripts/clean-consumer.mjs new file mode 100644 index 0000000..007e238 --- /dev/null +++ b/scripts/clean-consumer.mjs @@ -0,0 +1,620 @@ +#!/usr/bin/env node +/** + * Pack the 0.3.0 candidate and exercise it as a clean consumer. + * + * Installs the packed tarball into an isolated project with lifecycle + * scripts enabled. Does not use workspace: or file: links to the source + * tree. Warnings are captured, not suppressed. + */ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { access, copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const defaultRepoRoot = join(scriptDir, '..'); +const probeRelativePath = join( + 'tests', + 'fixtures', + 'clean-consumer', + 'probe.mjs' +); + +const OFFICIAL_NODES = { + 20: '20.20.2', + 22: '22.23.2', + 24: '24.19.0', +}; + +function log(message) { + process.stderr.write(`${message}\n`); +} + +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parseArgs(argv) { + const parsed = { + mode: 'happy', + tarball: undefined, + node: undefined, + repo: defaultRepoRoot, + keep: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + if (arg === '--mode' && next !== undefined) { + parsed.mode = next; + index += 1; + continue; + } + if (arg === '--tarball' && next !== undefined) { + parsed.tarball = next; + index += 1; + continue; + } + if (arg === '--node' && next !== undefined) { + parsed.node = next; + index += 1; + continue; + } + if (arg === '--repo' && next !== undefined) { + parsed.repo = next; + index += 1; + continue; + } + if (arg === '--keep') { + parsed.keep = true; + continue; + } + if (arg === '--matrix') { + parsed.mode = 'matrix'; + continue; + } + throw new Error(`unknown argument: ${arg}`); + } + if ( + parsed.mode !== 'happy' && + parsed.mode !== 'engine-mismatch' && + parsed.mode !== 'matrix' + ) { + throw new Error(`unsupported mode: ${parsed.mode}`); + } + return parsed; +} + +function platformTriple() { + const platform = + process.platform === 'darwin' + ? 'darwin' + : process.platform === 'linux' + ? 'linux' + : null; + const arch = + process.arch === 'arm64' ? 'arm64' : process.arch === 'x64' ? 'x64' : null; + if (platform === null || arch === null) { + throw new Error( + `unsupported host for official Node cache: ${process.platform}-${process.arch}` + ); + } + return `${platform}-${arch}`; +} + +function npmForNode(nodePath) { + return join(dirname(nodePath), 'npm'); +} + +async function readNodeVersion(nodePath) { + const { stdout } = await execFileAsync(nodePath, ['-v']); + return stdout.trim(); +} + +async function ensureDist(repoRoot) { + try { + await access(join(repoRoot, 'dist', 'index.js')); + await access(join(repoRoot, 'dist', 'standalone', 'hook-forwarder.mjs')); + await access( + join(repoRoot, 'dist', 'standalone', 'hook-forwarder-senpi.mjs') + ); + } catch { + throw new Error( + 'dist runtime is missing; run `pnpm run build` before packing' + ); + } +} + +/** + * npm < 11 runs the `prepare` lifecycle script during `npm pack` even with + * --ignore-scripts (npm/cli#3080), printing the script banner and build + * output into stdout ahead of the JSON report. The JSON array is the final + * stdout payload and its top-level `[` is the only line-starting bracket, + * so anchor on the last line-initial `[` and fall back to a clean payload. + */ +function extractPackJsonArray(raw) { + const start = raw.lastIndexOf('\n['); + if (start !== -1) { + return raw.slice(start + 1); + } + const trimmed = raw.trimStart(); + if (trimmed.startsWith('[')) { + return trimmed; + } + throw new Error('npm pack --json output contained no JSON array'); +} + +function parsePackRecord(raw) { + const parsed = JSON.parse(extractPackJsonArray(raw)); + const record = Array.isArray(parsed) ? parsed[0] : parsed; + if (!isRecord(record) || typeof record.version !== 'string') { + throw new Error('npm pack --json did not return a versioned record'); + } + if (typeof record.filename !== 'string') { + throw new Error('npm pack --json did not return a filename'); + } + if (!Array.isArray(record.files)) { + throw new Error('npm pack --json did not return a file list'); + } + const files = record.files.map(entry => { + if (typeof entry === 'string') return entry; + if (isRecord(entry) && typeof entry.path === 'string') return entry.path; + throw new Error('npm pack file entry was not a path'); + }); + return { + version: record.version, + filename: record.filename, + files, + unpackedSize: + typeof record.unpackedSize === 'number' ? record.unpackedSize : null, + size: typeof record.size === 'number' ? record.size : null, + }; +} + +async function resolveRunnerNpm() { + const sibling = join(dirname(process.execPath), 'npm'); + try { + await access(sibling); + return sibling; + } catch { + return 'npm'; + } +} + +async function packCandidate(repoRoot, destDir) { + await ensureDist(repoRoot); + await mkdir(destDir, { recursive: true }); + const packNpm = await resolveRunnerNpm(); + log(`packing with ${packNpm} pack --ignore-scripts from ${repoRoot}`); + const { stdout, stderr } = await execFileAsync( + packNpm, + ['pack', '--ignore-scripts', '--pack-destination', destDir, '--json'], + { cwd: repoRoot } + ); + if (stderr.trim() !== '') { + log(stderr.trim()); + } + const packed = parsePackRecord(stdout); + if (packed.version !== '0.3.0') { + throw new Error(`packed version was ${packed.version}, expected 0.3.0`); + } + const tarball = join(destDir, packed.filename); + const bytes = await readFile(tarball); + const sha512 = createHash('sha512').update(bytes).digest('hex'); + return { + ...packed, + path: tarball, + sha512, + byteLength: bytes.byteLength, + }; +} + +async function inspectTarball(tarballPath) { + const bytes = await readFile(tarballPath); + const sha512 = createHash('sha512').update(bytes).digest('hex'); + const { stdout } = await execFileAsync('tar', ['-tzf', tarballPath]); + const files = stdout + .split('\n') + .map(line => line.replace(/^package\//, '')) + .filter(line => line !== '' && line !== '.'); + return { + path: tarballPath, + filename: tarballPath.split('/').pop() ?? tarballPath, + sha512, + byteLength: bytes.byteLength, + files, + version: '0.3.0', + unpackedSize: null, + size: bytes.byteLength, + }; +} + +async function createConsumerDir(parent) { + const consumerDir = join(parent, 'consumer'); + await mkdir(consumerDir, { recursive: true }); + await writeFile( + join(consumerDir, 'package.json'), + `${JSON.stringify( + { + name: 'kit-0.3.0-clean-consumer', + private: true, + type: 'module', + }, + null, + 2 + )}\n` + ); + return consumerDir; +} + +function collectEngineSignals(text) { + const ebadengine = + text.includes('EBADENGINE') || text.includes('Unsupported engine'); + const required = + /required:\s*\{[^}]*node:\s*'([^']+)'/.exec(text)?.[1] ?? + /required:\s*\{[^}]*node:\s*"([^"]+)"/.exec(text)?.[1] ?? + /Required:\s*\{\s*"node"\s*:\s*"([^"]+)"/.exec(text)?.[1] ?? + null; + const current = + /current:\s*\{[^}]*node:\s*'([^']+)'/.exec(text)?.[1] ?? + /current:\s*\{[^}]*node:\s*"([^"]+)"/.exec(text)?.[1] ?? + /Actual:\s*\{[^}]*"node"\s*:\s*"([^"]+)"/.exec(text)?.[1] ?? + null; + return { ebadengine, required, current }; +} + +function consumerEnv(options) { + const nodeBin = dirname(options.nodePath); + const path = `${nodeBin}:/usr/bin:/bin`; + const env = { + PATH: path, + HOME: options.consumerDir, + TMPDIR: process.env['TMPDIR'] ?? tmpdir(), + LANG: process.env['LANG'] ?? 'C', + npm_config_ignore_scripts: 'false', + npm_config_userconfig: join(options.consumerDir, '.npmrc'), + npm_config_globalconfig: join(options.consumerDir, '.npmrc-global'), + npm_config_cache: join(options.consumerDir, '.npm-cache'), + npm_config_update_notifier: 'false', + npm_config_fund_scripts: '', + npm_config_engine_strict: options.engineStrict ? 'true' : '', + }; + return env; +} + +async function installTarball(options) { + const args = ['install']; + if (options.engineStrict) { + args.push('--engine-strict'); + } + args.push(options.tarball); + await writeFile(join(options.consumerDir, '.npmrc'), ''); + await writeFile(join(options.consumerDir, '.npmrc-global'), ''); + log( + `installing ${options.tarball} with ${options.npmPath} (scripts enabled${ + options.engineStrict ? ', engine-strict' : '' + })` + ); + try { + const { stdout, stderr } = await execFileAsync(options.npmPath, args, { + cwd: options.consumerDir, + env: consumerEnv(options), + }); + if (stdout.trim() !== '') log(stdout.trim()); + if (stderr.trim() !== '') log(stderr.trim()); + return { + exitCode: 0, + stdout, + stderr, + combined: `${stdout}\n${stderr}`, + scriptsEnabled: true, + engineStrict: options.engineStrict === true, + }; + } catch (error) { + const stdout = + isRecord(error) && typeof error.stdout === 'string' ? error.stdout : ''; + const stderr = + isRecord(error) && typeof error.stderr === 'string' ? error.stderr : ''; + const exitCode = + isRecord(error) && typeof error.code === 'number' ? error.code : 1; + if (stdout.trim() !== '') log(stdout.trim()); + if (stderr.trim() !== '') log(stderr.trim()); + return { + exitCode, + stdout, + stderr, + combined: `${stdout}\n${stderr}`, + scriptsEnabled: true, + engineStrict: options.engineStrict === true, + }; + } +} + +async function runProbe(nodePath, consumerDir, repoRoot) { + const probeSource = join(repoRoot, probeRelativePath); + const probeDest = join(consumerDir, 'probe.mjs'); + await copyFile(probeSource, probeDest); + log(`running consumer probe with ${nodePath}`); + const { stdout, stderr } = await execFileAsync(nodePath, [probeDest], { + cwd: consumerDir, + }); + if (stderr.trim() !== '') log(stderr.trim()); + const parsed = JSON.parse(stdout); + if (!isRecord(parsed) || parsed.ok !== true) { + throw new Error(`consumer probe failed: ${stdout}`); + } + return parsed; +} + +async function runHappy(options) { + const workspace = await mkdtemp(join(tmpdir(), 'kit-0.3.0-clean-consumer-')); + try { + const consumerDir = await createConsumerDir(workspace); + const nodeVersion = await readNodeVersion(options.nodePath); + const install = await installTarball({ + nodePath: options.nodePath, + npmPath: options.npmPath, + consumerDir, + tarball: options.tarball.path, + }); + if (install.exitCode !== 0) { + throw new Error( + `clean consumer install failed under ${nodeVersion} with exit ${String(install.exitCode)}` + ); + } + const probe = await runProbe( + options.nodePath, + consumerDir, + options.repoRoot + ); + return { + ok: true, + mode: 'happy', + node: { + version: nodeVersion, + execPath: options.nodePath, + npmPath: options.npmPath, + }, + tarball: { + filename: options.tarball.filename, + path: options.tarball.path, + sha512: options.tarball.sha512, + files: options.tarball.files, + byteLength: options.tarball.byteLength, + unpackedSize: options.tarball.unpackedSize, + size: options.tarball.size, + version: options.tarball.version, + }, + install: { + exitCode: install.exitCode, + scriptsEnabled: true, + ignoreScripts: false, + workspaceLinked: false, + fileSourceTree: false, + stdout: install.stdout, + stderr: install.stderr, + }, + probe, + }; + } finally { + if (!options.keep) { + await rm(workspace, { recursive: true, force: true }); + } else { + log(`kept workspace ${workspace}`); + } + } +} + +async function runEngineMismatch(options) { + const workspace = await mkdtemp(join(tmpdir(), 'kit-0.3.0-engine-mismatch-')); + try { + const nodeVersion = await readNodeVersion(options.nodePath); + const major = Number(nodeVersion.slice(1).split('.')[0]); + if (major >= 22) { + throw new Error( + `engine-mismatch mode requires Node 20, received ${nodeVersion}` + ); + } + const warnedDir = await createConsumerDir(join(workspace, 'warned')); + const warned = await installTarball({ + nodePath: options.nodePath, + npmPath: options.npmPath, + consumerDir: warnedDir, + tarball: options.tarball.path, + }); + const warnedSignals = collectEngineSignals(warned.combined); + const strictDir = await createConsumerDir(join(workspace, 'strict')); + const strict = await installTarball({ + nodePath: options.nodePath, + npmPath: options.npmPath, + consumerDir: strictDir, + tarball: options.tarball.path, + engineStrict: true, + }); + const strictSignals = collectEngineSignals(strict.combined); + const surfaced = + warnedSignals.ebadengine || + strictSignals.ebadengine || + strict.exitCode !== 0; + if (!surfaced) { + throw new Error( + `Node ${nodeVersion} install did not surface engines.node mismatch` + ); + } + if (strict.exitCode === 0) { + throw new Error( + 'engine-strict install succeeded; Node 20 was treated as supported' + ); + } + return { + ok: true, + mode: 'engine-mismatch', + node: { + version: nodeVersion, + execPath: options.nodePath, + npmPath: options.npmPath, + }, + tarball: { + filename: options.tarball.filename, + path: options.tarball.path, + sha512: options.tarball.sha512, + files: options.tarball.files, + byteLength: options.tarball.byteLength, + unpackedSize: options.tarball.unpackedSize, + size: options.tarball.size, + version: options.tarball.version, + }, + requiredEngine: '>=22.0.0', + treatedAsSupported: false, + warningInstall: { + exitCode: warned.exitCode, + ebadengine: warnedSignals.ebadengine, + required: warnedSignals.required, + current: warnedSignals.current, + stdout: warned.stdout, + stderr: warned.stderr, + }, + engineStrictInstall: { + exitCode: strict.exitCode, + ebadengine: strictSignals.ebadengine, + required: strictSignals.required, + current: strictSignals.current, + stdout: strict.stdout, + stderr: strict.stderr, + }, + }; + } finally { + if (!options.keep) { + await rm(workspace, { recursive: true, force: true }); + } else { + log(`kept workspace ${workspace}`); + } + } +} + +async function verifyOfficialChecksum(version, tarName, tarPath) { + const { stdout } = await execFileAsync('curl', [ + '-fsSL', + `https://nodejs.org/dist/v${version}/SHASUMS256.txt`, + ]); + const line = stdout.split('\n').find(entry => entry.endsWith(tarName)); + if (line === undefined) { + throw new Error(`SHASUMS256.txt has no entry for ${tarName}`); + } + const expected = line.split(/\s+/)[0]; + const actual = createHash('sha256').update(await readFile(tarPath)).digest('hex'); + if (actual !== expected) { + throw new Error(`sha256 mismatch for ${tarName}`); + } +} + +async function ensureOfficialNode(cacheDir, version) { + const triple = platformTriple(); + const unpacked = join(cacheDir, `node-v${version}-${triple}`); + const execPath = join(unpacked, 'bin', 'node'); + const npmPath = join(unpacked, 'bin', 'npm'); + try { + await access(execPath); + await access(npmPath); + return { version, execPath, npmPath }; + } catch { + // download below + } + await mkdir(cacheDir, { recursive: true }); + const tarName = `node-v${version}-${triple}.tar.gz`; + const tarPath = join(cacheDir, tarName); + const url = `https://nodejs.org/dist/v${version}/${tarName}`; + log(`downloading official Node ${version} from ${url}`); + await execFileAsync('curl', ['-fsSL', url, '-o', tarPath]); + await verifyOfficialChecksum(version, tarName, tarPath); + await execFileAsync('tar', ['-xzf', tarPath, '-C', cacheDir]); + await access(execPath); + return { version, execPath, npmPath }; +} + +async function resolveTarball(args, packDir) { + if (args.tarball !== undefined) { + return inspectTarball(args.tarball); + } + return packCandidate(args.repo, packDir); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const packDir = await mkdtemp(join(tmpdir(), 'kit-0.3.0-pack-')); + try { + if (args.mode === 'matrix') { + const cacheDir = join(args.repo, '.cache', 'node'); + const node20 = await ensureOfficialNode(cacheDir, OFFICIAL_NODES[20]); + const node22 = await ensureOfficialNode(cacheDir, OFFICIAL_NODES[22]); + const node24 = await ensureOfficialNode(cacheDir, OFFICIAL_NODES[24]); + const tarball = await resolveTarball(args, packDir); + const happy22 = await runHappy({ + nodePath: node22.execPath, + npmPath: node22.npmPath, + tarball, + repoRoot: args.repo, + keep: args.keep, + }); + const happy24 = await runHappy({ + nodePath: node24.execPath, + npmPath: node24.npmPath, + tarball, + repoRoot: args.repo, + keep: args.keep, + }); + const mismatch20 = await runEngineMismatch({ + nodePath: node20.execPath, + npmPath: node20.npmPath, + tarball, + keep: args.keep, + }); + const report = { + ok: happy22.ok && happy24.ok && mismatch20.ok, + mode: 'matrix', + tarball: happy22.tarball, + node22: happy22, + node24: happy24, + node20: mismatch20, + officialNodes: OFFICIAL_NODES, + }; + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + return; + } + + const nodePath = args.node ?? process.execPath; + const npmPath = npmForNode(nodePath); + const tarball = await resolveTarball(args, packDir); + const report = + args.mode === 'engine-mismatch' + ? await runEngineMismatch({ + nodePath, + npmPath, + tarball, + keep: args.keep, + }) + : await runHappy({ + nodePath, + npmPath, + tarball, + repoRoot: args.repo, + keep: args.keep, + }); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } finally { + await rm(packDir, { recursive: true, force: true }); + } +} + +try { + await main(); +} catch (error) { + const message = error instanceof Error ? error.stack ?? error.message : String(error); + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/scripts/senpi-compatibility-probe.mts b/scripts/senpi-compatibility-probe.mts new file mode 100644 index 0000000..091d502 --- /dev/null +++ b/scripts/senpi-compatibility-probe.mts @@ -0,0 +1,521 @@ +#!/usr/bin/env -S pnpm exec tsx +/** + * Fixture-first, non-mutating Pi/Senpi compatibility probe. + * + * This records library findings against the frozen future-harness checklist. + * It is not an integration approval and does not expose a harness ABI. + */ +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { + encodeSenpiCwdDirname, + foldSenpiBlockChanges, + listAllSenpiSessions, + listSenpiSessions, + tailSenpiSession, +} from '../src/senpi/processing/index.js'; +import { getSenpiSessionMarkerPath } from '../src/senpi/processing/checkpoint.js'; +import { + assertSenpiSmokeProbeInputsUnchanged, + hashSenpiSmokeProbeInputs, + type SenpiSmokeProbeHash, +} from './senpi-smoke-safety.js'; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const defaultDescriptor = join( + repoRoot, + 'tests/fixtures/senpi/compatibility-probe-v1.json' +); + +interface ProbeDescriptor { + readonly schemaVersion: 1; + readonly subject: string; + readonly evaluatedAt: string; + readonly checkpointMode: 'automatic' | 'manual'; + readonly sessionFixture: string; + readonly rewriteFixture: string; +} + +interface ProtectedPath { + readonly label: string; + readonly path: string; +} + +export class SenpiCompatibilityMutationGuardError extends Error { + constructor(message: string) { + super(message); + this.name = 'SenpiCompatibilityMutationGuardError'; + } +} + +function loadDescriptor(path: string): ProbeDescriptor { + const raw: unknown = JSON.parse(readFileSync(path, 'utf8')); + if ( + typeof raw !== 'object' || + raw === null || + !('schemaVersion' in raw) || + raw.schemaVersion !== 1 || + !('subject' in raw) || + typeof raw.subject !== 'string' || + !('evaluatedAt' in raw) || + typeof raw.evaluatedAt !== 'string' || + !('checkpointMode' in raw) || + (raw.checkpointMode !== 'manual' && raw.checkpointMode !== 'automatic') || + !('sessionFixture' in raw) || + typeof raw.sessionFixture !== 'string' || + !('rewriteFixture' in raw) || + typeof raw.rewriteFixture !== 'string' + ) { + throw new Error(`Invalid compatibility probe descriptor: ${path}`); + } + return raw as ProbeDescriptor; +} + +function assertManualCheckpointMode(descriptor: ProbeDescriptor): void { + if (descriptor.checkpointMode !== 'manual') { + throw new SenpiCompatibilityMutationGuardError( + 'Mutation guard rejected checkpointMode=automatic; compatibility probes require manual checkpointing' + ); + } +} + +async function hashProtected( + paths: readonly ProtectedPath[] +): Promise { + return hashSenpiSmokeProbeInputs(paths.map(entry => entry.path)); +} + +function stableHashes( + paths: readonly ProtectedPath[], + hashes: readonly SenpiSmokeProbeHash[] +): Record { + return Object.fromEntries( + paths.map((entry, index) => { + const digest = hashes[index]?.digest; + return [ + entry.label, + digest?.startsWith('missing:') === true + ? 'missing' + : (digest ?? 'missing'), + ]; + }) + ); +} + +async function readHeaderCwd(path: string): Promise { + const contents = await readFile(path, 'utf8'); + const first = contents.split('\n', 1)[0]; + if (first === undefined || first === '') { + throw new Error(`Session fixture has no header: ${path}`); + } + const header: unknown = JSON.parse(first); + if ( + typeof header !== 'object' || + header === null || + !('cwd' in header) || + typeof header.cwd !== 'string' + ) { + throw new Error(`Session fixture has no cwd: ${path}`); + } + return header.cwd; +} + +function liveProtectedPaths(sessionPath: string, cwd: string): ProtectedPath[] { + const markerPath = getSenpiSessionMarkerPath(sessionPath); + const sessionsSegment = `${join('agent', 'sessions')}`; + const marker = sessionPath.lastIndexOf(sessionsSegment); + const agentHome = + marker === -1 + ? dirname(dirname(sessionPath)) + : sessionPath.slice(0, marker + 'agent'.length); + return [ + { label: 'live-source', path: sessionPath }, + { label: 'live-marker', path: markerPath }, + { label: 'live-config', path: join(agentHome, 'settings.json') }, + { label: 'live-agent-hooks', path: join(agentHome, 'hooks.json') }, + { label: 'live-project-hooks', path: join(cwd, '.senpi', 'hooks.json') }, + { + label: 'live-project-trust', + path: join(cwd, '.senpi', 'hooks-state.json'), + }, + ]; +} + +async function runProbe(options: { + readonly descriptorPath: string; + readonly liveSession?: string; +}): Promise> { + const descriptor = loadDescriptor(options.descriptorPath); + assertManualCheckpointMode(descriptor); + + const fixtureSource = resolve(repoRoot, descriptor.sessionFixture); + const rewriteFixture = resolve(repoRoot, descriptor.rewriteFixture); + const source = + options.liveSession === undefined + ? fixtureSource + : resolve(options.liveSession); + const sourceCwd = await readHeaderCwd(source); + const livePaths = + options.liveSession === undefined + ? [] + : liveProtectedPaths(source, sourceCwd); + const liveBefore = await hashProtected(livePaths); + + const scratch = await mkdtemp(join(tmpdir(), 't29-pi-senpi-probe-')); + try { + const agentHome = join(scratch, 'agent'); + const projectCwd = sourceCwd; + const sessionDir = join( + agentHome, + 'sessions', + encodeSenpiCwdDirname(projectCwd) + ); + const sessionPath = join(sessionDir, 'probe-session.jsonl'); + const markerPath = getSenpiSessionMarkerPath(sessionPath, { + markerDir: join(scratch, 'markers'), + allowedMarkerRoots: [scratch], + }); + const configPath = join(agentHome, 'settings.json'); + const agentHooksPath = join(agentHome, 'hooks.json'); + const projectHooksPath = join(scratch, 'project', '.senpi', 'hooks.json'); + const trustPath = join(scratch, 'project', '.senpi', 'hooks-state.json'); + + await mkdir(sessionDir, { recursive: true }); + await mkdir(dirname(markerPath), { recursive: true }); + await mkdir(dirname(projectHooksPath), { recursive: true }); + await copyFile(source, sessionPath); + await writeFile(markerPath, '{}\n'); + await writeFile(configPath, '{"fixture":true}\n'); + await writeFile(agentHooksPath, '{"hooks":{}}\n'); + await writeFile(projectHooksPath, '{"hooks":{}}\n'); + await writeFile(trustPath, '{"version":1,"hooks":{}}\n'); + + const protectedPaths: ProtectedPath[] = [ + { label: 'source', path: source }, + { label: 'copied-transcript', path: sessionPath }, + { label: 'marker', path: markerPath }, + { label: 'config', path: configPath }, + { label: 'agent-hooks', path: agentHooksPath }, + { label: 'project-hooks', path: projectHooksPath }, + { label: 'trust', path: trustPath }, + ]; + const before = await hashProtected(protectedPaths); + + const all = await listAllSenpiSessions({ agentHome }); + const scoped = await listSenpiSessions(projectCwd, { agentHome }); + const valid = scoped.filter(entry => entry.kind === 'valid'); + if (valid.length !== 1) { + throw new Error( + `Expected one scoped fixture session, found ${valid.length}` + ); + } + + const first = await tailSenpiSession(sessionPath, { + checkpointMode: 'manual', + markerDir: dirname(markerPath), + allowedMarkerRoots: [scratch], + includeOffPath: true, + }); + const second = await tailSenpiSession(sessionPath, { + checkpointMode: 'manual', + checkpoint: first.checkpoint, + markerDir: dirname(markerPath), + allowedMarkerRoots: [scratch], + includeOffPath: true, + }); + const blocks = foldSenpiBlockChanges(first.changes); + + // Rewrite detection is exercised only on an expendable simulation copy; + // the copied probe transcript above remains protected and byte-identical. + const rewritePath = join(scratch, 'rewrite-simulation.jsonl'); + await copyFile(source, rewritePath); + const rewriteInitial = await tailSenpiSession(rewritePath, { + checkpointMode: 'manual', + }); + await copyFile(rewriteFixture, rewritePath); + const rewriteResult = await tailSenpiSession(rewritePath, { + checkpointMode: 'manual', + checkpoint: rewriteInitial.checkpoint, + }); + + const after = await hashProtected(protectedPaths); + assertSenpiSmokeProbeInputsUnchanged(before, after); + const liveAfter = await hashProtected(livePaths); + assertSenpiSmokeProbeInputsUnchanged(liveBefore, liveAfter); + + const packageJson = JSON.parse( + await readFile(join(repoRoot, 'package.json'), 'utf8') + ) as { + exports?: Record; + engines?: { node?: string }; + }; + const processingModule = + await import('@libar-dev/agent-harness-kit/senpi/processing'); + const senpiModule = await import('@libar-dev/agent-harness-kit/senpi'); + const exportsListed = ['./senpi', './senpi/processing'].filter( + subpath => packageJson.exports?.[subpath] !== undefined + ); + const dynamicImportsLoaded = + typeof processingModule.tailSenpiSession === 'function' && + typeof senpiModule.resolveSenpiAgentHome === 'function'; + const nodeMajor = Number.parseInt( + process.versions.node.split('.')[0] ?? '0', + 10 + ); + const hasProvenance = blocks.every( + block => block.entryId !== '' && block.origin !== undefined + ); + const rewriteDetected = + rewriteResult.reset && + rewriteResult.mutations.some(mutation => mutation.index === 0); + const forkDetected = first.offPath.length > 0; + const incrementalTail = + second.previousByteOffset === first.nextByteOffset && + second.nextByteOffset === first.nextByteOffset && + second.mutations.length === 0; + + let candidateSha = 'unknown'; + try { + candidateSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repoRoot, + encoding: 'utf8', + }).trim(); + } catch { + // Codeload/tarball installs have no .git; keep the probe runnable and + // degrade only the SHA label used for report metadata. + } + + return { + schemaVersion: 1, + subject: descriptor.subject, + candidateSha, + evaluatedAt: descriptor.evaluatedAt, + gates: { + A: [ + row( + 'A-01', + 'FAIL', + { + identity: { + scheme: 'native-id', + collisionPolicy: 'none', + claudeIdsByteIdentical: true, + }, + }.identity + ), + row( + 'A-02', + 'FAIL', + { + projectScoping: { + encodingFunction: 'encodeSenpiCwdDirname', + roundTrips: false, + perProjectListing: valid.length === 1, + }, + }.projectScoping + ), + row( + 'A-03', + 'FAIL', + { + discovery: { + bounded: true, + cancellable: false, + progressSignal: null, + }, + }.discovery + ), + row( + 'A-04', + incrementalTail ? 'PASS' : 'FAIL', + { + checkpointing: { + incrementalTail, + mode: 'manual', + commitsAfterDurableApply: true, + staleConflictTyped: + typeof processingModule.StaleCheckpointConflict === + 'function', + }, + }.checkpointing + ), + row( + 'A-05', + rewriteDetected && forkDetected ? 'PASS' : 'FAIL', + { + resetFork: { + detectsRewrite: rewriteDetected, + emitsMutationEvents: + rewriteResult.mutations.length > 0 && + first.mutations.length > 0, + modelDocumented: + 'docs/reference/senpi-adapter.md#session-v3-tree-projection-tail-and-checkpoints', + }, + }.resetFork + ), + row( + 'A-06', + 'FAIL', + { + normalization: { + lossyByDesign: false, + fallbackKind: 'native-metadata', + provenancePersisted: false, + }, + }.normalization + ), + row( + 'A-07', + exportsListed.length === 2 && + dynamicImportsLoaded && + nodeMajor >= 22 + ? 'PASS' + : 'FAIL', + { + package: { + exportsListed, + esmDynamicImport: dynamicImportsLoaded, + nodeRangeSatisfied: + packageJson.engines?.node === '>=22.0.0' && nodeMajor >= 22, + }, + }.package + ), + row( + 'A-08', + exportsListed.length === 2 ? 'PASS' : 'FAIL', + { + cleanConsumer: { + importsResolve: exportsListed.length === 2, + requiresPatching: false, + }, + }.cleanConsumer + ), + row( + 'A-09', + 'FAIL', + { + performance: { + watchDefaultsDocumented: true, + fallbackBounded: false, + noPerBlockListEvents: true, + }, + }.performance + ), + row( + 'A-10', + 'PASS', + { + persistence: { + versionedParser: true, + observePathWritesNativeStore: false, + }, + }.persistence + ), + ], + B: [], + }, + gateAResult: 'FAIL', + gateBResult: 'NOT_EVALUATED', + overall: 'rejected', + probe: { + reportVersion: 1, + sourceKind: + options.liveSession === undefined + ? 'sanitized-fixture' + : 'read-only-live-copy', + manualCheckpointForced: true, + autoCheckpointRejected: true, + hashesUnchanged: true, + protectedHashesBefore: stableHashes(protectedPaths, before), + protectedHashesAfter: stableHashes(protectedPaths, after), + liveSourceHashesBefore: stableHashes(livePaths, liveBefore), + liveSourceHashesAfter: stableHashes(livePaths, liveAfter), + discovery: { + allCount: all.length, + scopedCount: scoped.length, + nativeIdentity: valid[0]?.info.id ?? null, + canonicalIdentityAvailable: false, + collisionQuarantineAvailable: false, + }, + incrementalTail: { + verified: incrementalTail, + firstOffset: first.nextByteOffset, + secondOffset: second.nextByteOffset, + }, + resetFork: { + rewriteDetected, + forkDetected, + offPathCount: first.offPath.length, + }, + provenance: { + availableOnNativeBlocks: hasProvenance, + persistedForCockpitTranscriptBlocks: false, + }, + productApproval: { + ownerApproval: false, + blessedPi: false, + integrationReady: false, + }, + }, + }; + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +function row( + id: string, + verdict: 'PASS' | 'FAIL' | 'N/A', + evidence: Record +): Record { + return { id, verdict, evidence, justification: '' }; +} + +async function main(): Promise { + const { values } = parseArgs({ + options: { + descriptor: { type: 'string', default: defaultDescriptor }, + 'live-session': { type: 'string' }, + output: { type: 'string' }, + }, + }); + const report = await runProbe({ + descriptorPath: resolve(values.descriptor), + ...(values['live-session'] === undefined + ? {} + : { liveSession: values['live-session'] }), + }); + const json = `${JSON.stringify(report, null, 2)}\n`; + if (values.output === undefined) { + process.stdout.write(json); + } else { + const output = resolve(values.output); + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, json); + } +} + +if ( + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + main().catch((error: unknown) => { + console.error( + error instanceof Error ? `${error.name}: ${error.message}` : String(error) + ); + process.exitCode = 1; + }); +} diff --git a/scripts/senpi-live-smoke.mts b/scripts/senpi-live-smoke.mts new file mode 100644 index 0000000..eca06a8 --- /dev/null +++ b/scripts/senpi-live-smoke.mts @@ -0,0 +1,110 @@ +#!/usr/bin/env -S pnpm exec tsx +/** + * Live smoke probe for the senpi adapter against a real OmO agent store. + * + * Read-only by contract: resolves the agent home, lists sessions, tails one + * session file in manual checkpoint mode, and folds its block change stream. + * Never writes tail markers, trust, hooks, or config. Use it to answer "does + * the adapter see what the running harness sees?" without touching the store. + * + * Usage: + * pnpm senpi:smoke # probe $PI_SESSION_FILE (the live session) + * pnpm senpi:smoke # probe an explicit session file + * pnpm senpi:smoke --project # scope listing to one project cwd + */ +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { resolveSenpiAgentHome } from '../src/senpi/home.js'; +import { getSenpiSessionsRoot } from '../src/senpi/processing/discovery.js'; +import { + listAllSenpiSessions, + listSenpiSessions, +} from '../src/senpi/processing/listing.js'; +import { getSenpiSessionMarkerPath } from '../src/senpi/processing/checkpoint.js'; +import { tailSenpiSession } from '../src/senpi/processing/tail.js'; +import { foldSenpiBlockChanges } from '../src/senpi/processing/blocks.js'; +import { + assertSenpiSmokeProbeInputsUnchanged, + hashSenpiSmokeProbeInputs, +} from './senpi-smoke-safety.js'; + +const { values } = parseArgs({ + options: { + project: { type: 'string' }, + }, + allowPositionals: true, +}); + +const [sessionArg] = values.project === undefined ? process.argv.slice(2) : []; +const sessionFile = sessionArg ?? process.env.PI_SESSION_FILE; +if (sessionFile === undefined || sessionFile === '') { + console.error( + 'No session file: pass a path or run inside OmO with PI_SESSION_FILE set.' + ); + process.exit(1); +} + +const home = resolveSenpiAgentHome(); +console.log('agentHome:', home); +console.log('sessionsRoot:', getSenpiSessionsRoot(home)); + +const all = await listAllSenpiSessions({ agentHome: home }); +const valid = all.filter(entry => entry.kind === 'valid'); +const invalid = all.filter(entry => entry.kind !== 'valid'); +console.log( + `listAllSenpiSessions: ${valid.length} valid, ${invalid.length} invalid` +); +for (const bad of invalid.slice(0, 5)) { + console.log( + ' invalid:', + bad.path.split('/').pop(), + String(bad.error).slice(0, 80) + ); +} + +if (values.project !== undefined) { + const scoped = await listSenpiSessions(values.project, { agentHome: home }); + for (const entry of scoped) { + if (entry.kind === 'valid') { + console.log( + ` ${entry.info.id.slice(0, 8)} msgs=${entry.info.messageCount} first=${JSON.stringify(entry.info.firstMessage?.slice(0, 60))}` + ); + } + } +} + +const probePaths = [ + getSenpiSessionMarkerPath(sessionFile), + join(home, 'settings.json'), + sessionFile, +] as const; +const probeBefore = await hashSenpiSmokeProbeInputs(probePaths); + +const tail = await tailSenpiSession(sessionFile, { checkpointMode: 'manual' }); +console.log( + `tail: records=${tail.records.length} leafKind=${tail.leaf.kind} bytes=${tail.nextByteOffset}/${tail.fileSize} gen=${tail.generation} rev=${tail.revision} reset=${String(tail.reset)} diagnostics=${tail.diagnostics.length}` +); +for (const diagnostic of tail.diagnostics.slice(0, 5)) { + console.log(' diagnostic:', JSON.stringify(diagnostic).slice(0, 120)); +} + +const blocks = foldSenpiBlockChanges(tail.changes); +const byRole = new Map(); +let customTyped = 0; +for (const block of blocks) { + byRole.set(block.role, (byRole.get(block.role) ?? 0) + 1); + if (block.customType !== undefined) customTyped += 1; +} +console.log( + `blocks: ${blocks.length} roles=${JSON.stringify(Object.fromEntries(byRole))} customTyped=${customTyped}` +); + +const second = await tailSenpiSession(sessionFile, { + checkpointMode: 'manual', +}); +console.log( + `re-tail stable: ${second.revision === tail.revision && second.mutations.length === 0 ? 'yes' : 'NO'} (rev=${second.revision}, mutations=${second.mutations.length})` +); + +const probeAfter = await hashSenpiSmokeProbeInputs(probePaths); +assertSenpiSmokeProbeInputsUnchanged(probeBefore, probeAfter); diff --git a/scripts/senpi-smoke-safety.ts b/scripts/senpi-smoke-safety.ts new file mode 100644 index 0000000..6c1cd2c --- /dev/null +++ b/scripts/senpi-smoke-safety.ts @@ -0,0 +1,99 @@ +import { createHash } from 'node:crypto'; +import { readdir, readFile, stat } from 'node:fs/promises'; +import { join } from 'node:path'; + +/** Content digest of one probe path. Missing paths use a stable sentinel. */ +export interface SenpiSmokeProbeHash { + readonly path: string; + readonly digest: string; +} + +/** + * SHA-256 file or directory digest. A missing path hashes to `missing:` + * so a later create is visible. Directory digests cover sorted child names + * and their recursive contents. + * + * @param path - File or directory to hash. + * @returns Hex digest or the missing-path sentinel. + */ +export async function hashSenpiSmokeProbePath(path: string): Promise { + let info; + try { + info = await stat(path); + } catch (error: unknown) { + if (hasErrorCode(error, 'ENOENT')) { + return `missing:${path}`; + } + throw error; + } + if (info.isDirectory()) { + const names = (await readdir(path)).sort(); + const parts = await Promise.all( + names.map(async name => { + const child = join(path, name); + return `${name}:${await hashSenpiSmokeProbePath(child)}`; + }) + ); + return createHash('sha256').update(parts.join('|')).digest('hex'); + } + return createHash('sha256') + .update(await readFile(path)) + .digest('hex'); +} + +/** + * Snapshot marker, config, and transcript hashes for a smoke probe. + * + * @param paths - Absolute paths that must stay byte-identical. + * @returns One digest per path, in input order. + */ +export async function hashSenpiSmokeProbeInputs( + paths: readonly string[] +): Promise { + return Promise.all( + paths.map(async path => ({ + path, + digest: await hashSenpiSmokeProbePath(path), + })) + ); +} + +/** + * Compare two probe snapshots and throw when any digest changed. + * + * @param before - Hashes captured before the probe. + * @param after - Hashes captured after the probe. + * @throws If any path is new, missing, or has a different digest. + */ +export function assertSenpiSmokeProbeInputsUnchanged( + before: readonly SenpiSmokeProbeHash[], + after: readonly SenpiSmokeProbeHash[] +): void { + if (before.length !== after.length) { + throw new Error( + `Smoke probe path count changed: before=${before.length} after=${after.length}` + ); + } + const changed: string[] = []; + for (const [index, previous] of before.entries()) { + const next = after[index]; + if (next === undefined || next.path !== previous.path) { + throw new Error('Smoke probe path set changed'); + } + if (next.digest !== previous.digest) { + changed.push(previous.path); + } + } + if (changed.length > 0) { + throw new Error(`Smoke probe inputs mutated: ${changed.join(', ')}`); + } +} + +function hasErrorCode(error: unknown, code: string): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === code + ); +} diff --git a/scripts/sync-upstream-senpi.mjs b/scripts/sync-upstream-senpi.mjs new file mode 100644 index 0000000..138f608 --- /dev/null +++ b/scripts/sync-upstream-senpi.mjs @@ -0,0 +1,469 @@ +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(scriptDir, '..'); +const vendorDir = resolve(repoRoot, 'docs', 'upstream', 'senpi'); +const pinPath = resolve(vendorDir, 'pin.json'); + +const engineVersion = '2026.8.19'; +const pinnedAt = '2026-08-21'; +const source = 'npm'; + +/** @type {ReadonlyArray<{ localName: string, upstreamPath: string }>} */ +const sourceFiles = [ + { localName: 'session-format.md', upstreamPath: 'docs/session-format.md' }, + { localName: 'settings.md', upstreamPath: 'docs/settings.md' }, + { + localName: 'environment-variables.md', + upstreamPath: 'docs/environment-variables.md', + }, + ...[ + 'types', + 'index', + 'schema', + 'trust', + 'trust-storage', + 'config-loader', + 'command-runner', + 'dispatcher', + 'output-parser', + 'output-bounds', + 'safety', + 'matcher', + 'command', + 'handler', + 'diagnostics', + 'plugin-loader', + 'plugin-manifest', + 'lifecycle-adapter', + 'tool-adapter', + 'stop-adapter', + 'prompt-adapter', + ].map((name) => ({ + localName: `hooks/${name}.d.ts`, + upstreamPath: `dist/core/extensions/builtin/hooks/${name}.d.ts`, + })), + ...['trust', 'output-parser', 'trust-storage', 'output-bounds', 'types'].map( + (name) => ({ + localName: `hooks/${name}.js`, + upstreamPath: `dist/core/extensions/builtin/hooks/${name}.js`, + }), + ), +]; + +const notes = [ + 'Artifact of record is the npm tarball for @code-yeongyu/senpi@2026.8.19; registryIntegrity is npm view dist.integrity (sha512).', + 'Vendored set is complete at pin time: 3 docs, 21 hooks .d.ts, and 5 hooks .js implementations required because .d.ts signatures alone are insufficient (hashCommandHook, output parsing, trust storage lock paths, output bounds, types constants).', + 'No runtime import of @code-yeongyu/senpi; drift checks re-extract from a provided local tarball only (no network at check time).', + 'Upstream package license is MIT (code-yeongyu/senpi).', +]; + +const fixtureRedump = + 'copy small redacted *.jsonl from ~/.omo/agent/sessions// into tests/fixtures/senpi/'; + +/** + * Print CLI usage to stdout. + * @returns {void} + */ +export function printUsage() { + console.log(`Sync vendored Senpi contract files from an npm tarball. + +Usage: + node scripts/sync-upstream-senpi.mjs --tarball [--check] + +Options: + --tarball Path to code-yeongyu-senpi-*.tgz (required; no network fetch). + --check Verify the vendor and pin manifest without writing files. + --help Show this help. +`); +} + +/** + * Parse CLI arguments for the senpi upstream sync script. + * @param {string[]} args + * @returns {{ tarballPath: string, check: boolean } | null} + */ +export function parseArgs(args) { + let tarballPath = null; + let check = false; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === '--check') { + check = true; + } else if (arg === '--help' || arg === '-h') { + printUsage(); + return null; + } else if (arg === '--tarball') { + const value = args[index + 1]; + if (!value || value.startsWith('-')) { + throw new Error('--tarball requires a path argument'); + } + tarballPath = resolve(value); + index += 1; + } else if (arg.startsWith('--tarball=')) { + tarballPath = resolve(arg.slice('--tarball='.length)); + } else if (arg.startsWith('-')) { + throw new Error(`Unknown option: ${arg}`); + } else { + throw new Error(`Unexpected positional argument: ${arg}`); + } + } + + if (!tarballPath) { + throw new Error('--tarball is required'); + } + + return { tarballPath, check }; +} + +/** + * Compute lowercase hex SHA-256 of a buffer. + * @param {Buffer} bytes + * @returns {string} + */ +export function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +/** + * Constant-time-ish buffer equality (length then byte compare). + * @param {Buffer | null} left + * @param {Buffer | null} right + * @returns {boolean} + */ +export function bytesEqual(left, right) { + if (!left || !right) { + return left === right; + } + if (left.length !== right.length) { + return false; + } + return left.equals(right); +} + +/** + * Read an existing vendored file as a Buffer, or null if missing. + * @param {string} localName + * @returns {Buffer | null} + */ +export function readExisting(localName) { + const path = resolve(vendorDir, localName); + if (!existsSync(path)) { + return null; + } + return readFileSync(path); +} + +/** + * Read and parse the pinned pin.json manifest, or null if absent. + * @returns {Record | null} + */ +export function readPinnedManifest() { + if (!existsSync(pinPath)) { + return null; + } + + try { + return JSON.parse(readFileSync(pinPath, 'utf8')); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Could not parse ${pinPath}: ${detail}`); + } +} + +/** + * List relative file paths under a directory (files only, recursive). + * @param {string} rootDir + * @returns {string[]} + */ +export function listFilesRecursive(rootDir) { + /** @type {string[]} */ + const out = []; + + /** + * @param {string} dir + * @returns {void} + */ + function walk(dir) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + walk(full); + } else if (entry.isFile()) { + out.push(relative(rootDir, full).split(sep).join('/')); + } + } + } + + if (existsSync(rootDir)) { + walk(rootDir); + } + return out.sort(); +} + +/** + * Extract the pinned source file set from a local npm tarball into destDir. + * Paths are written with package/ stripped (docs/... and hooks/... layout). + * @param {string} tarballPath + * @param {string} destDir + * @returns {Map} + */ +export function extractFromTarball(tarballPath, destDir) { + if (!existsSync(tarballPath)) { + throw new Error(`Tarball does not exist: ${tarballPath}`); + } + + const extractRoot = mkdtempSync(join(tmpdir(), 'senpi-upstream-extract-')); + try { + const tarArgs = [ + '-xzf', + tarballPath, + '-C', + extractRoot, + ...sourceFiles.map((source) => `package/${source.upstreamPath}`), + ]; + execFileSync('tar', tarArgs, { stdio: ['ignore', 'pipe', 'pipe'] }); + + /** @type {Map} */ + const expected = new Map(); + mkdirSync(destDir, { recursive: true }); + + for (const source of sourceFiles) { + const extractedPath = join(extractRoot, 'package', source.upstreamPath); + if (!existsSync(extractedPath)) { + throw new Error( + `Missing upstream path in tarball: package/${source.upstreamPath}`, + ); + } + const bytes = readFileSync(extractedPath); + const outPath = join(destDir, source.localName); + mkdirSync(dirname(outPath), { recursive: true }); + writeFileSync(outPath, bytes); + expected.set(source.localName, bytes); + } + + return expected; + } finally { + rmSync(extractRoot, { recursive: true, force: true }); + } +} + +/** + * Build the pin.json object for the current expected file set. + * @param {{ registryIntegrity: string, expected: Map }} params + * @returns {Record} + */ +export function createPin({ registryIntegrity, expected }) { + /** @type {Record} */ + const files = {}; + for (const source of sourceFiles) { + const bytes = expected.get(source.localName); + if (!bytes) { + throw new Error(`Missing expected bytes for ${source.localName}`); + } + files[source.localName] = { + upstreamPath: source.upstreamPath, + sha256: sha256(bytes), + }; + } + + return { + engineVersion, + pinnedAt, + source, + registryIntegrity, + files, + fixtureRedump, + notes, + }; +} + +/** + * Serialize pin.json with trailing newline (stable bytes). + * @param {Record} pin + * @returns {Buffer} + */ +export function pinBytes(pin) { + return Buffer.from(`${JSON.stringify(pin, null, 2)}\n`, 'utf8'); +} + +/** + * Resolve registryIntegrity for a rewrite: keep existing pin value when present. + * @param {Record | null} existingPin + * @returns {string} + */ +export function resolveRegistryIntegrity(existingPin) { + const value = existingPin?.registryIntegrity; + if (typeof value === 'string' && value.length > 0) { + return value; + } + throw new Error( + 'pin.json is missing registryIntegrity; re-run an initial vendor with npm view dist.integrity recorded', + ); +} + +/** + * Compare vendored tree + pin against expected tarball extraction. + * @param {{ + * expected: Map, + * expectedPinBytes: Buffer, + * check: boolean, + * }} params + * @returns {{ drifted: string[], entries: Array<{ path: string, status: string }> }} + */ +export function compareVendor({ expected, expectedPinBytes, check }) { + /** @type {Array<{ path: string, status: string }>} */ + const entries = []; + /** @type {string[]} */ + const drifted = []; + + const expectedNames = new Set(sourceFiles.map((source) => source.localName)); + + for (const source of sourceFiles) { + const actual = readExisting(source.localName); + const desired = expected.get(source.localName) ?? null; + const path = `docs/upstream/senpi/${source.localName}`; + let status; + if (bytesEqual(actual, desired)) { + status = 'unchanged'; + } else if (check) { + status = 'drifted'; + drifted.push(path); + } else if (actual) { + status = 'updated'; + } else { + status = 'added'; + } + entries.push({ path, status }); + } + + // Extra files under vendor (excluding pin.json and NOTICE) are drift. + const onDisk = listFilesRecursive(vendorDir).filter( + (name) => name !== 'pin.json' && name !== 'NOTICE', + ); + for (const name of onDisk) { + if (!expectedNames.has(name)) { + const path = `docs/upstream/senpi/${name}`; + entries.push({ path, status: check ? 'extra' : 'removed' }); + if (check) { + drifted.push(path); + } + } + } + + const actualPin = readExisting('pin.json'); + const pinRel = 'docs/upstream/senpi/pin.json'; + if (bytesEqual(actualPin, expectedPinBytes)) { + entries.push({ path: pinRel, status: 'unchanged' }); + } else if (check) { + entries.push({ path: pinRel, status: 'drifted' }); + drifted.push(pinRel); + } else if (actualPin) { + entries.push({ path: pinRel, status: 'updated' }); + } else { + entries.push({ path: pinRel, status: 'added' }); + } + + return { drifted, entries }; +} + +/** + * Write expected vendored files and pin.json to disk. + * @param {{ expected: Map, expectedPinBytes: Buffer }} params + * @returns {void} + */ +export function writeVendor({ expected, expectedPinBytes }) { + mkdirSync(vendorDir, { recursive: true }); + mkdirSync(resolve(vendorDir, 'hooks'), { recursive: true }); + + // Remove stray vendored files not in the pin set (keep NOTICE). + const expectedNames = new Set(sourceFiles.map((source) => source.localName)); + for (const name of listFilesRecursive(vendorDir)) { + if (name === 'pin.json' || name === 'NOTICE') { + continue; + } + if (!expectedNames.has(name)) { + rmSync(resolve(vendorDir, name), { force: true }); + } + } + + for (const source of sourceFiles) { + const bytes = expected.get(source.localName); + if (!bytes) { + throw new Error(`Missing expected bytes for ${source.localName}`); + } + const outPath = resolve(vendorDir, source.localName); + mkdirSync(dirname(outPath), { recursive: true }); + writeFileSync(outPath, bytes); + } + writeFileSync(pinPath, expectedPinBytes); +} + +/** + * CLI entrypoint. + * @returns {Promise} + */ +export async function main() { + const options = parseArgs(process.argv.slice(2)); + if (!options) { + return; + } + + const existingPin = readPinnedManifest(); + const registryIntegrity = resolveRegistryIntegrity(existingPin); + + const extractDest = mkdtempSync(join(tmpdir(), 'senpi-upstream-vendor-')); + try { + const expected = extractFromTarball(options.tarballPath, extractDest); + const expectedPinBytes = pinBytes( + createPin({ registryIntegrity, expected }), + ); + const { drifted, entries } = compareVendor({ + expected, + expectedPinBytes, + check: options.check, + }); + + if (options.check) { + if (drifted.length > 0) { + for (const file of drifted) { + console.error(`Senpi upstream vendor drift detected: ${file}`); + } + process.exitCode = 1; + return; + } + console.log('Senpi upstream vendor is in sync.'); + return; + } + + writeVendor({ expected, expectedPinBytes }); + console.log('Sync summary:'); + for (const entry of entries) { + console.log(` ${entry.status.padEnd(9)} ${entry.path}`); + } + } finally { + rmSync(extractDest, { recursive: true, force: true }); + } +} + +try { + await main(); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`sync-upstream-senpi: ${message}`); + process.exitCode = 1; +} diff --git a/src/forwarder/assets.ts b/src/forwarder/assets.ts index 9675e2f..f12bab5 100644 --- a/src/forwarder/assets.ts +++ b/src/forwarder/assets.ts @@ -1,3 +1,11 @@ +/** Pack-relative path of the Claude standalone hook forwarder. */ +export const STANDALONE_HOOK_FORWARDER_ASSET = + 'dist/standalone/hook-forwarder.mjs'; + +/** Pack-relative path of the Senpi observe-only standalone hook forwarder. */ +export const STANDALONE_SENPI_HOOK_FORWARDER_ASSET = + 'dist/standalone/hook-forwarder-senpi.mjs'; + /** Shell wrapper installed by endpoint-discovery consumers. */ export const RUN_HOOK_WRAPPER_SH = `#!/bin/sh # libar-cockpit managed hook wrapper (v1). Safe to delete; the app rewrites it. diff --git a/src/forwarder/hook-forwarder-senpi.ts b/src/forwarder/hook-forwarder-senpi.ts new file mode 100644 index 0000000..170c367 --- /dev/null +++ b/src/forwarder/hook-forwarder-senpi.ts @@ -0,0 +1,192 @@ +/** + * Senpi observe-only hook forwarder (standalone asset source). + * + * Modeled on `hook-forwarder.ts`: reads one senpi `HookInputWire` envelope + * from stdin (1 MiB cap) and best-effort POSTs it VERBATIM as JSON to the + * endpoint URL configured through `SENPI_HOOK_FORWARD_URL`. + * + * URL policy: only `http`/`https` URLs whose host is `127.0.0.1`, + * `localhost`, or `::1` are accepted unless + * `SENPI_HOOK_FORWARD_ALLOW_REMOTE=1` unlocks remote hosts. Redirects are + * never followed. + * + * Observe-only guarantee: this script NEVER emits gate or decision JSON to + * stdout and ALWAYS exits 0. Unreachable endpoints, internal timeouts, + * malformed stdin, and every other failure mode are swallowed silently. + * + * Bundled by the package build step (esbuild, `--target=node22`) to + * `dist/standalone/hook-forwarder-senpi.mjs`. + */ + +import { pathToFileURL } from 'node:url'; + +import { readBoundedTimedStdin } from '../internal/stdin.js'; + +/** The seven senpi hook events (vendored contract, engine 2026.8.19). */ +const SENPI_HOOK_EVENTS: ReadonlySet = new Set([ + 'PreToolUse', + 'PostToolUse', + 'UserPromptSubmit', + 'SessionStart', + 'PreCompact', + 'PostCompact', + 'Stop', +]); + +const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_STDIN_MAX_BYTES = 1024 * 1024; +const LOOPBACK_HOSTS: ReadonlySet = new Set([ + '127.0.0.1', + 'localhost', + '::1', +]); + +interface ForwardedEnvelope { + readonly event: string; +} + +/** + * Forward one stdin senpi hook envelope to the configured endpoint. + * + * Observe-only guarantee: resolves without writing anything to stdout and + * without ever throwing; the process entry point always exits 0. + * + * Rejects non-http(s) URLs and, unless `SENPI_HOOK_FORWARD_ALLOW_REMOTE=1`, + * any host other than `127.0.0.1`, `localhost`, or `::1`. Stdin is capped + * at 1 MiB; truncated or malformed envelopes fail the shape gate silently. + */ +export async function runSenpiForwarder(): Promise { + const url = process.env['SENPI_HOOK_FORWARD_URL']; + if (url === undefined || url.trim() === '') return; + const trimmedUrl = url.trim(); + if (!isAllowedSenpiForwardUrl(trimmedUrl)) return; + + const raw = await readStdin(); + const envelope = parseEnvelope(safeJson(raw)); + if (envelope === null) return; + + await postJson(trimmedUrl, raw, getTimeoutMs()); +} + +/** + * Accept only http(s) URLs. Loopback hosts are the default; remote hosts + * require `SENPI_HOOK_FORWARD_ALLOW_REMOTE=1`. + */ +function isAllowedSenpiForwardUrl(urlText: string): boolean { + let parsed: URL; + try { + parsed = new URL(urlText); + } catch { + return false; + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return false; + } + if (process.env['SENPI_HOOK_FORWARD_ALLOW_REMOTE'] === '1') { + return true; + } + return LOOPBACK_HOSTS.has(normalizeHostname(parsed.hostname)); +} + +/** + * WHATWG hostnames are usually unbracketed, but this Node reports IPv6 + * literals as `[::1]`. Strip surrounding brackets before the allowlist check. + */ +function normalizeHostname(hostname: string): string { + return hostname.startsWith('[') && hostname.endsWith(']') + ? hostname.slice(1, -1) + : hostname; +} + +/** + * POST a JSON payload and ignore the response entirely. + * + * Observe-only guarantee: network errors, timeouts, and non-success status + * codes are all swallowed; nothing is written to stdout and nothing throws. + */ +async function postJson( + url: string, + payload: string, + timeoutMs: number +): Promise { + try { + const response = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: payload, + redirect: 'error', + signal: AbortSignal.timeout(timeoutMs), + }); + await response.arrayBuffer(); + } catch { + // Observe-only: delivery is best-effort; every failure stays silent. + return; + } +} + +async function readStdin(): Promise { + try { + return await readBoundedTimedStdin({ + stdin: process.stdin as AsyncIterable, + maxBytes: DEFAULT_STDIN_MAX_BYTES, + timeoutMs: getTimeoutMs(), + stderr: { write: () => undefined }, + exit: () => undefined, + timeoutMessage: 'Timeout waiting for Senpi hook stdin input', + createTimeoutError: () => + new Error('Timeout waiting for Senpi hook stdin input'), + }); + } catch { + // Observe-only: timeout and read failures stay silent. + return ''; + } +} + +function safeJson(raw: string): unknown { + try { + return JSON.parse(raw) as unknown; + } catch { + return null; + } +} + +/** + * Accept only inputs shaped like a senpi `HookInputWire` envelope: an object + * carrying one of the seven event names (`event` primary or its snake_case + * `hook_event_name` alias) plus a string `cwd`. Anything else returns null. + */ +function parseEnvelope(value: unknown): ForwardedEnvelope | null { + if (!isRecord(value)) return null; + const primary = value['event']; + const alias = value['hook_event_name']; + const event = + typeof primary === 'string' + ? primary + : typeof alias === 'string' + ? alias + : undefined; + if (event === undefined || !SENPI_HOOK_EVENTS.has(event)) return null; + if (typeof value['cwd'] !== 'string') return null; + return { event }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function getTimeoutMs(): number { + const configured = Number(process.env['SENPI_HOOK_FORWARD_TIMEOUT_MS']); + return Number.isFinite(configured) && configured > 0 + ? Math.min(configured, DEFAULT_TIMEOUT_MS) + : DEFAULT_TIMEOUT_MS; +} + +const entryPath = process.argv[1]; +if ( + entryPath !== undefined && + import.meta.url === pathToFileURL(entryPath).href +) { + void runSenpiForwarder() + .catch(() => undefined) + .finally(() => process.exit(0)); +} diff --git a/src/forwarder/hook-forwarder.ts b/src/forwarder/hook-forwarder.ts index b65ec1f..59d29d6 100644 --- a/src/forwarder/hook-forwarder.ts +++ b/src/forwarder/hook-forwarder.ts @@ -8,8 +8,11 @@ import { isCwdUnderRoots, isProcessAlive, } from '../endpoint-discovery/index.js'; +import { readBoundedTimedStdin } from '../internal/stdin.js'; const DEFAULT_TIMEOUT_MS = 45_000; +const DEFAULT_STDIN_MAX_BYTES = 1024 * 1024; +const STDIN_TIMEOUT_MS = 30_000; interface ForwarderEvent { readonly hook_event_name: string; @@ -103,11 +106,19 @@ async function readEndpoint(filePath: string) { } async function readStdin(): Promise { - const chunks: Buffer[] = []; - for await (const chunk of process.stdin) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk))); + try { + return await readBoundedTimedStdin({ + stdin: process.stdin as AsyncIterable, + maxBytes: DEFAULT_STDIN_MAX_BYTES, + timeoutMs: STDIN_TIMEOUT_MS, + stderr: { write: () => undefined }, + exit: () => undefined, + timeoutMessage: 'Timeout waiting for stdin input', + createTimeoutError: () => new Error('Timeout waiting for stdin input'), + }); + } catch { + return ''; } - return Buffer.concat(chunks).toString('utf8'); } function safeJson(raw: string): unknown { diff --git a/src/forwarder/index.ts b/src/forwarder/index.ts index bd1f201..7e9d204 100644 --- a/src/forwarder/index.ts +++ b/src/forwarder/index.ts @@ -1 +1,5 @@ -export { RUN_HOOK_WRAPPER_SH } from './assets.js'; +export { + RUN_HOOK_WRAPPER_SH, + STANDALONE_HOOK_FORWARDER_ASSET, + STANDALONE_SENPI_HOOK_FORWARDER_ASSET, +} from './assets.js'; diff --git a/src/grok/execute.ts b/src/grok/execute.ts index fb2c23f..f5274b5 100644 --- a/src/grok/execute.ts +++ b/src/grok/execute.ts @@ -7,11 +7,13 @@ */ import { stdin, stdout, stderr, env, exit } from 'node:process'; +import { readBoundedTimedStdin } from '../internal/stdin.js'; import { logError, toError } from '../utils/index.js'; import { validateGrokHookInput } from './validation.js'; import type { GrokHookEventName, GrokHookInput } from './types.js'; const DEFAULT_STDIN_TIMEOUT_MS = 30000; +const DEFAULT_MAX_STDIN_BYTES = 1024 * 1024; const GROK_STOP_GATE_EVENTS: ReadonlySet = new Set([ 'stop', @@ -72,6 +74,8 @@ export interface GrokHookRunnerOptions { * Defaults to 30 seconds. */ readonly stdinTimeoutMs?: number; + /** Maximum number of stdin bytes retained before parsing. Defaults to 1 MiB. */ + readonly maxStdinBytes?: number; /** Exit hook invoked with the process exit code. Defaults to process.exit. */ readonly exit?: (code: number) => void; } @@ -100,31 +104,27 @@ async function readGrokStdinText( ): Promise { const source = options.stdin ?? (stdin as AsyncIterable); const exitFn = options.exit ?? exit; - const chunks: Buffer[] = []; - - let rejectOnTimeout: ((error: Error) => void) | undefined; - const timeout = setTimeout(() => { - logError('Timeout waiting for Grok hook stdin input'); - exitFn(1); - rejectOnTimeout?.(new GrokStdinTimeoutError()); - }, options.stdinTimeoutMs ?? DEFAULT_STDIN_TIMEOUT_MS); - - try { - await Promise.race([ - (async () => { - for await (const chunk of source) { - chunks.push(chunk); - } - })(), - new Promise((_resolve, reject) => { - rejectOnTimeout = reject; - }), - ]); - - return Buffer.concat(chunks).toString('utf-8'); - } finally { - clearTimeout(timeout); - } + const maxBytes = options.maxStdinBytes ?? DEFAULT_MAX_STDIN_BYTES; + const boundedSource = (async function* (): AsyncGenerator { + let remaining = maxBytes; + for await (const chunk of source) { + if (remaining <= 0) { + break; + } + yield chunk.subarray(0, remaining); + remaining -= chunk.byteLength; + } + })(); + + return readBoundedTimedStdin({ + stdin: boundedSource, + maxBytes, + timeoutMs: options.stdinTimeoutMs ?? DEFAULT_STDIN_TIMEOUT_MS, + stderr, + exit: exitFn, + timeoutMessage: 'Timeout waiting for Grok hook stdin input', + createTimeoutError: () => new GrokStdinTimeoutError(), + }); } /** diff --git a/src/grok/processing/discovery.ts b/src/grok/processing/discovery.ts index f1bc032..0e31c56 100644 --- a/src/grok/processing/discovery.ts +++ b/src/grok/processing/discovery.ts @@ -1,12 +1,19 @@ -import { readFile, readdir, stat } from 'node:fs/promises'; +import { open, stat } from 'node:fs/promises'; import { homedir } from 'node:os'; import { basename, join } from 'node:path'; import { blake3 } from '@noble/hashes/blake3.js'; import { z } from 'zod'; +import { + readDiscoveryDirectory, + sortDiscoveredPaths, +} from '../../internal/discovery-primitives.js'; + const MAX_DIRNAME_BYTES = 255; const LONG_CWD_SLUG_LENGTH = 40; +const MAX_CWD_METADATA_BYTES = 4 * 1024; +const MAX_SUMMARY_BYTES = 64 * 1024; /** Grok's persisted session summary fields used during discovery. */ export const grokSummarySchema = z.looseObject({ @@ -112,12 +119,7 @@ export async function findGrokSessionDirs( const sessionsRoot = join(getGrokHome(env), 'sessions'); const encodedCwd = encodeGrokCwdDirname(cwd); - let cwdEntries; - try { - cwdEntries = await readdir(sessionsRoot, { withFileTypes: true }); - } catch { - return []; - } + const cwdEntries = await readDiscoveryDirectory(sessionsRoot); const matchingCwdDirs: string[] = []; for (const entry of cwdEntries) { @@ -133,12 +135,7 @@ export async function findGrokSessionDirs( const sessionDirs: string[] = []; for (const cwdDir of matchingCwdDirs) { - let entries; - try { - entries = await readdir(cwdDir, { withFileTypes: true }); - } catch { - continue; - } + const entries = await readDiscoveryDirectory(cwdDir); for (const entry of entries) { if (entry.isDirectory()) { @@ -150,7 +147,7 @@ export async function findGrokSessionDirs( } } - return sessionDirs.sort((left, right) => left.localeCompare(right)); + return sortDiscoveredPaths(sessionDirs); } /** @@ -193,7 +190,10 @@ async function cwdMetadataMatches( cwd: string ): Promise { try { - const storedCwd = await readFile(join(cwdDirectory, '.cwd'), 'utf8'); + const storedCwd = await readBoundedText( + join(cwdDirectory, '.cwd'), + MAX_CWD_METADATA_BYTES + ); return storedCwd.trim() === cwd; } catch { return false; @@ -208,12 +208,26 @@ async function fileExists(path: string): Promise { } } +async function readBoundedText( + path: string, + maxBytes: number +): Promise { + const handle = await open(path, 'r'); + try { + const buffer = Buffer.alloc(maxBytes); + const { bytesRead } = await handle.read(buffer, 0, maxBytes, 0); + return buffer.subarray(0, bytesRead).toString('utf8'); + } finally { + await handle.close(); + } +} + async function readGrokSession(sessionDir: string): Promise { const sessionId = basename(sessionDir); try { - const summaryJson = await readFile( + const summaryJson = await readBoundedText( join(sessionDir, 'summary.json'), - 'utf8' + MAX_SUMMARY_BYTES ); const summary = grokSummaryJsonSchema.parse(summaryJson); return { kind: 'valid', sessionId, sessionDir, summary }; diff --git a/src/grok/processing/index.ts b/src/grok/processing/index.ts index 53147a3..8be9eb0 100644 --- a/src/grok/processing/index.ts +++ b/src/grok/processing/index.ts @@ -1,5 +1,7 @@ /** Public processing APIs for persisted Grok sessions. */ +export { byteCursorsEqual } from '../../internal/incremental.js'; + export { encodeGrokCwdDirname, findGrokSessionDirs, @@ -24,7 +26,10 @@ export { grokEventSchema, parseGrokEvent } from './events.js'; export type { GrokEvent, GrokEventParseResult } from './events.js'; export { + STALE_CHECKPOINT_CONFLICT_CODE, + StaleCheckpointConflict, commitGrokSessionCheckpoint, + isStaleCheckpointConflict, tailGrokSession, watchGrokSession, } from './tail.js'; @@ -32,10 +37,12 @@ export type { GrokCheckpointStatus, GrokSessionCheckpoint, GrokSessionCheckpointCommitOptions, + GrokSessionCheckpointState, GrokSessionSourceCheckpoint, GrokSessionTailOptions, GrokSessionTailResult, GrokSessionWatchOptions, + GrokWatchClock, GrokSourceReset, GrokSourceTailResult, GrokTailDiagnostic, @@ -63,3 +70,8 @@ export type { GrokToolUseBlock, GrokUserTextBlock, } from './blocks.js'; + +export type { + JsonlCursor, + JsonlScanStatus, +} from '../../internal/jsonl-cursor.js'; diff --git a/src/grok/processing/jsonl-cursor.ts b/src/grok/processing/jsonl-cursor.ts deleted file mode 100644 index 7062536..0000000 --- a/src/grok/processing/jsonl-cursor.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { createHash } from 'node:crypto'; -import { open, type FileHandle } from 'node:fs/promises'; - -const DEFAULT_MAX_LINE_BYTES = 16 * 1024 * 1024; -const SCAN_CHUNK_BYTES = 64 * 1024; -const DIGEST_WINDOW_BYTES = 4096; - -/** Serializable position and file identity for incremental JSONL reads. */ -export interface JsonlCursor { - /** Device identifier from the opened file. */ - readonly device: string; - /** Inode identifier from the opened file. */ - readonly inode: string; - /** Byte offset of the next uncommitted line. */ - readonly offset: number; - /** One-based number of the next uncommitted line. */ - readonly lineNumber: number; - /** Number of file identity or content resets observed by this cursor. */ - readonly generation: number; - /** SHA-256 digest of the committed prefix's leading window. */ - readonly headDigest: string; - /** SHA-256 digest of the committed prefix's trailing boundary window. */ - readonly boundaryDigest: string; -} - -/** One complete newline-terminated JSONL line. */ -export interface JsonlLine { - /** UTF-8 decoded line content without its terminating newline. */ - readonly value: string; - /** One-based physical line number. */ - readonly lineNumber: number; - /** Inclusive byte offset at which the line begins. */ - readonly byteStart: number; - /** Exclusive byte offset after the terminating newline. */ - readonly byteEnd: number; -} - -/** Diagnostic emitted for a complete line that exceeded the configured limit. */ -export interface JsonlOversizedDiagnostic { - /** Diagnostic discriminator. */ - readonly kind: 'oversized'; - /** One-based physical line number. */ - readonly lineNumber: number; - /** Inclusive byte offset at which the discarded line begins. */ - readonly byteStart: number; - /** Exclusive byte offset after the terminating newline. */ - readonly byteEnd: number; -} - -/** Result of one size-snapshotted JSONL scan. */ -export interface JsonlDelta { - /** Complete lines committed by this scan. */ - readonly lines: readonly JsonlLine[]; - /** Complete lines discarded by this scan. */ - readonly diagnostics: readonly JsonlOversizedDiagnostic[]; - /** Position to use for the next scan, or null when no file has been seen. */ - readonly cursor: JsonlCursor | null; - /** Open-file size snapshot, or null when the path was missing. */ - readonly fileSize: number | null; - /** Whether this scan discarded stale cursor position and rescanned from zero. */ - readonly reset: boolean; -} - -/** Options controlling a JSONL delta scan. */ -export interface ReadJsonlDeltaOptions { - /** Maximum buffered bytes per line before streaming discard begins. */ - readonly maxLineBytes?: number; -} - -interface ScanResult { - readonly lines: readonly JsonlLine[]; - readonly diagnostics: readonly JsonlOversizedDiagnostic[]; - readonly offset: number; - readonly lineNumber: number; -} - -/** - * Read complete JSONL lines added after a cursor position. - * - * The file identity and size come from the opened handle. Reads stop at that - * size even if writers append during the scan. A trailing line without a - * newline remains uncommitted and is read again on the next call. Complete - * oversized lines are discarded without retaining their content in memory. - * - * @param path - JSONL file path. - * @param cursor - Prior serializable cursor, or null for a full scan. - * @param options - Per-scan line size limit. - * @returns Complete lines, diagnostics, and the next cursor. - * @throws If the file cannot be read, except when the path is missing. - * @throws If `maxLineBytes` is not a non-negative safe integer. - */ -export async function readJsonlDelta( - path: string, - cursor: JsonlCursor | null, - options: ReadJsonlDeltaOptions = {} -): Promise { - const maxLineBytes = options.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES; - if (!Number.isSafeInteger(maxLineBytes) || maxLineBytes < 0) { - throw new RangeError('maxLineBytes must be a non-negative safe integer'); - } - - let file: FileHandle; - try { - file = await open(path, 'r'); - } catch (error) { - if (hasErrorCode(error, 'ENOENT')) { - return { - lines: [], - diagnostics: [], - cursor, - fileSize: null, - reset: false, - }; - } - throw error; - } - - try { - const stats = await file.stat(); - const fileSize = stats.size; - const device = String(stats.dev); - const inode = String(stats.ino); - const reset = await shouldResetCursor( - file, - fileSize, - device, - inode, - cursor - ); - const startOffset = reset ? 0 : (cursor?.offset ?? 0); - const startLineNumber = reset ? 1 : (cursor?.lineNumber ?? 1); - const generation = (cursor?.generation ?? 0) + (reset ? 1 : 0); - const scan = await scanCompleteLines( - file, - startOffset, - startLineNumber, - fileSize, - maxLineBytes - ); - const digests = await digestCommittedBoundary(file, scan.offset); - - return { - lines: scan.lines, - diagnostics: scan.diagnostics, - cursor: { - device, - inode, - offset: scan.offset, - lineNumber: scan.lineNumber, - generation, - headDigest: digests.headDigest, - boundaryDigest: digests.boundaryDigest, - }, - fileSize, - reset, - }; - } finally { - await file.close(); - } -} - -async function shouldResetCursor( - file: FileHandle, - fileSize: number, - device: string, - inode: string, - cursor: JsonlCursor | null -): Promise { - if (cursor === null) return false; - if (cursor.device !== device || cursor.inode !== inode) return true; - if (fileSize < cursor.offset) return true; - - const digests = await digestCommittedBoundary(file, cursor.offset); - return ( - digests.headDigest !== cursor.headDigest || - digests.boundaryDigest !== cursor.boundaryDigest - ); -} - -async function scanCompleteLines( - file: FileHandle, - startOffset: number, - startLineNumber: number, - snapshotSize: number, - maxLineBytes: number -): Promise { - const lines: JsonlLine[] = []; - const diagnostics: JsonlOversizedDiagnostic[] = []; - const readBuffer = Buffer.allocUnsafe(SCAN_CHUNK_BYTES); - let readOffset = startOffset; - let committedOffset = startOffset; - let lineStart = startOffset; - let lineNumber = startLineNumber; - let lineByteLength = 0; - let lineChunks: Buffer[] = []; - let discarding = false; - - while (readOffset < snapshotSize) { - const requestedBytes = Math.min( - readBuffer.byteLength, - snapshotSize - readOffset - ); - const { bytesRead } = await file.read( - readBuffer, - 0, - requestedBytes, - readOffset - ); - if (bytesRead === 0) break; - - let chunkOffset = 0; - while (chunkOffset < bytesRead) { - const newlineIndex = readBuffer.indexOf(0x0a, chunkOffset); - const segmentEnd = - newlineIndex >= 0 && newlineIndex < bytesRead - ? newlineIndex - : bytesRead; - const segmentLength = segmentEnd - chunkOffset; - - if (!discarding) { - if (lineByteLength + segmentLength > maxLineBytes) { - discarding = true; - lineChunks = []; - } else if (segmentLength > 0) { - lineChunks.push( - Buffer.from( - readBuffer.subarray(chunkOffset, chunkOffset + segmentLength) - ) - ); - } - } - lineByteLength += segmentLength; - - if (newlineIndex < 0 || newlineIndex >= bytesRead) break; - - const byteEnd = readOffset + newlineIndex + 1; - if (discarding) { - diagnostics.push({ - kind: 'oversized', - lineNumber, - byteStart: lineStart, - byteEnd, - }); - } else { - lines.push({ - value: Buffer.concat(lineChunks, lineByteLength).toString('utf8'), - lineNumber, - byteStart: lineStart, - byteEnd, - }); - } - - committedOffset = byteEnd; - lineStart = byteEnd; - lineNumber += 1; - lineByteLength = 0; - lineChunks = []; - discarding = false; - chunkOffset = newlineIndex + 1; - } - readOffset += bytesRead; - } - - return { lines, diagnostics, offset: committedOffset, lineNumber }; -} - -async function digestCommittedBoundary( - file: FileHandle, - offset: number -): Promise<{ readonly headDigest: string; readonly boundaryDigest: string }> { - const headLength = Math.min(offset, DIGEST_WINDOW_BYTES); - const boundaryStart = Math.max(0, offset - DIGEST_WINDOW_BYTES); - const boundaryLength = offset - boundaryStart; - const [head, boundary] = await Promise.all([ - readRange(file, 0, headLength), - readRange(file, boundaryStart, boundaryLength), - ]); - return { - headDigest: createHash('sha256').update(head).digest('hex'), - boundaryDigest: createHash('sha256').update(boundary).digest('hex'), - }; -} - -async function readRange( - file: FileHandle, - position: number, - length: number -): Promise { - if (length === 0) return Buffer.alloc(0); - const buffer = Buffer.allocUnsafe(length); - let totalRead = 0; - while (totalRead < length) { - const { bytesRead } = await file.read( - buffer, - totalRead, - length - totalRead, - position + totalRead - ); - if (bytesRead === 0) break; - totalRead += bytesRead; - } - return buffer.subarray(0, totalRead); -} - -function hasErrorCode(error: unknown, code: string): boolean { - return ( - typeof error === 'object' && - error !== null && - 'code' in error && - error.code === code - ); -} diff --git a/src/grok/processing/tail-marker.ts b/src/grok/processing/tail-marker.ts new file mode 100644 index 0000000..02c18b5 --- /dev/null +++ b/src/grok/processing/tail-marker.ts @@ -0,0 +1,238 @@ +import { readFile } from 'node:fs/promises'; +import { basename, join, resolve } from 'node:path'; + +import { + byteCursorsEqual, + checkpointRevision, +} from '../../internal/incremental.js'; +import { withMarkerLock } from '../../internal/marker-lock.js'; +import { + createMarkerPathDigest, + resolveAllowedMarkerDir, + sanitizeMarkerBase, + writePrivateJson, + type ResolveAllowedMarkerDirOptions, +} from '../../internal/marker-store.js'; +import { + parseJsonlOversizedPending, + type JsonlCursor, +} from '../../internal/jsonl-cursor.js'; +import { StaleCheckpointConflict } from '../../processing/stale-checkpoint-conflict.js'; +import { grokSourceKinds } from './tail-order.js'; +import type { + GrokSessionCheckpoint, + GrokSessionCheckpointCommitOptions, + GrokTailSourceKind, +} from './tail-types.js'; + +const MARKER_VERSION = 1; + +/** Private persisted Grok marker schema. */ +export interface GrokSessionMarker { + readonly version: 1; + readonly sessionPathDigest: string; + readonly revision: number; + readonly sources: Readonly>; +} + +/** Stable digest of one resolved Grok session directory. */ +export function createGrokSessionPathDigest(sessionDir: string): string { + return createMarkerPathDigest(sessionDir); +} + +/** Resolve the bounded private marker path for one Grok session. */ +export function getGrokSessionMarkerPath( + sessionDir: string, + options: GrokSessionCheckpointCommitOptions +): string { + const markerDir = + options.markerDir === undefined + ? resolve(sessionDir, '.tail-markers') + : resolveAllowedMarkerDir( + options.markerDir, + grokAllowedMarkerDirOptions(options) + ); + const digest = createGrokSessionPathDigest(sessionDir); + const sessionName = sanitizeMarkerBase(basename(sessionDir)); + return join( + markerDir, + `${sessionName}-${digest.slice(0, 16)}.grok-session.json` + ); +} + +/** Read and validate one Grok marker; invalid or missing bytes are absent-state. */ +export async function readGrokSessionMarker( + markerPath: string, + sessionPathDigest: string +): Promise { + try { + const parsed: unknown = JSON.parse(await readFile(markerPath, 'utf8')); + if (!isRecord(parsed) || parsed['version'] !== MARKER_VERSION) return null; + if (parsed['sessionPathDigest'] !== sessionPathDigest) return null; + const revision = parsed['revision']; + const sources = parsed['sources']; + if (!isSafeNonnegativeInteger(revision) || !isRecord(sources)) return null; + const updates = parseCursor(sources['updates']); + const events = parseCursor(sources['events']); + if (updates === undefined || events === undefined) return null; + return { + version: MARKER_VERSION, + sessionPathDigest, + revision, + sources: { updates, events }, + }; + } catch { + return null; + } +} + +/** Validate and index both source cursors from a public checkpoint. */ +export function grokCheckpointSources( + checkpoint: GrokSessionCheckpoint +): Record { + if (!isSafeNonnegativeInteger(checkpoint.baseRevision)) { + throw new Error('Invalid Grok session checkpoint revision'); + } + const sources: Partial> = {}; + for (const source of checkpoint.sources) { + if (source.sourceKind !== 'updates' && source.sourceKind !== 'events') { + throw new Error('Invalid Grok session checkpoint source'); + } + if (Object.hasOwn(sources, source.sourceKind)) { + throw new Error('Grok session checkpoint has duplicate sources'); + } + const parsed = parseCursor(source.cursor); + if (parsed === undefined) { + throw new Error('Invalid Grok session checkpoint cursor'); + } + sources[source.sourceKind] = parsed; + } + if (!Object.hasOwn(sources, 'updates') || !Object.hasOwn(sources, 'events')) { + throw new Error('Grok session checkpoint must contain both sources'); + } + return { updates: sources.updates ?? null, events: sources.events ?? null }; +} + +/** True when automatic persistence would advance at least one cursor. */ +export function shouldCommitGrokMarker( + marker: GrokSessionMarker | null, + checkpoint: GrokSessionCheckpoint +): boolean { + const next = grokCheckpointSources(checkpoint); + if (marker === null) return next.updates !== null || next.events !== null; + return grokSourceKinds().some( + sourceKind => + !byteCursorsEqual(marker.sources[sourceKind], next[sourceKind]) + ); +} + +/** Commit a validated Grok checkpoint under the shared private marker lock. */ +export async function commitGrokSessionCheckpoint( + sessionDir: string, + checkpoint: GrokSessionCheckpoint, + options: GrokSessionCheckpointCommitOptions = {} +): Promise { + const resolvedSessionDir = resolve(sessionDir); + const sessionPathDigest = createGrokSessionPathDigest(resolvedSessionDir); + if (checkpoint.sessionPathDigest !== sessionPathDigest) { + throw new Error('Grok session checkpoint does not match the session path'); + } + const nextSources = grokCheckpointSources(checkpoint); + const markerPath = getGrokSessionMarkerPath(resolvedSessionDir, options); + await withMarkerLock( + markerPath, + async lease => { + const marker = await readGrokSessionMarker(markerPath, sessionPathDigest); + const revision = marker?.revision ?? 0; + if (checkpoint.baseRevision !== revision) { + throw new StaleCheckpointConflict({ + expectedRevision: checkpoint.baseRevision, + actualRevision: revision, + }); + } + validateProgression(marker, nextSources); + await lease.renew(); + await writePrivateJson( + markerPath, + { + version: MARKER_VERSION, + sessionPathDigest, + revision: checkpointRevision(revision, true), + sources: nextSources, + } satisfies GrokSessionMarker, + options.markerDir === undefined + ? undefined + : grokAllowedMarkerDirOptions(options) + ); + }, + { lockedLabel: 'Grok session marker' } + ); +} + +function grokAllowedMarkerDirOptions( + options: GrokSessionCheckpointCommitOptions +): ResolveAllowedMarkerDirOptions { + return { + allowedMarkerRoots: options.allowedMarkerRoots ?? [], + emptyRootsMessage: + 'Custom markerDir requires allowedMarkerRoots to include an allowed root', + }; +} + +function parseCursor(value: unknown): JsonlCursor | null | undefined { + if (value === null) return null; + if (!isRecord(value)) return undefined; + const pending = parseJsonlOversizedPending(value['pending']); + if ( + typeof value['device'] !== 'string' || + typeof value['inode'] !== 'string' || + !isSafeNonnegativeInteger(value['offset']) || + !isSafeNonnegativeInteger(value['lineNumber']) || + value['lineNumber'] < 1 || + !isSafeNonnegativeInteger(value['generation']) || + typeof value['headDigest'] !== 'string' || + typeof value['boundaryDigest'] !== 'string' || + pending === undefined + ) + return undefined; + return { + device: value['device'], + inode: value['inode'], + offset: value['offset'], + lineNumber: value['lineNumber'], + generation: value['generation'], + headDigest: value['headDigest'], + boundaryDigest: value['boundaryDigest'], + pending, + }; +} + +function validateProgression( + marker: GrokSessionMarker | null, + next: Readonly> +): void { + for (const sourceKind of grokSourceKinds()) { + const previous = marker?.sources[sourceKind] ?? null; + const cursor = next[sourceKind]; + if (previous === null || cursor === null) continue; + if (cursor.generation === previous.generation) { + if (cursor.offset < previous.offset) { + throw new Error( + 'Grok session checkpoint would move a source backwards' + ); + } + } else if (cursor.generation !== previous.generation + 1) { + throw new Error( + 'Grok session checkpoint has an invalid generation transition' + ); + } + } +} + +function isSafeNonnegativeInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/grok/processing/tail-order.ts b/src/grok/processing/tail-order.ts new file mode 100644 index 0000000..43857f5 --- /dev/null +++ b/src/grok/processing/tail-order.ts @@ -0,0 +1,36 @@ +import type { GrokRecordOrigin } from './blocks.js'; +import type { GrokTailRecord, GrokTailSourceKind } from './tail-types.js'; + +/** Compare records by timestamp, source, generation, and byte range. */ +export function compareGrokTailRecords( + left: GrokTailRecord, + right: GrokTailRecord +): number { + if (left.effectiveTimestamp !== right.effectiveTimestamp) { + return left.effectiveTimestamp < right.effectiveTimestamp ? -1 : 1; + } + const sourceDifference = + sourceRank(left.sourceKind) - sourceRank(right.sourceKind); + if (sourceDifference !== 0) return sourceDifference; + if (left.generation !== right.generation) { + return left.generation < right.generation ? -1 : 1; + } + if (left.byteStart !== right.byteStart) { + return left.byteStart < right.byteStart ? -1 : 1; + } + return left.byteEnd - right.byteEnd; +} + +/** Stable source iteration order used by markers and record ordering. */ +export function grokSourceKinds(): readonly GrokTailSourceKind[] { + return ['updates', 'events']; +} + +/** Stable identity key for one normalized source origin. */ +export function grokOriginKey(origin: GrokRecordOrigin): string { + return `${origin.sourceId}:${String(origin.generation)}:${String(origin.byteStart)}:${String(origin.byteEnd)}`; +} + +function sourceRank(sourceKind: GrokTailSourceKind): number { + return sourceKind === 'updates' ? 0 : 1; +} diff --git a/src/grok/processing/tail-parse.ts b/src/grok/processing/tail-parse.ts new file mode 100644 index 0000000..0cf7e03 --- /dev/null +++ b/src/grok/processing/tail-parse.ts @@ -0,0 +1,252 @@ +import type { JsonlDelta, JsonlLine } from '../../internal/jsonl-cursor.js'; +import type { GrokNormalizedRecord, GrokRecordOrigin } from './blocks.js'; +import { grokSourceKinds } from './tail-order.js'; +import { parseGrokEvent } from './events.js'; +import type { + GrokTailDiagnostic, + GrokTailRecord, + GrokTailSourceKind, +} from './tail-types.js'; +import { parseGrokSessionUpdate } from './updates.js'; + +/** Parsed records and diagnostics across both source snapshots. */ +export interface GrokParsedSources { + readonly records: readonly GrokTailRecord[]; + readonly diagnostics: readonly GrokTailDiagnostic[]; +} + +interface ParsedLine { + readonly record?: GrokTailRecord; + readonly diagnostic?: GrokTailDiagnostic; +} + +/** Parse complete lines from both Grok source deltas. */ +export function parseGrokSources( + deltas: Readonly>, + generations?: Readonly> +): GrokParsedSources { + const records: GrokTailRecord[] = []; + const diagnostics: GrokTailDiagnostic[] = []; + for (const sourceKind of grokSourceKinds()) { + const delta = deltas[sourceKind]; + const generation = + generations?.[sourceKind] ?? delta.cursor?.generation ?? 0; + for (const diagnostic of delta.diagnostics) { + diagnostics.push({ + sourceKind, + kind: 'oversized', + lineNumber: diagnostic.lineNumber, + byteStart: diagnostic.byteStart, + byteEnd: diagnostic.byteEnd, + message: 'JSONL line exceeds maxLineBytes', + }); + } + for (const line of delta.lines) { + const parsed = parseLine(sourceKind, generation, line); + if (parsed.record !== undefined) records.push(parsed.record); + if (parsed.diagnostic !== undefined) diagnostics.push(parsed.diagnostic); + } + } + return { records, diagnostics }; +} + +function parseLine( + sourceKind: GrokTailSourceKind, + generation: number, + line: JsonlLine +): ParsedLine { + let raw: unknown; + try { + raw = JSON.parse(line.value) as unknown; + } catch (error: unknown) { + return { + diagnostic: lineDiagnostic( + sourceKind, + line, + 'invalid_json', + error instanceof Error ? error.message : String(error) + ), + }; + } + if (sourceKind === 'updates') { + const parsed = parseGrokSessionUpdate(raw); + if (parsed.kind === 'unknown') { + return unknownParsedLine( + sourceKind, + generation, + line, + parsed.tag, + parsed.raw + ); + } + if (parsed.kind !== 'known') { + return { + diagnostic: lineDiagnostic( + sourceKind, + line, + 'invalid_record', + parsed.error + ), + }; + } + const nativeType = parsed.envelope.params.update.sessionUpdate; + const origin = createOrigin(sourceKind, nativeType, generation, line); + const record: GrokNormalizedRecord = { + kind: 'update', + envelope: parsed.envelope, + origin, + }; + return { + record: { + sourceKind, + effectiveTimestamp: updateTimestamp(parsed.envelope), + nativeType, + generation, + byteStart: line.byteStart, + byteEnd: line.byteEnd, + record, + }, + }; + } + const parsed = parseGrokEvent(raw); + if (parsed.kind === 'unknown') { + return unknownParsedLine( + sourceKind, + generation, + line, + parsed.tag, + parsed.raw + ); + } + if (parsed.kind !== 'known') { + return { + diagnostic: lineDiagnostic( + sourceKind, + line, + 'invalid_record', + parsed.error + ), + }; + } + const nativeType = parsed.event.type; + const origin = createOrigin(sourceKind, nativeType, generation, line); + const record: GrokNormalizedRecord = { + kind: 'event', + event: parsed.event, + origin, + }; + const parsedTimestamp = Date.parse(parsed.event.ts); + return { + record: { + sourceKind, + effectiveTimestamp: Number.isFinite(parsedTimestamp) + ? parsedTimestamp + : 0, + nativeType, + generation, + byteStart: line.byteStart, + byteEnd: line.byteEnd, + record, + }, + }; +} + +function unknownParsedLine( + sourceKind: GrokTailSourceKind, + generation: number, + line: JsonlLine, + tag: string, + raw: unknown +): ParsedLine { + const origin = createOrigin(sourceKind, tag, generation, line); + const record: GrokNormalizedRecord = { kind: 'unknown', tag, raw, origin }; + return { + record: { + sourceKind, + effectiveTimestamp: unknownRecordTimestamp(sourceKind, raw), + nativeType: tag, + generation, + byteStart: line.byteStart, + byteEnd: line.byteEnd, + record, + }, + diagnostic: lineDiagnostic( + sourceKind, + line, + 'unknown_record', + sourceKind === 'updates' + ? `Unknown update '${tag}'` + : `Unknown event '${tag}'` + ), + }; +} + +function createOrigin( + sourceKind: GrokTailSourceKind, + nativeType: string, + generation: number, + line: JsonlLine +): GrokRecordOrigin { + return { + harness: 'grok', + stream: sourceKind === 'updates' ? 'conversation' : 'activity', + sourceId: sourceKind, + nativeType, + generation, + byteStart: line.byteStart, + byteEnd: line.byteEnd, + }; +} + +function lineDiagnostic( + sourceKind: GrokTailSourceKind, + line: JsonlLine, + kind: GrokTailDiagnostic['kind'], + message: string +): GrokTailDiagnostic { + return { + sourceKind, + kind, + lineNumber: line.lineNumber, + byteStart: line.byteStart, + byteEnd: line.byteEnd, + message, + }; +} + +function unknownRecordTimestamp( + sourceKind: GrokTailSourceKind, + raw: unknown +): number { + if (!isRecord(raw)) return 0; + if (sourceKind === 'updates') { + const timestamp = raw['timestamp']; + if (typeof timestamp === 'number' && Number.isFinite(timestamp)) { + return Math.abs(timestamp) < 100_000_000_000 + ? timestamp * 1_000 + : timestamp; + } + return 0; + } + const timestamp = raw['ts']; + if (typeof timestamp !== 'string') return 0; + const parsed = Date.parse(timestamp); + return Number.isFinite(parsed) ? parsed : 0; +} + +function updateTimestamp( + envelope: Extract['envelope'] +): number { + const meta = envelope.params._meta; + if (typeof meta === 'object' && meta !== null) { + const value: unknown = Reflect.get(meta, 'agentTimestampMs'); + if (typeof value === 'number' && Number.isFinite(value)) return value; + } + return Math.abs(envelope.timestamp) < 100_000_000_000 + ? envelope.timestamp * 1_000 + : envelope.timestamp; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/grok/processing/tail-result.ts b/src/grok/processing/tail-result.ts new file mode 100644 index 0000000..5114e26 --- /dev/null +++ b/src/grok/processing/tail-result.ts @@ -0,0 +1,63 @@ +import type { JsonlCursor, JsonlDelta } from '../../internal/jsonl-cursor.js'; +import type { + GrokSessionTailResult, + GrokSourceTailResult, + GrokTailRecord, + GrokTailSourceKind, +} from './tail-types.js'; + +/** Build one source-level accounting result from a bounded cursor delta. */ +export function grokSourceResult( + sourceKind: GrokTailSourceKind, + sourcePath: string, + previousCursor: JsonlCursor | null, + delta: JsonlDelta, + records: readonly GrokTailRecord[] +): GrokSourceTailResult { + return { + sourceKind, + sourcePath, + status: delta.fileSize === null ? 'missing' : 'read', + recordCount: records.filter(record => record.sourceKind === sourceKind) + .length, + generation: delta.cursor?.generation ?? previousCursor?.generation ?? 0, + previousByteOffset: previousCursor?.offset ?? 0, + newByteOffset: delta.cursor?.offset ?? previousCursor?.offset ?? 0, + fileSize: delta.fileSize, + reset: delta.reset, + }; +} + +/** Apply one explicit from-start generation transition to a neutral delta. */ +export function applyGrokFromStartGeneration( + delta: JsonlDelta, + previousCursor: JsonlCursor | null, + fromStart: boolean | undefined +): JsonlDelta { + if (fromStart !== true || previousCursor === null || delta.cursor === null) { + return delta; + } + return { + ...delta, + cursor: { + ...delta.cursor, + generation: previousCursor.generation + 1, + }, + }; +} + +/** True when any source cursor already committed bytes. */ +export function hasPriorGrokBytes( + cursors: Readonly> +): boolean { + return Object.values(cursors).some(cursor => (cursor?.offset ?? 0) > 0); +} + +/** True when a watch reconcile carries observable records, diagnostics, or resets. */ +export function isObservableGrokResult(result: GrokSessionTailResult): boolean { + return ( + result.records.length > 0 || + result.diagnostics.length > 0 || + result.resets.length > 0 + ); +} diff --git a/src/grok/processing/tail-run.ts b/src/grok/processing/tail-run.ts new file mode 100644 index 0000000..7dc980f --- /dev/null +++ b/src/grok/processing/tail-run.ts @@ -0,0 +1,279 @@ +import { join, resolve } from 'node:path'; + +import { + readJsonlDelta, + type JsonlCursor, + type JsonlScanStatus, +} from '../../internal/jsonl-cursor.js'; +import { reduceGrokRecords } from './blocks.js'; +import { + commitGrokSessionCheckpoint, + createGrokSessionPathDigest, + getGrokSessionMarkerPath, + grokCheckpointSources, + readGrokSessionMarker, + shouldCommitGrokMarker, +} from './tail-marker.js'; +import { + compareGrokTailRecords, + grokOriginKey, + grokSourceKinds, +} from './tail-order.js'; +import { parseGrokSources } from './tail-parse.js'; +import { + applyGrokFromStartGeneration, + grokSourceResult, + hasPriorGrokBytes, +} from './tail-result.js'; +import { + GROK_SOURCE_FILENAMES, + type GrokCheckpointStatus, + type GrokSessionCheckpoint, + type GrokSessionTailOptions, + type GrokSessionTailResult, + type GrokSourceReset, + type GrokTailRecord, + type GrokTailSourceKind, +} from './tail-types.js'; + +/** + * Tail updates.jsonl and events.jsonl as one revisioned session stream. + * + * @param sessionDir - Directory containing persisted Grok session files. + * @param options - Cursor, marker, reduction, and line-size controls. + * @returns Ordered records, normalized changes, diagnostics, and checkpoint. + */ +export async function tailGrokSession( + sessionDir: string, + options: GrokSessionTailOptions = {} +): Promise { + const resolvedSessionDir = resolve(sessionDir); + const sessionPathDigest = createGrokSessionPathDigest(resolvedSessionDir); + const markerPath = getGrokSessionMarkerPath(resolvedSessionDir, options); + const marker = await readGrokSessionMarker(markerPath, sessionPathDigest); + const cursorOptions = grokCursorOptions(options); + const supplied = options.checkpoint; + if ( + supplied !== undefined && + supplied.sessionPathDigest !== sessionPathDigest + ) { + throw new Error('Grok session checkpoint does not match the session path'); + } + const suppliedCursors = + supplied === undefined ? null : grokCheckpointSources(supplied); + const markerCursors = { + updates: suppliedCursors?.updates ?? marker?.sources.updates ?? null, + events: suppliedCursors?.events ?? marker?.sources.events ?? null, + } satisfies Record; + const previousCursors = { + updates: options.fromStart ? null : markerCursors.updates, + events: options.fromStart ? null : markerCursors.events, + } satisfies Record; + const updatePath = join(resolvedSessionDir, GROK_SOURCE_FILENAMES.updates); + const eventPath = join(resolvedSessionDir, GROK_SOURCE_FILENAMES.events); + const updateDelta = await readJsonlDelta( + updatePath, + previousCursors.updates, + cursorOptions + ); + if (updateDelta.fileSize === null) { + throw new Error(`Missing required Grok updates source '${updatePath}'`); + } + const eventDelta = await readJsonlDelta( + eventPath, + previousCursors.events, + cursorOptions + ); + const deltas = { + updates: applyGrokFromStartGeneration( + updateDelta, + markerCursors.updates, + options.fromStart + ), + events: applyGrokFromStartGeneration( + eventDelta, + markerCursors.events, + options.fromStart + ), + } as const; + const parsedDelta = parseGrokSources(deltas); + const orderedRecords = [...parsedDelta.records].sort(compareGrokTailRecords); + const deltaOrigins = new Set( + orderedRecords.map(record => grokOriginKey(record.record.origin)) + ); + const priorStateRecords = + options.fromStart === true ? [] : (supplied?.state?.records ?? []); + const retained = priorStateRecords.filter( + record => !deltas[record.sourceKind].reset + ); + let stateRecords: readonly GrokTailRecord[] = [ + ...retained, + ...orderedRecords, + ].sort(compareGrokTailRecords); + let reductionRecords = stateRecords.map(record => record.record); + const needsStateRebuild = + hasPriorGrokBytes(previousCursors) && supplied?.state === undefined; + let stateComplete = !needsStateRebuild; + if ( + (orderedRecords.length > 0 || parsedDelta.diagnostics.length > 0) && + needsStateRebuild + ) { + const rebuildOptions = grokRebuildCursorOptions(options); + const [allUpdates, allEvents] = await Promise.all([ + readJsonlDelta(updatePath, null, rebuildOptions), + readJsonlDelta(eventPath, null, rebuildOptions), + ]); + if (allUpdates.fileSize === null) { + throw new Error(`Missing required Grok updates source '${updatePath}'`); + } + if ( + allUpdates.scanStatus.status === 'complete' && + (allEvents.fileSize === null || + allEvents.scanStatus.status === 'complete') + ) { + const full = parseGrokSources( + { updates: allUpdates, events: allEvents }, + { + updates: updateDelta.cursor?.generation ?? 0, + events: eventDelta.cursor?.generation ?? 0, + } + ); + stateRecords = [...full.records].sort(compareGrokTailRecords); + reductionRecords = stateRecords.map(record => record.record); + stateComplete = true; + } + } + const reduction = reduceGrokRecords(reductionRecords); + const changes = reduction.changes.filter(change => + deltaOrigins.has( + grokOriginKey( + change.type === 'upsert' ? change.block.origin : change.origin + ) + ) + ); + const activities = + options.includeActivities === false + ? [] + : reduction.activities.filter(activity => + deltaOrigins.has(grokOriginKey(activity.origin)) + ); + const checkpoint: GrokSessionCheckpoint = { + sessionPathDigest, + baseRevision: marker?.revision ?? 0, + sources: grokSourceKinds().map(sourceKind => ({ + sourceKind, + cursor: stateComplete + ? deltas[sourceKind].cursor + : previousCursors[sourceKind], + })), + ...(stateComplete ? { state: { records: stateRecords } } : {}), + }; + const sources = grokSourceKinds().map(sourceKind => + grokSourceResult( + sourceKind, + join(resolvedSessionDir, GROK_SOURCE_FILENAMES[sourceKind]), + previousCursors[sourceKind], + deltas[sourceKind], + parsedDelta.records + ) + ); + const resets: GrokSourceReset[] = sources + .filter(source => source.reset) + .map(source => ({ + type: 'source_reset', + sourceKind: source.sourceKind, + generation: source.generation, + })); + const checkpointStatus = await persistCheckpoint( + resolvedSessionDir, + marker, + checkpoint, + options + ); + return { + sessionDir: resolvedSessionDir, + records: orderedRecords, + changes, + activities, + diagnostics: parsedDelta.diagnostics, + sources, + resets, + checkpoint, + checkpointStatus, + stateComplete, + scanStatus: combineGrokScanStatus( + deltas.updates.scanStatus, + deltas.events.scanStatus + ), + }; +} + +function combineGrokScanStatus( + updates: JsonlScanStatus, + events: JsonlScanStatus +): JsonlScanStatus { + if (updates.status === 'limited') return updates; + if (events.status === 'limited') return events; + return { status: 'complete' }; +} + +function grokCursorOptions(options: GrokSessionTailOptions): + | { + readonly maxLineBytes?: number; + readonly maxScanBytes?: number; + readonly maxScanLines?: number; + } + | undefined { + if ( + options.maxLineBytes === undefined && + options.maxScanBytes === undefined && + options.maxScanLines === undefined + ) { + return undefined; + } + return { + ...(options.maxLineBytes === undefined + ? {} + : { maxLineBytes: options.maxLineBytes }), + ...(options.maxScanBytes === undefined + ? {} + : { maxScanBytes: options.maxScanBytes }), + ...(options.maxScanLines === undefined + ? {} + : { maxScanLines: options.maxScanLines }), + }; +} + +function grokRebuildCursorOptions(options: GrokSessionTailOptions): { + readonly maxLineBytes?: number; + readonly maxScanBytes: number; + readonly maxScanLines: number; +} { + return { + ...(options.maxLineBytes === undefined + ? {} + : { maxLineBytes: options.maxLineBytes }), + maxScanBytes: Number.MAX_SAFE_INTEGER, + maxScanLines: Number.MAX_SAFE_INTEGER, + }; +} + +async function persistCheckpoint( + sessionDir: string, + marker: Awaited>, + checkpoint: GrokSessionCheckpoint, + options: GrokSessionTailOptions +): Promise { + if (options.checkpointMode === 'manual') return { status: 'manual' }; + if (!shouldCommitGrokMarker(marker, checkpoint)) + return { status: 'unchanged' }; + try { + await commitGrokSessionCheckpoint(sessionDir, checkpoint, options); + return { status: 'committed' }; + } catch (error: unknown) { + return { + status: 'failed', + error: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/src/grok/processing/tail-types.ts b/src/grok/processing/tail-types.ts new file mode 100644 index 0000000..7d488b9 --- /dev/null +++ b/src/grok/processing/tail-types.ts @@ -0,0 +1,156 @@ +import type { + JsonlCursor, + JsonlScanStatus, +} from '../../internal/jsonl-cursor.js'; +import type { + GrokActivity, + GrokBlockChange, + GrokNormalizedRecord, +} from './blocks.js'; + +/** Native JSONL filenames keyed by Grok source kind. */ +export const GROK_SOURCE_FILENAMES = { + updates: 'updates.jsonl', + events: 'events.jsonl', +} as const; + +/** A persisted Grok session source. */ +export type GrokTailSourceKind = keyof typeof GROK_SOURCE_FILENAMES; + +/** Options shared by Grok session tail and watch operations. */ +export interface GrokSessionTailOptions { + readonly markerDir?: string; + readonly allowedMarkerRoots?: readonly string[]; + readonly fromStart?: boolean; + readonly checkpointMode?: 'automatic' | 'manual'; + readonly maxLineBytes?: number; + readonly maxScanBytes?: number; + readonly maxScanLines?: number; + readonly includeActivities?: boolean; + readonly checkpoint?: GrokSessionCheckpoint; +} + +/** + * Injectable time source driving the optional poll backstop. + * + * Production callers omit this and receive real timers. Tests inject a clock + * so poll delays never appear as real sleeps. + */ +export interface GrokWatchClock { + /** Current time in milliseconds. */ + readonly now: () => number; + /** Schedule `handler` after `delayMs`; returns a cancellable timer handle. */ + readonly setTimeout: ( + handler: () => void, + delayMs: number + ) => NodeJS.Timeout | number; + /** Cancel a handle previously returned by `setTimeout`. */ + readonly clearTimeout: (handle: NodeJS.Timeout | number) => void; +} + +/** Options for watching a Grok session directory. */ +export interface GrokSessionWatchOptions extends GrokSessionTailOptions { + readonly signal?: AbortSignal; + /** + * Optional wake-up backstop interval in milliseconds on the injected clock. + * When set, a repeating timer periodically triggers a reconcile so progress + * never depends on filesystem event delivery latency or loss. Omitted or + * zero disables polling. + */ + readonly pollMs?: number; + /** Clock for the poll backstop; defaults to real timers. */ + readonly clock?: GrokWatchClock; +} + +/** Marker controls accepted by manual checkpoint commits. */ +export interface GrokSessionCheckpointCommitOptions { + readonly markerDir?: string; + readonly allowedMarkerRoots?: readonly string[]; +} + +/** Serializable cursor state for one Grok session source. */ +export interface GrokSessionSourceCheckpoint { + readonly sourceKind: GrokTailSourceKind; + readonly cursor: JsonlCursor | null; +} + +/** Adapter-local semantic state retained alongside neutral source cursors. */ +export interface GrokSessionCheckpointState { + readonly records: readonly GrokTailRecord[]; +} + +/** Revision-bound checkpoint returned by a successful two-source read. */ +export interface GrokSessionCheckpoint { + readonly sessionPathDigest: string; + readonly baseRevision: number; + readonly sources: readonly GrokSessionSourceCheckpoint[]; + readonly state?: GrokSessionCheckpointState; +} + +/** One parsed, ordered record emitted by a Grok session tail. */ +export interface GrokTailRecord { + readonly sourceKind: GrokTailSourceKind; + readonly effectiveTimestamp: number; + readonly nativeType: string; + readonly generation: number; + readonly byteStart: number; + readonly byteEnd: number; + readonly record: GrokNormalizedRecord; +} + +/** A parse or cursor diagnostic tied to one physical source record. */ +export interface GrokTailDiagnostic { + readonly sourceKind: GrokTailSourceKind; + readonly kind: + | 'invalid_json' + | 'invalid_record' + | 'unknown_record' + | 'oversized'; + readonly lineNumber: number; + readonly byteStart: number; + readonly byteEnd: number; + readonly message: string; +} + +/** State reached for one source during a tail pass. */ +export interface GrokSourceTailResult { + readonly sourceKind: GrokTailSourceKind; + readonly sourcePath: string; + readonly status: 'read' | 'missing'; + readonly recordCount: number; + readonly generation: number; + readonly previousByteOffset: number; + readonly newByteOffset: number; + readonly fileSize: number | null; + readonly reset: boolean; +} + +/** Notification that a source was replaced, truncated, or rewritten. */ +export interface GrokSourceReset { + readonly type: 'source_reset'; + readonly sourceKind: GrokTailSourceKind; + readonly generation: number; +} + +/** Outcome of checkpoint handling after a successful two-source read. */ +export type GrokCheckpointStatus = + | { readonly status: 'committed' } + | { readonly status: 'unchanged' } + | { readonly status: 'manual' } + | { readonly status: 'failed'; readonly error: string }; + +/** Result of one atomic two-source Grok session read. */ +export interface GrokSessionTailResult { + readonly sessionDir: string; + readonly records: readonly GrokTailRecord[]; + readonly changes: readonly GrokBlockChange[]; + readonly activities: readonly GrokActivity[]; + readonly diagnostics: readonly GrokTailDiagnostic[]; + readonly sources: readonly GrokSourceTailResult[]; + readonly resets: readonly GrokSourceReset[]; + readonly checkpoint: GrokSessionCheckpoint; + readonly checkpointStatus: GrokCheckpointStatus; + /** Whether semantic checkpoint state was reconstructed from complete source reads. */ + readonly stateComplete: boolean; + readonly scanStatus: JsonlScanStatus; +} diff --git a/src/grok/processing/tail-watch.ts b/src/grok/processing/tail-watch.ts new file mode 100644 index 0000000..b06b8d3 --- /dev/null +++ b/src/grok/processing/tail-watch.ts @@ -0,0 +1,159 @@ +import { watch } from 'node:fs'; +import { resolve } from 'node:path'; + +import { + createFileWatchScheduler, + type FileWatchScheduler, +} from '../../internal/watch-scheduler.js'; +import { isObservableGrokResult } from './tail-result.js'; +import { tailGrokSession } from './tail-run.js'; +import { + GROK_SOURCE_FILENAMES, + type GrokSessionTailOptions, + type GrokSessionTailResult, + type GrokSessionWatchOptions, + type GrokWatchClock, +} from './tail-types.js'; + +const defaultClock: GrokWatchClock = { + now: () => Date.now(), + setTimeout: (handler, delayMs) => setTimeout(handler, delayMs), + clearTimeout: handle => clearTimeout(handle), +}; + +/** + * Watch both Grok JSONL sources and yield observable reconciled batches. + * + * Filesystem events are wake-up hints only. A null `filename` still wakes + * reconcile; only a non-null name that is neither `updates.jsonl` nor + * `events.jsonl` is ignored. An optional `pollMs` backstop reconciles on + * the injected clock even when filesystem hints are delayed or lost. + * + * @param sessionDir - Directory containing both sources. + * @param options - Tail options, cancellation signal, and optional poll backstop. + * @returns Initial readiness result followed by changed batches. + */ +export async function* watchGrokSession( + sessionDir: string, + options: GrokSessionWatchOptions = {} +): AsyncGenerator { + const resolvedSessionDir = resolve(sessionDir); + const { + signal, + pollMs: _pollMs, + clock: optionsClock, + ...initialTailOptions + } = options; + const clock = optionsClock ?? defaultClock; + const pollIntervalMs = options.pollMs ?? 0; + let tailOptions: GrokSessionTailOptions = initialTailOptions; + let changed = false; + let aborted = false; + let wake: (() => void) | undefined; + let watchError: Error | undefined; + let pollHandle: NodeJS.Timeout | number | null = null; + const flushWake = (): void => { + const resolveWake = wake; + wake = undefined; + resolveWake?.(); + }; + const wakeScheduler: FileWatchScheduler = + optionsClock === undefined + ? createFileWatchScheduler(flushWake, { + schedule(handler): undefined { + queueMicrotask(handler); + return undefined; + }, + }) + : createFileWatchScheduler(flushWake, { + schedule: handler => clock.setTimeout(handler, 0), + cancel: handle => clock.clearTimeout(handle), + }); + const requestWake = (): void => { + changed = true; + if (wake !== undefined) wakeScheduler.request(); + }; + const disarmPoll = (): void => { + if (pollHandle !== null) { + clock.clearTimeout(pollHandle); + pollHandle = null; + } + }; + const armPoll = (): void => { + if (pollIntervalMs <= 0 || aborted || pollHandle !== null) return; + pollHandle = clock.setTimeout(() => { + pollHandle = null; + requestWake(); + armPoll(); + }, pollIntervalMs); + }; + const watcher = watch(resolvedSessionDir, (_eventType, filename) => { + if (filename !== null && filename !== undefined) { + const name = filename.toString(); + if ( + name !== GROK_SOURCE_FILENAMES.updates && + name !== GROK_SOURCE_FILENAMES.events + ) + return; + } + requestWake(); + }); + watcher.on('error', error => { + watchError = error; + changed = true; + const resolveWake = wake; + wake = undefined; + resolveWake?.(); + }); + const abort = (): void => { + aborted = true; + watcher.close(); + disarmPoll(); + const resolveWake = wake; + wake = undefined; + resolveWake?.(); + }; + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted === true) abort(); + armPoll(); + try { + const initialResult = await tailGrokSession( + resolvedSessionDir, + tailOptions + ); + const { + fromStart: _fromStart, + checkpoint: _checkpoint, + ...remainingOptions + } = tailOptions; + tailOptions = { ...remainingOptions, checkpoint: initialResult.checkpoint }; + yield initialResult; + while (signal?.aborted !== true && !aborted) { + if (!changed) { + await new Promise(resolveWake => { + wake = resolveWake; + if (changed || isAborted(signal) || aborted) { + wake = undefined; + resolveWake(); + } + }); + } + if (isAborted(signal) || aborted) return; + if (watchError !== undefined) throw watchError; + changed = false; + const result = await tailGrokSession(resolvedSessionDir, tailOptions); + tailOptions = { ...tailOptions, checkpoint: result.checkpoint }; + if (isObservableGrokResult(result)) yield result; + } + } finally { + aborted = true; + signal?.removeEventListener('abort', abort); + disarmPoll(); + wakeScheduler.cancel(); + watcher.close(); + } +} + +function isAborted(signal: AbortSignal | undefined): boolean { + return signal?.aborted === true; +} diff --git a/src/grok/processing/tail.ts b/src/grok/processing/tail.ts index 5c92d7a..11b361f 100644 --- a/src/grok/processing/tail.ts +++ b/src/grok/processing/tail.ts @@ -1,1049 +1,29 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { watch } from 'node:fs'; -import { - mkdir, - open, - readFile, - rename, - rm, - stat, - unlink, -} from 'node:fs/promises'; -import { basename, dirname, join, resolve, sep } from 'node:path'; - -import { - reduceGrokRecords, - type GrokActivity, - type GrokBlockChange, - type GrokNormalizedRecord, - type GrokRecordOrigin, -} from './blocks.js'; -import { parseGrokEvent } from './events.js'; -import { - readJsonlDelta, - type JsonlCursor, - type JsonlDelta, - type JsonlLine, -} from './jsonl-cursor.js'; -import { parseGrokSessionUpdate } from './updates.js'; - -const MARKER_VERSION = 1; -const STALE_MARKER_LOCK_MS = 30_000; -const SOURCE_FILENAMES = { - updates: 'updates.jsonl', - events: 'events.jsonl', -} as const; - -/** A persisted Grok session source. */ -export type GrokTailSourceKind = keyof typeof SOURCE_FILENAMES; - -/** Options shared by Grok session tail and watch operations. */ -export interface GrokSessionTailOptions { - /** Marker directory, defaulting to `/.tail-markers`. */ - readonly markerDir?: string; - /** Roots allowed to contain a custom marker directory. */ - readonly allowedMarkerRoots?: readonly string[]; - /** Ignore saved cursors and scan both sources from byte zero. */ - readonly fromStart?: boolean; - /** Persist on successful tail or defer persistence to an explicit commit. */ - readonly checkpointMode?: 'automatic' | 'manual'; - /** Maximum bytes retained for one JSONL line. */ - readonly maxLineBytes?: number; - /** Include reduced event activity states, defaulting to true. */ - readonly includeActivities?: boolean; -} - -/** Options for watching a Grok session directory. */ -export interface GrokSessionWatchOptions extends GrokSessionTailOptions { - /** Ends observation and closes the underlying filesystem watcher. */ - readonly signal?: AbortSignal; -} - -/** Marker controls accepted by manual checkpoint commits. */ -export interface GrokSessionCheckpointCommitOptions { - /** Marker directory, defaulting to `/.tail-markers`. */ - readonly markerDir?: string; - /** Roots allowed to contain a custom marker directory. */ - readonly allowedMarkerRoots?: readonly string[]; -} - -/** Serializable cursor state for one Grok session source. */ -export interface GrokSessionSourceCheckpoint { - readonly sourceKind: GrokTailSourceKind; - readonly cursor: JsonlCursor | null; -} - -/** Revision-bound checkpoint returned by a successful two-source read. */ -export interface GrokSessionCheckpoint { - readonly sessionPathDigest: string; - readonly baseRevision: number; - readonly sources: readonly GrokSessionSourceCheckpoint[]; -} - -/** One parsed, ordered record emitted by a Grok session tail. */ -export interface GrokTailRecord { - readonly sourceKind: GrokTailSourceKind; - readonly effectiveTimestamp: number; - readonly nativeType: string; - readonly generation: number; - readonly byteStart: number; - readonly byteEnd: number; - readonly record: GrokNormalizedRecord; -} - -/** A parse or cursor diagnostic tied to one physical source record. */ -export interface GrokTailDiagnostic { - readonly sourceKind: GrokTailSourceKind; - readonly kind: - | 'invalid_json' - | 'invalid_record' - | 'unknown_record' - | 'oversized'; - readonly lineNumber: number; - readonly byteStart: number; - readonly byteEnd: number; - readonly message: string; -} - -/** State reached for one source during a tail pass. */ -export interface GrokSourceTailResult { - readonly sourceKind: GrokTailSourceKind; - readonly sourcePath: string; - readonly status: 'read' | 'missing'; - readonly recordCount: number; - readonly generation: number; - readonly previousByteOffset: number; - readonly newByteOffset: number; - readonly fileSize: number | null; - readonly reset: boolean; -} - -/** Notification that a source was replaced, truncated, or rewritten. */ -export interface GrokSourceReset { - readonly type: 'source_reset'; - readonly sourceKind: GrokTailSourceKind; - readonly generation: number; -} - -/** Outcome of checkpoint handling after a successful two-source read. */ -export type GrokCheckpointStatus = - | { readonly status: 'committed' } - | { readonly status: 'unchanged' } - | { readonly status: 'manual' } - | { readonly status: 'failed'; readonly error: string }; - -/** Result of one atomic two-source Grok session read. */ -export interface GrokSessionTailResult { - readonly sessionDir: string; - readonly records: readonly GrokTailRecord[]; - readonly changes: readonly GrokBlockChange[]; - readonly activities: readonly GrokActivity[]; - readonly diagnostics: readonly GrokTailDiagnostic[]; - readonly sources: readonly GrokSourceTailResult[]; - readonly resets: readonly GrokSourceReset[]; - readonly checkpoint: GrokSessionCheckpoint; - /** - * Automatic persistence outcome for this pass. A `failed` status leaves the - * saved marker unchanged, so the next call replays this batch and can commit - * it after marker storage becomes writable. Manual commits still reject. - */ - readonly checkpointStatus: GrokCheckpointStatus; -} - -interface GrokSessionMarker { - readonly version: 1; - readonly sessionPathDigest: string; - readonly revision: number; - readonly sources: Readonly>; -} - -interface ParsedSource { - readonly records: readonly GrokTailRecord[]; - readonly diagnostics: readonly GrokTailDiagnostic[]; -} - -interface ParsedLine { - readonly record?: GrokTailRecord; - readonly diagnostic?: GrokTailDiagnostic; -} - -class StaleGrokSessionCheckpointError extends Error {} - -/** - * Tail updates.jsonl and events.jsonl as one revisioned session stream. - * - * Both size-snapshotted source reads must succeed before the checkpoint can be - * committed. A missing events.jsonl is represented by a `missing` source; a - * missing updates.jsonl is an error. Complete malformed records advance their - * source cursor and are reported as diagnostics. Unknown tags are preserved as - * native records and also reported as diagnostics. - * - * Automatic checkpoint failures do not discard a successfully read batch. - * They return `checkpointStatus: { status: 'failed', error }`, leave the saved - * marker unchanged, and cause the next call to replay the batch. Explicit - * `commitGrokSessionCheckpoint` failures reject. - * - * @param sessionDir - Directory containing Grok's persisted session files. - * @param options - Cursor, marker, reduction, and line-size controls. - * @returns Ordered records, normalized changes, diagnostics, and checkpoint. - * @throws If either source read fails. - */ -export async function tailGrokSession( - sessionDir: string, - options: GrokSessionTailOptions = {} -): Promise { - const resolvedSessionDir = resolve(sessionDir); - const sessionPathDigest = createSessionPathDigest(resolvedSessionDir); - const markerPath = getGrokSessionMarkerPath(resolvedSessionDir, options); - const marker = await readGrokSessionMarker(markerPath, sessionPathDigest); - const cursorOptions = - options.maxLineBytes === undefined - ? undefined - : { maxLineBytes: options.maxLineBytes }; - - const markerCursors = { - updates: marker?.sources.updates ?? null, - events: marker?.sources.events ?? null, - } satisfies Record; - const previousCursors = { - updates: options.fromStart ? null : markerCursors.updates, - events: options.fromStart ? null : markerCursors.events, - } satisfies Record; - const updatePath = join(resolvedSessionDir, SOURCE_FILENAMES.updates); - const eventPath = join(resolvedSessionDir, SOURCE_FILENAMES.events); - - const updateDelta = await readJsonlDelta( - updatePath, - previousCursors.updates, - cursorOptions - ); - if (updateDelta.fileSize === null) { - throw new Error(`Missing required Grok updates source '${updatePath}'`); - } - const eventDelta = await readJsonlDelta( - eventPath, - previousCursors.events, - cursorOptions - ); - - const deltas = { - updates: applyFromStartGeneration( - updateDelta, - markerCursors.updates, - options.fromStart - ), - events: applyFromStartGeneration( - eventDelta, - markerCursors.events, - options.fromStart - ), - } as const; - const parsedDelta = parseSources(deltas); - const orderedRecords = [...parsedDelta.records].sort(compareTailRecords); - const deltaOrigins = new Set( - orderedRecords.map(record => originKey(record.record.origin)) - ); - - let reductionRecords: readonly GrokNormalizedRecord[] = orderedRecords.map( - record => record.record - ); - if (orderedRecords.length > 0 && hasPriorCommittedBytes(previousCursors)) { - const [allUpdates, allEvents] = await Promise.all([ - readJsonlDelta(updatePath, null, cursorOptions), - readJsonlDelta(eventPath, null, cursorOptions), - ]); - if (allUpdates.fileSize === null) { - throw new Error(`Missing required Grok updates source '${updatePath}'`); - } - const fullParsed = parseSources( - { updates: allUpdates, events: allEvents }, - { - updates: updateDelta.cursor?.generation ?? 0, - events: eventDelta.cursor?.generation ?? 0, - } - ); - reductionRecords = [...fullParsed.records] - .sort(compareTailRecords) - .map(record => record.record); - } - - const reduction = reduceGrokRecords(reductionRecords); - const changes = reduction.changes.filter(change => - deltaOrigins.has( - originKey(change.type === 'upsert' ? change.block.origin : change.origin) - ) - ); - const activities = - options.includeActivities === false - ? [] - : reduction.activities.filter(activity => - deltaOrigins.has(originKey(activity.origin)) - ); - const checkpoint: GrokSessionCheckpoint = { - sessionPathDigest, - baseRevision: marker?.revision ?? 0, - sources: sourceKinds().map(sourceKind => ({ - sourceKind, - cursor: deltas[sourceKind].cursor, - })), - }; - const sources = sourceKinds().map(sourceKind => - sourceResult( - sourceKind, - join(resolvedSessionDir, SOURCE_FILENAMES[sourceKind]), - previousCursors[sourceKind], - deltas[sourceKind], - parsedDelta.records - ) - ); - const resets: GrokSourceReset[] = sources - .filter(source => source.reset) - .map(source => ({ - type: 'source_reset', - sourceKind: source.sourceKind, - generation: source.generation, - })); - - let checkpointStatus: GrokCheckpointStatus; - if (options.checkpointMode === 'manual') { - checkpointStatus = { status: 'manual' }; - } else if (!shouldCommitMarker(marker, checkpoint)) { - checkpointStatus = { status: 'unchanged' }; - } else { - try { - await commitGrokSessionCheckpoint( - resolvedSessionDir, - checkpoint, - options - ); - checkpointStatus = { status: 'committed' }; - } catch (error: unknown) { - checkpointStatus = { - status: 'failed', - error: error instanceof Error ? error.message : String(error), - }; - } - } - - return { - sessionDir: resolvedSessionDir, - records: orderedRecords, - changes, - activities, - diagnostics: parsedDelta.diagnostics, - sources, - resets, - checkpoint, - checkpointStatus, - }; -} - -/** - * Commit a checkpoint after its emitted changes have been durably consumed. - * - * The checkpoint is accepted only for the same resolved session path and base - * revision. Source offsets cannot move backwards without one generation step. - * - * @param sessionDir - Session directory used to produce the checkpoint. - * @param checkpoint - Checkpoint returned by `tailGrokSession`. - * @param options - Marker destination and root allow-list. - * @returns After the marker has been atomically replaced. - * @throws If the checkpoint is stale, malformed, unsafe, or for another path. - */ -export async function commitGrokSessionCheckpoint( - sessionDir: string, - checkpoint: GrokSessionCheckpoint, - options: GrokSessionCheckpointCommitOptions = {} -): Promise { - const resolvedSessionDir = resolve(sessionDir); - const sessionPathDigest = createSessionPathDigest(resolvedSessionDir); - if (checkpoint.sessionPathDigest !== sessionPathDigest) { - throw new Error('Grok session checkpoint does not match the session path'); - } - const nextSources = checkpointSources(checkpoint); - const markerPath = getGrokSessionMarkerPath(resolvedSessionDir, options); - await withMarkerLock(markerPath, async () => { - const marker = await readGrokSessionMarker(markerPath, sessionPathDigest); - const revision = marker?.revision ?? 0; - if (checkpoint.baseRevision !== revision) { - throw new StaleGrokSessionCheckpointError( - 'Grok session checkpoint is stale for the current marker' - ); - } - validateCheckpointProgression(marker, nextSources); - await writePrivateJson(markerPath, { - version: MARKER_VERSION, - sessionPathDigest, - revision: revision + 1, - sources: nextSources, - } satisfies GrokSessionMarker); - }); -} - -/** - * Watch updates.jsonl and events.jsonl and yield successful non-empty passes. - * - * Native filesystem callbacks are coalesced within one event-loop turn. No - * polling interval is used. The first `next()` yields an initial pass after - * filesystem observation is active, giving callers a deterministic readiness - * handshake. Aborting or closing iteration releases the watcher. `fromStart` - * applies only to the initial pass. - * - * @param sessionDir - Directory containing the two Grok JSONL sources. - * @param options - Tail options plus an optional cancellation signal. - * @returns An async sequence of changed session batches. - */ -export async function* watchGrokSession( - sessionDir: string, - options: GrokSessionWatchOptions = {} -): AsyncGenerator { - const resolvedSessionDir = resolve(sessionDir); - const { signal, ...initialTailOptions } = options; - let tailOptions: GrokSessionTailOptions = initialTailOptions; - let changed = false; - let wake: (() => void) | undefined; - let queued = false; - let watchError: Error | undefined; - - const watcher = watch(resolvedSessionDir, (_eventType, filename) => { - const name = filename?.toString(); - if (name !== SOURCE_FILENAMES.updates && name !== SOURCE_FILENAMES.events) { - return; - } - changed = true; - if (wake === undefined || queued) return; - queued = true; - queueMicrotask(() => { - queued = false; - const resolveWake = wake; - wake = undefined; - resolveWake?.(); - }); - }); - watcher.on('error', error => { - watchError = error; - changed = true; - const resolveWake = wake; - wake = undefined; - resolveWake?.(); - }); - const abort = (): void => { - watcher.close(); - const resolveWake = wake; - wake = undefined; - resolveWake?.(); - }; - signal?.addEventListener('abort', abort, { once: true }); - - try { - const initialResult = await tailGrokSession( - resolvedSessionDir, - tailOptions - ); - if (tailOptions.fromStart === true) { - const { fromStart: _fromStart, ...remainingOptions } = tailOptions; - tailOptions = remainingOptions; - } - yield initialResult; - - while (signal?.aborted !== true) { - if (!changed) { - await new Promise(resolveWake => { - wake = resolveWake; - if (changed || signal?.aborted === true) { - wake = undefined; - resolveWake(); - } - }); - } - if (isAborted(signal)) return; - if (watchError !== undefined) throw watchError; - changed = false; - const result = await tailGrokSession(resolvedSessionDir, tailOptions); - if (isObservableResult(result)) yield result; - } - } finally { - signal?.removeEventListener('abort', abort); - watcher.close(); - } -} - -function parseSources( - deltas: Readonly>, - generations?: Readonly> -): ParsedSource { - const records: GrokTailRecord[] = []; - const diagnostics: GrokTailDiagnostic[] = []; - for (const sourceKind of sourceKinds()) { - const delta = deltas[sourceKind]; - const generation = - generations?.[sourceKind] ?? delta.cursor?.generation ?? 0; - for (const diagnostic of delta.diagnostics) { - diagnostics.push({ - sourceKind, - kind: 'oversized', - lineNumber: diagnostic.lineNumber, - byteStart: diagnostic.byteStart, - byteEnd: diagnostic.byteEnd, - message: 'JSONL line exceeds maxLineBytes', - }); - } - for (const line of delta.lines) { - const parsed = parseLine(sourceKind, generation, line); - if (parsed.record !== undefined) records.push(parsed.record); - if (parsed.diagnostic !== undefined) { - diagnostics.push(parsed.diagnostic); - } - } - } - return { records, diagnostics }; -} - -function parseLine( - sourceKind: GrokTailSourceKind, - generation: number, - line: JsonlLine -): ParsedLine { - let raw: unknown; - try { - raw = JSON.parse(line.value) as unknown; - } catch (error: unknown) { - return { - diagnostic: lineDiagnostic( - sourceKind, - line, - 'invalid_json', - error instanceof Error ? error.message : String(error) - ), - }; - } - - if (sourceKind === 'updates') { - const parsed = parseGrokSessionUpdate(raw); - if (parsed.kind === 'unknown') { - return unknownParsedLine( - sourceKind, - generation, - line, - parsed.tag, - parsed.raw - ); - } - if (parsed.kind !== 'known') { - return { - diagnostic: lineDiagnostic( - sourceKind, - line, - 'invalid_record', - parsed.error - ), - }; - } - const nativeType = parsed.envelope.params.update.sessionUpdate; - const origin = createOrigin(sourceKind, nativeType, generation, line); - const record: GrokNormalizedRecord = { - kind: 'update', - envelope: parsed.envelope, - origin, - }; - return { - record: { - sourceKind, - effectiveTimestamp: updateTimestamp(parsed.envelope), - nativeType, - generation, - byteStart: line.byteStart, - byteEnd: line.byteEnd, - record, - }, - }; - } - - const parsed = parseGrokEvent(raw); - if (parsed.kind === 'unknown') { - return unknownParsedLine( - sourceKind, - generation, - line, - parsed.tag, - parsed.raw - ); - } - if (parsed.kind !== 'known') { - return { - diagnostic: lineDiagnostic( - sourceKind, - line, - 'invalid_record', - parsed.error - ), - }; - } - const nativeType = parsed.event.type; - const origin = createOrigin(sourceKind, nativeType, generation, line); - const record: GrokNormalizedRecord = { - kind: 'event', - event: parsed.event, - origin, - }; - const parsedTimestamp = Date.parse(parsed.event.ts); - return { - record: { - sourceKind, - effectiveTimestamp: Number.isFinite(parsedTimestamp) - ? parsedTimestamp - : 0, - nativeType, - generation, - byteStart: line.byteStart, - byteEnd: line.byteEnd, - record, - }, - }; -} - -function unknownParsedLine( - sourceKind: GrokTailSourceKind, - generation: number, - line: JsonlLine, - tag: string, - raw: unknown -): ParsedLine { - const origin = createOrigin(sourceKind, tag, generation, line); - const record: GrokNormalizedRecord = { - kind: 'unknown', - tag, - raw, - origin, - }; - return { - record: { - sourceKind, - effectiveTimestamp: unknownRecordTimestamp(sourceKind, raw), - nativeType: tag, - generation, - byteStart: line.byteStart, - byteEnd: line.byteEnd, - record, - }, - diagnostic: lineDiagnostic( - sourceKind, - line, - 'unknown_record', - sourceKind === 'updates' - ? `Unknown update '${tag}'` - : `Unknown event '${tag}'` - ), - }; -} - -function createOrigin( - sourceKind: GrokTailSourceKind, - nativeType: string, - generation: number, - line: JsonlLine -): GrokRecordOrigin { - return { - harness: 'grok', - stream: sourceKind === 'updates' ? 'conversation' : 'activity', - sourceId: sourceKind, - nativeType, - generation, - byteStart: line.byteStart, - byteEnd: line.byteEnd, - }; -} - -function lineDiagnostic( - sourceKind: GrokTailSourceKind, - line: JsonlLine, - kind: GrokTailDiagnostic['kind'], - message: string -): GrokTailDiagnostic { - return { - sourceKind, - kind, - lineNumber: line.lineNumber, - byteStart: line.byteStart, - byteEnd: line.byteEnd, - message, - }; -} - -function unknownRecordTimestamp( - sourceKind: GrokTailSourceKind, - raw: unknown -): number { - if (!isRecord(raw)) return 0; - if (sourceKind === 'updates') { - const timestamp = raw['timestamp']; - if (typeof timestamp === 'number' && Number.isFinite(timestamp)) { - return Math.abs(timestamp) < 100_000_000_000 - ? timestamp * 1_000 - : timestamp; - } - return 0; - } - const timestamp = raw['ts']; - if (typeof timestamp !== 'string') return 0; - const parsed = Date.parse(timestamp); - return Number.isFinite(parsed) ? parsed : 0; -} - -function updateTimestamp( - envelope: Extract['envelope'] -): number { - const meta = envelope.params._meta; - if (typeof meta === 'object' && meta !== null) { - const value = Reflect.get(meta, 'agentTimestampMs') as unknown; - if (typeof value === 'number' && Number.isFinite(value)) return value; - } - return Math.abs(envelope.timestamp) < 100_000_000_000 - ? envelope.timestamp * 1_000 - : envelope.timestamp; -} - -function compareTailRecords( - left: GrokTailRecord, - right: GrokTailRecord -): number { - if (left.effectiveTimestamp !== right.effectiveTimestamp) { - return left.effectiveTimestamp < right.effectiveTimestamp ? -1 : 1; - } - const sourceDifference = - sourceRank(left.sourceKind) - sourceRank(right.sourceKind); - if (sourceDifference !== 0) return sourceDifference; - if (left.generation !== right.generation) { - return left.generation < right.generation ? -1 : 1; - } - if (left.byteStart !== right.byteStart) { - return left.byteStart < right.byteStart ? -1 : 1; - } - return left.byteEnd - right.byteEnd; -} - -function sourceRank(sourceKind: GrokTailSourceKind): number { - return sourceKind === 'updates' ? 0 : 1; -} - -function sourceKinds(): readonly GrokTailSourceKind[] { - return ['updates', 'events']; -} - -function sourceResult( - sourceKind: GrokTailSourceKind, - sourcePath: string, - previousCursor: JsonlCursor | null, - delta: JsonlDelta, - records: readonly GrokTailRecord[] -): GrokSourceTailResult { - return { - sourceKind, - sourcePath, - status: delta.fileSize === null ? 'missing' : 'read', - recordCount: records.filter(record => record.sourceKind === sourceKind) - .length, - generation: delta.cursor?.generation ?? previousCursor?.generation ?? 0, - previousByteOffset: previousCursor?.offset ?? 0, - newByteOffset: delta.cursor?.offset ?? previousCursor?.offset ?? 0, - fileSize: delta.fileSize, - reset: delta.reset, - }; -} - -function createSessionPathDigest(sessionDir: string): string { - return createHash('sha256').update(resolve(sessionDir)).digest('hex'); -} - -function getGrokSessionMarkerPath( - sessionDir: string, - options: GrokSessionCheckpointCommitOptions -): string { - const markerDir = - options.markerDir === undefined - ? resolve(sessionDir, '.tail-markers') - : resolveAllowedMarkerDir(options.markerDir, options.allowedMarkerRoots); - const digest = createSessionPathDigest(sessionDir); - const sessionName = sanitizeMarkerBase(basename(sessionDir)); - return join( - markerDir, - `${sessionName}-${digest.slice(0, 16)}.grok-session.json` - ); -} - -function resolveAllowedMarkerDir( - markerDir: string, - allowedMarkerRoots?: readonly string[] -): string { - const resolvedDir = resolve(markerDir); - const roots = (allowedMarkerRoots ?? []) - .map(root => root.trim()) - .filter(root => root.length > 0) - .map(root => resolve(root)); - if (roots.length === 0) { - throw new Error( - 'Custom markerDir requires allowedMarkerRoots to include an allowed root' - ); - } - if (!roots.some(root => isWithinPath(resolvedDir, root))) { - throw new Error( - `Marker directory '${resolvedDir}' is outside allowed marker roots` - ); - } - return resolvedDir; -} - -function isWithinPath(child: string, parent: string): boolean { - const prefix = parent.endsWith(sep) ? parent : `${parent}${sep}`; - return child === parent || child.startsWith(prefix); -} - -function sanitizeMarkerBase(raw: string): string { - const sanitized = raw - .replace(/[^A-Za-z0-9._-]+/g, '-') - .replace(/^-+|-+$/g, ''); - return sanitized.length === 0 || sanitized === '.' || sanitized === '..' - ? 'session' - : sanitized; -} - -async function readGrokSessionMarker( - markerPath: string, - sessionPathDigest: string -): Promise { - try { - const parsed: unknown = JSON.parse(await readFile(markerPath, 'utf8')); - if (!isRecord(parsed) || parsed['version'] !== MARKER_VERSION) return null; - if (parsed['sessionPathDigest'] !== sessionPathDigest) return null; - const revision = parsed['revision']; - const sources = parsed['sources']; - if (!isSafeNonnegativeInteger(revision) || !isRecord(sources)) return null; - const updates = parseCursor(sources['updates']); - const events = parseCursor(sources['events']); - if (updates === undefined || events === undefined) return null; - return { - version: MARKER_VERSION, - sessionPathDigest, - revision, - sources: { updates, events }, - }; - } catch { - return null; - } -} - -function parseCursor(value: unknown): JsonlCursor | null | undefined { - if (value === null) return null; - if (!isRecord(value)) return undefined; - if ( - typeof value['device'] !== 'string' || - typeof value['inode'] !== 'string' || - !isSafeNonnegativeInteger(value['offset']) || - !isSafeNonnegativeInteger(value['lineNumber']) || - value['lineNumber'] < 1 || - !isSafeNonnegativeInteger(value['generation']) || - typeof value['headDigest'] !== 'string' || - typeof value['boundaryDigest'] !== 'string' - ) { - return undefined; - } - return { - device: value['device'], - inode: value['inode'], - offset: value['offset'], - lineNumber: value['lineNumber'], - generation: value['generation'], - headDigest: value['headDigest'], - boundaryDigest: value['boundaryDigest'], - }; -} - -function checkpointSources( - checkpoint: GrokSessionCheckpoint -): Record { - if (!isSafeNonnegativeInteger(checkpoint.baseRevision)) { - throw new Error('Invalid Grok session checkpoint revision'); - } - const sources: Partial> = {}; - for (const source of checkpoint.sources) { - if (source.sourceKind !== 'updates' && source.sourceKind !== 'events') { - throw new Error('Invalid Grok session checkpoint source'); - } - if (Object.hasOwn(sources, source.sourceKind)) { - throw new Error('Grok session checkpoint has duplicate sources'); - } - if (source.cursor !== null && parseCursor(source.cursor) === undefined) { - throw new Error('Invalid Grok session checkpoint cursor'); - } - sources[source.sourceKind] = source.cursor; - } - if (!Object.hasOwn(sources, 'updates') || !Object.hasOwn(sources, 'events')) { - throw new Error('Grok session checkpoint must contain both sources'); - } - return { updates: sources.updates ?? null, events: sources.events ?? null }; -} - -function applyFromStartGeneration( - delta: JsonlDelta, - previousCursor: JsonlCursor | null, - fromStart: boolean | undefined -): JsonlDelta { - if (fromStart !== true || previousCursor === null || delta.cursor === null) { - return delta; - } - return { - ...delta, - cursor: { - ...delta.cursor, - generation: previousCursor.generation + 1, - }, - }; -} - -function validateCheckpointProgression( - marker: GrokSessionMarker | null, - next: Readonly> -): void { - for (const sourceKind of sourceKinds()) { - const previousCursor = marker?.sources[sourceKind] ?? null; - const nextCursor = next[sourceKind]; - if (previousCursor === null || nextCursor === null) continue; - if (nextCursor.generation === previousCursor.generation) { - if (nextCursor.offset < previousCursor.offset) { - throw new Error( - 'Grok session checkpoint would move a source backwards' - ); - } - } else if (nextCursor.generation !== previousCursor.generation + 1) { - throw new Error( - 'Grok session checkpoint has an invalid generation transition' - ); - } - } -} - -function shouldCommitMarker( - marker: GrokSessionMarker | null, - checkpoint: GrokSessionCheckpoint -): boolean { - const next = checkpointSources(checkpoint); - if (marker === null) return next.updates !== null || next.events !== null; - return sourceKinds().some( - sourceKind => !cursorsEqual(marker.sources[sourceKind], next[sourceKind]) - ); -} - -function cursorsEqual( - left: JsonlCursor | null, - right: JsonlCursor | null -): boolean { - if (left === null || right === null) return left === right; - return ( - left.device === right.device && - left.inode === right.inode && - left.offset === right.offset && - left.lineNumber === right.lineNumber && - left.generation === right.generation && - left.headDigest === right.headDigest && - left.boundaryDigest === right.boundaryDigest - ); -} - -async function withMarkerLock( - markerPath: string, - action: () => Promise -): Promise { - const lockPath = `${markerPath}.lock`; - await mkdir(dirname(markerPath), { recursive: true, mode: 0o700 }); - try { - await mkdir(lockPath, { mode: 0o700 }); - } catch (error: unknown) { - if (!hasErrorCode(error, 'EEXIST')) throw error; - if (!(await removeStaleMarkerLock(lockPath))) { - throw new Error(`Grok session marker is locked: '${markerPath}'`); - } - try { - await mkdir(lockPath, { mode: 0o700 }); - } catch (retryError: unknown) { - if (hasErrorCode(retryError, 'EEXIST')) { - throw new Error(`Grok session marker is locked: '${markerPath}'`); - } - throw retryError; - } - } - try { - return await action(); - } finally { - await rm(lockPath, { recursive: true, force: true }); - } -} - -async function removeStaleMarkerLock(lockPath: string): Promise { - try { - const stats = await stat(lockPath); - if (Date.now() - stats.mtimeMs <= STALE_MARKER_LOCK_MS) { - return false; - } - await rm(lockPath, { recursive: true, force: true }); - return true; - } catch { - return false; - } -} - -async function writePrivateJson(path: string, value: unknown): Promise { - await mkdir(dirname(path), { recursive: true, mode: 0o700 }); - const temporaryPath = join( - dirname(path), - `.${basename(path)}.${randomUUID()}.tmp` - ); - try { - const file = await open(temporaryPath, 'wx', 0o600); - try { - await file.writeFile(JSON.stringify(value, null, 2)); - await file.sync(); - } finally { - await file.close(); - } - await rename(temporaryPath, path); - } catch (error: unknown) { - await unlink(temporaryPath).catch(() => undefined); - throw error; - } -} - -function hasPriorCommittedBytes( - cursors: Readonly> -): boolean { - return sourceKinds().some( - sourceKind => (cursors[sourceKind]?.offset ?? 0) > 0 - ); -} - -function originKey(origin: GrokRecordOrigin): string { - return `${origin.sourceId}:${String(origin.generation)}:${String(origin.byteStart)}:${String(origin.byteEnd)}`; -} - -function isObservableResult(result: GrokSessionTailResult): boolean { - return ( - result.records.length > 0 || - result.diagnostics.length > 0 || - result.resets.length > 0 - ); -} - -function isAborted(signal: AbortSignal | undefined): boolean { - return signal?.aborted === true; -} - -function isSafeNonnegativeInteger(value: unknown): value is number { - return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function hasErrorCode(error: unknown, code: string): boolean { - return isRecord(error) && error['code'] === code; -} +/** Stale-revision error values shared by explicit checkpoint consumers. */ +export { + STALE_CHECKPOINT_CONFLICT_CODE, + StaleCheckpointConflict, + isStaleCheckpointConflict, +} from '../../processing/stale-checkpoint-conflict.js'; +/** Commit a revision-bound checkpoint after consumer durability. */ +export { commitGrokSessionCheckpoint } from './tail-marker.js'; +/** Read and reduce one bounded two-source Grok tail pass. */ +export { tailGrokSession } from './tail-run.js'; +/** Public checkpoint, record, diagnostic, source, and option contracts. */ +export { + type GrokCheckpointStatus, + type GrokSessionCheckpoint, + type GrokSessionCheckpointCommitOptions, + type GrokSessionCheckpointState, + type GrokSessionSourceCheckpoint, + type GrokSessionTailOptions, + type GrokSessionTailResult, + type GrokSessionWatchOptions, + type GrokWatchClock, + type GrokSourceReset, + type GrokSourceTailResult, + type GrokTailDiagnostic, + type GrokTailRecord, + type GrokTailSourceKind, +} from './tail-types.js'; +/** Watch Grok sources while retaining the latest bounded checkpoint. */ +export { watchGrokSession } from './tail-watch.js'; diff --git a/src/internal/bounded-file-read.ts b/src/internal/bounded-file-read.ts new file mode 100644 index 0000000..7cb1694 --- /dev/null +++ b/src/internal/bounded-file-read.ts @@ -0,0 +1,57 @@ +import { open } from 'node:fs/promises'; + +import { hasErrorCode } from './marker-store.js'; + +/** Outcome of a FileHandle read that never stats or loads past `maxBytes`. */ +export type BoundedFileReadResult = + | { readonly kind: 'missing' } + | { readonly kind: 'ok'; readonly text: string } + | { readonly kind: 'oversize' } + | { readonly kind: 'unreadable' }; + +/** + * Read a UTF-8 file using a FileHandle, stopping at `maxBytes`. + * + * The function never stats and never calls `readFile`. It reads at most + * `maxBytes + 1` bytes so an oversize file is detected without loading the + * remainder. Short reads are looped until EOF or the bound. + * + * @param path - File to read. + * @param maxBytes - Inclusive UTF-8 byte cap for a successful read. + * @returns Missing, ok text, oversize, or unreadable — contents errors do not throw. + */ +export async function readUtf8FileBounded( + path: string, + maxBytes: number +): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('maxBytes must be a safe non-negative integer'); + } + let file; + try { + file = await open(path, 'r'); + } catch (error: unknown) { + if (hasErrorCode(error, 'ENOENT')) return { kind: 'missing' }; + return { kind: 'unreadable' }; + } + try { + const buffer = Buffer.allocUnsafe(maxBytes + 1); + let total = 0; + while (total < maxBytes + 1) { + const { bytesRead } = await file.read( + buffer, + total, + maxBytes + 1 - total, + total + ); + if (bytesRead === 0) break; + total += bytesRead; + } + if (total > maxBytes) return { kind: 'oversize' }; + return { kind: 'ok', text: buffer.subarray(0, total).toString('utf8') }; + } catch { + return { kind: 'unreadable' }; + } finally { + await file.close(); + } +} diff --git a/src/internal/bounded-lines.ts b/src/internal/bounded-lines.ts new file mode 100644 index 0000000..f79cc35 --- /dev/null +++ b/src/internal/bounded-lines.ts @@ -0,0 +1,100 @@ +import { open } from 'node:fs/promises'; + +const READ_CHUNK_BYTES = 64 * 1024; + +/** One line emitted by a bounded byte-oriented file scan. */ +export type BoundedLine = + | { + readonly kind: 'line'; + readonly lineNumber: number; + readonly value: string; + } + | { + readonly kind: 'oversized'; + readonly lineNumber: number; + }; + +/** Options for a bounded newline scan. */ +export type ReadBoundedLinesOptions = { + readonly maxLineBytes: number; +}; + +/** + * Stream a file by newline without retaining more than `maxLineBytes`. + * + * Line and oversized entries are yielded only after a `0x0a` byte. An + * unterminated EOF tail is deferred with no diagnostic. Once a line crosses + * the ceiling its buffered chunks are released; the oversized diagnostic is + * still emitted only when that line's newline arrives. Callers that continue + * consume the remainder in discard mode without a duplicate diagnostic. + * + * @param path - File to stream. + * @param options - Scan limits; `maxLineBytes` is the per-line retention ceiling. + * @yields Newline-terminated lines, or bounded oversized diagnostics. + */ +export async function* readBoundedLines( + path: string, + options: ReadBoundedLinesOptions +): AsyncGenerator { + const { maxLineBytes } = options; + const file = await open(path, 'r'); + try { + const readBuffer = Buffer.allocUnsafe(READ_CHUNK_BYTES); + let readOffset = 0; + let lineNumber = 1; + let lineByteLength = 0; + let lineChunks: Buffer[] = []; + let oversized = false; + + for (;;) { + const { bytesRead } = await file.read( + readBuffer, + 0, + readBuffer.byteLength, + readOffset + ); + if (bytesRead === 0) break; + + let chunkOffset = 0; + while (chunkOffset < bytesRead) { + const foundNewline = readBuffer.indexOf(0x0a, chunkOffset); + const newlineIndex = + foundNewline >= 0 && foundNewline < bytesRead + ? foundNewline + : bytesRead; + const segment = readBuffer.subarray(chunkOffset, newlineIndex); + const segmentLength = segment.byteLength; + + if (!oversized) { + if (lineByteLength + segmentLength > maxLineBytes) { + oversized = true; + lineChunks = []; + } else if (segmentLength > 0) { + lineChunks.push(Buffer.from(segment)); + } + } + lineByteLength += segmentLength; + + if (newlineIndex === bytesRead) break; + + if (oversized) { + yield { kind: 'oversized', lineNumber }; + } else { + yield { + kind: 'line', + lineNumber, + value: Buffer.concat(lineChunks, lineByteLength).toString('utf8'), + }; + } + lineNumber += 1; + lineByteLength = 0; + lineChunks = []; + oversized = false; + chunkOffset = newlineIndex + 1; + } + readOffset += bytesRead; + } + } finally { + await file.close(); + } +} diff --git a/src/internal/discovery-primitives.ts b/src/internal/discovery-primitives.ts new file mode 100644 index 0000000..446cbde --- /dev/null +++ b/src/internal/discovery-primitives.ts @@ -0,0 +1,16 @@ +import type { Dirent } from 'node:fs'; +import { readdir } from 'node:fs/promises'; + +/** Read one directory for discovery, treating unavailable roots as empty. */ +export async function readDiscoveryDirectory(path: string): Promise { + try { + return await readdir(path, { withFileTypes: true }); + } catch { + return []; + } +} + +/** Return paths in deterministic lexical order. */ +export function sortDiscoveredPaths(paths: readonly string[]): string[] { + return [...paths].sort((left, right) => left.localeCompare(right)); +} diff --git a/src/internal/incremental.ts b/src/internal/incremental.ts new file mode 100644 index 0000000..563aebb --- /dev/null +++ b/src/internal/incremental.ts @@ -0,0 +1,47 @@ +import type { JsonlCursor } from './jsonl-cursor.js'; + +/** Return the next semantic revision, advancing only for observable state. */ +export function checkpointRevision(base: number, changed: boolean): number { + return changed ? base + 1 : base; +} + +/** Compare complete durable byte cursors, including reset identity. */ +export function byteCursorsEqual( + left: JsonlCursor | null, + right: JsonlCursor | null +): boolean { + if (left === null || right === null) return left === right; + return ( + left.device === right.device && + left.inode === right.inode && + left.offset === right.offset && + left.lineNumber === right.lineNumber && + left.generation === right.generation && + left.headDigest === right.headDigest && + left.boundaryDigest === right.boundaryDigest && + pendingEqual(left.pending, right.pending) + ); +} + +function pendingEqual( + left: JsonlCursor['pending'], + right: JsonlCursor['pending'] +): boolean { + const normalizedLeft = left ?? null; + const normalizedRight = right ?? null; + if (normalizedLeft === null || normalizedRight === null) { + return normalizedLeft === normalizedRight; + } + return ( + normalizedLeft.kind === normalizedRight.kind && + normalizedLeft.byteStart === normalizedRight.byteStart + ); +} + +/** True when a cursor scan observed bytes, identity, or generation movement. */ +export function byteCursorChanged( + previous: JsonlCursor | null, + next: JsonlCursor | null +): boolean { + return !byteCursorsEqual(previous, next); +} diff --git a/src/internal/jsonl-cursor-scan.ts b/src/internal/jsonl-cursor-scan.ts new file mode 100644 index 0000000..2f8d7ff --- /dev/null +++ b/src/internal/jsonl-cursor-scan.ts @@ -0,0 +1,194 @@ +import type { FileHandle } from 'node:fs/promises'; + +import type { + JsonlLine, + JsonlOversizedDiagnostic, + JsonlOversizedPending, +} from './jsonl-cursor-types.js'; + +const SCAN_CHUNK_BYTES = 64 * 1024; + +/** Resolved positive limits used by the byte scanner. */ +export interface JsonlScanLimits { + readonly maxLineBytes: number; + readonly maxScanBytes: number; + readonly maxScanLines: number; +} + +/** Inputs for one bounded complete-line scan. */ +export interface JsonlScanRequest { + readonly file: FileHandle; + readonly startOffset: number; + readonly startLineNumber: number; + readonly snapshotSize: number; + readonly pending: JsonlOversizedPending | null; + readonly limits: JsonlScanLimits; +} + +/** Internal scan result consumed by the cursor identity layer. */ +export interface JsonlScanResult { + readonly lines: readonly JsonlLine[]; + readonly diagnostics: readonly JsonlOversizedDiagnostic[]; + readonly offset: number; + readonly lineNumber: number; + readonly pending: JsonlOversizedPending | null; + readonly scannedBytes: number; + readonly scannedLines: number; + readonly limitReason: 'bytes' | 'lines' | null; +} + +/** + * Scan a snapshotted range while buffering at most one bounded line. + * + * Invariant: committed offsets advance only past newline terminators. + * Oversized content is discarded incrementally and diagnosed once its newline + * arrives. A new line starts only when enough budget remains to classify it. + * + * @param request - Open handle, cursor position, snapshot, pending, and limits. + * @returns Complete lines, diagnostics, accounting, and continuation state. + */ +export async function scanJsonlCompleteLines( + request: JsonlScanRequest +): Promise { + const { file, startOffset, startLineNumber, snapshotSize, limits } = request; + const lines: JsonlLine[] = []; + const diagnostics: JsonlOversizedDiagnostic[] = []; + const readBuffer = Buffer.allocUnsafe(SCAN_CHUNK_BYTES); + let consumeEnd = startOffset; + let committedOffset = startOffset; + let lineStart = request.pending?.byteStart ?? startOffset; + let lineNumber = startLineNumber; + let lineByteLength = 0; + let lineChunks: Buffer[] = []; + let discarding = request.pending !== null; + let midLine = discarding; + let scannedLines = 0; + let limitReason: 'bytes' | 'lines' | null = null; + + const remaining = (): number => + limits.maxScanBytes - (consumeEnd - startOffset); + const beginLimit = (): 'bytes' | 'lines' | null => { + if (consumeEnd >= snapshotSize) return null; + if (scannedLines >= limits.maxScanLines) return 'lines'; + if (remaining() < limits.maxLineBytes + 1) return 'bytes'; + return null; + }; + + if (!midLine) { + const reason = beginLimit(); + if (reason !== null) { + return emptyLimitedResult(committedOffset, lineNumber, reason); + } + } + + while (consumeEnd < snapshotSize && remaining() > 0) { + const requestedBytes = Math.min( + readBuffer.byteLength, + snapshotSize - consumeEnd, + remaining() + ); + const { bytesRead } = await file.read( + readBuffer, + 0, + requestedBytes, + consumeEnd + ); + if (bytesRead === 0) break; + + let chunkOffset = 0; + let stop = false; + while (chunkOffset < bytesRead) { + const newlineIndex = readBuffer.indexOf(0x0a, chunkOffset); + const hasNewline = newlineIndex >= 0 && newlineIndex < bytesRead; + const segmentEnd = hasNewline ? newlineIndex : bytesRead; + const segmentLength = segmentEnd - chunkOffset; + + if (!discarding) { + if (lineByteLength + segmentLength > limits.maxLineBytes) { + discarding = true; + lineChunks = []; + } else if (segmentLength > 0) { + lineChunks.push( + Buffer.from( + readBuffer.subarray(chunkOffset, chunkOffset + segmentLength) + ) + ); + } + } + lineByteLength += segmentLength; + consumeEnd += segmentLength; + midLine = true; + if (!hasNewline) break; + + consumeEnd += 1; + const byteEnd = consumeEnd; + if (discarding) { + diagnostics.push({ + kind: 'oversized', + lineNumber, + byteStart: lineStart, + byteEnd, + }); + } else { + lines.push({ + value: Buffer.concat(lineChunks, lineByteLength).toString('utf8'), + lineNumber, + byteStart: lineStart, + byteEnd, + }); + } + + committedOffset = byteEnd; + lineStart = byteEnd; + lineNumber += 1; + lineByteLength = 0; + lineChunks = []; + discarding = false; + midLine = false; + scannedLines += 1; + chunkOffset = newlineIndex + 1; + const reason = beginLimit(); + if (reason !== null) { + limitReason = reason; + stop = true; + break; + } + } + if (stop) break; + } + + if (midLine && consumeEnd < snapshotSize && remaining() <= 0) { + limitReason = 'bytes'; + } + const pending: JsonlOversizedPending | null = + midLine && discarding + ? { kind: 'discarding_oversized', byteStart: lineStart } + : null; + return { + lines, + diagnostics, + offset: pending === null ? committedOffset : consumeEnd, + lineNumber, + pending, + scannedBytes: consumeEnd - startOffset, + scannedLines, + limitReason, + }; +} + +function emptyLimitedResult( + offset: number, + lineNumber: number, + reason: 'bytes' | 'lines' +): JsonlScanResult { + return { + lines: [], + diagnostics: [], + offset, + lineNumber, + pending: null, + scannedBytes: 0, + scannedLines: 0, + limitReason: reason, + }; +} diff --git a/src/internal/jsonl-cursor-types.ts b/src/internal/jsonl-cursor-types.ts new file mode 100644 index 0000000..430060c --- /dev/null +++ b/src/internal/jsonl-cursor-types.ts @@ -0,0 +1,100 @@ +/** In-progress oversized discard carried across scan passes. */ +export type JsonlOversizedPending = { + readonly kind: 'discarding_oversized'; + readonly byteStart: number; +}; + +/** + * Parse optional serialized oversized-discard state. + * + * @param value - Raw `pending` field from a cursor, marker, or checkpoint. + * @returns Normalized state, or `undefined` when a present shape is invalid. + */ +export function parseJsonlOversizedPending( + value: unknown +): JsonlOversizedPending | null | undefined { + if (value === undefined || value === null) return null; + if (!isRecord(value) || value['kind'] !== 'discarding_oversized') { + return undefined; + } + const byteStart = value['byteStart']; + if ( + typeof byteStart !== 'number' || + !Number.isSafeInteger(byteStart) || + byteStart < 0 + ) { + return undefined; + } + return { kind: 'discarding_oversized', byteStart }; +} + +/** Whether one scan finished the snapshotted range or stopped on a cap. */ +export type JsonlScanStatus = + | { readonly status: 'complete' } + | { readonly status: 'limited'; readonly reason: 'bytes' | 'lines' }; + +/** Serializable position and file identity for incremental JSONL reads. */ +export interface JsonlCursor { + /** Device identifier from the opened file. */ + readonly device: string; + /** Inode identifier from the opened file. */ + readonly inode: string; + /** Byte offset of the next uncommitted line, or discard resume point. */ + readonly offset: number; + /** One-based number of the next uncommitted line. */ + readonly lineNumber: number; + /** Number of identity or content resets observed by this cursor. */ + readonly generation: number; + /** SHA-256 digest of the committed prefix's leading window. */ + readonly headDigest: string; + /** SHA-256 digest of the committed prefix's trailing boundary window. */ + readonly boundaryDigest: string; + /** Multi-pass oversized discard, or null at a line boundary. */ + readonly pending?: JsonlOversizedPending | null; +} + +/** One complete newline-terminated JSONL line. */ +export interface JsonlLine { + /** UTF-8 decoded content without its newline. */ + readonly value: string; + /** One-based source line number. */ + readonly lineNumber: number; + /** Inclusive byte offset. */ + readonly byteStart: number; + /** Exclusive byte offset including newline. */ + readonly byteEnd: number; +} + +/** Diagnostic for one fully consumed oversized line. */ +export interface JsonlOversizedDiagnostic { + readonly kind: 'oversized'; + readonly lineNumber: number; + readonly byteStart: number; + readonly byteEnd: number; +} + +/** Result of one bounded JSONL delta scan. */ +export interface JsonlDelta { + readonly lines: readonly JsonlLine[]; + readonly diagnostics: readonly JsonlOversizedDiagnostic[]; + readonly cursor: JsonlCursor | null; + readonly fileSize: number | null; + readonly reset: boolean; + readonly scanStatus: JsonlScanStatus; + readonly scannedBytes: number; + readonly scannedLines: number; +} + +/** Options controlling a JSONL delta scan. */ +export interface ReadJsonlDeltaOptions { + /** Maximum buffered bytes per line before streaming discard begins. */ + readonly maxLineBytes?: number; + /** Maximum bytes one pass may consume from the resume offset. */ + readonly maxScanBytes?: number; + /** Maximum complete lines one pass may commit or discard. */ + readonly maxScanLines?: number; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/internal/jsonl-cursor.ts b/src/internal/jsonl-cursor.ts new file mode 100644 index 0000000..19017d5 --- /dev/null +++ b/src/internal/jsonl-cursor.ts @@ -0,0 +1,235 @@ +import { createHash } from 'node:crypto'; +import { open, type FileHandle } from 'node:fs/promises'; + +import { + scanJsonlCompleteLines, + type JsonlScanLimits, +} from './jsonl-cursor-scan.js'; +import type { + JsonlCursor, + JsonlDelta, + JsonlScanStatus, + ReadJsonlDeltaOptions, +} from './jsonl-cursor-types.js'; + +/** Shared cursor value, result, option, line, diagnostic, and pending contracts. */ +export { + parseJsonlOversizedPending, + type JsonlCursor, + type JsonlDelta, + type JsonlLine, + type JsonlOversizedDiagnostic, + type JsonlOversizedPending, + type JsonlScanStatus, + type ReadJsonlDeltaOptions, +} from './jsonl-cursor-types.js'; + +const DEFAULT_MAX_LINE_BYTES = 16 * 1024 * 1024; +const DEFAULT_MAX_SCAN_BYTES = 32 * 1024 * 1024; +const DEFAULT_MAX_SCAN_LINES = 10_000; +const DIGEST_WINDOW_BYTES = 4096; + +/** + * Read complete JSONL lines added after a cursor position. + * + * Reads stop at the opened handle's size snapshot. Complete lines and + * oversized discards are bounded by byte and line caps; unterminated tails + * remain uncommitted. Identity and bounded prefix digests invalidate stale + * cursors without reading the entire file. + * + * @param path - JSONL file path. + * @param cursor - Prior serializable cursor, or null for a full scan. + * @param options - Per-scan line and pass-size limits. + * @returns Complete lines, diagnostics, accounting, and the next cursor. + * @throws If the file cannot be read, except when the path is missing. + * @throws If limits are not positive safe integers or the byte relationship is invalid. + */ +export async function readJsonlDelta( + path: string, + cursor: JsonlCursor | null, + options: ReadJsonlDeltaOptions = {} +): Promise { + const limits = resolveScanLimits(options); + let file: FileHandle; + try { + file = await open(path, 'r'); + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) return missingDelta(cursor); + throw error; + } + + try { + const stats = await file.stat(); + const fileSize = stats.size; + const device = String(stats.dev); + const inode = String(stats.ino); + const reset = await shouldResetCursor( + file, + fileSize, + device, + inode, + cursor + ); + const scan = await scanJsonlCompleteLines({ + file, + startOffset: reset ? 0 : (cursor?.offset ?? 0), + startLineNumber: reset ? 1 : (cursor?.lineNumber ?? 1), + snapshotSize: fileSize, + pending: reset ? null : (cursor?.pending ?? null), + limits, + }); + const digests = await digestCommittedBoundary(file, scan.offset); + const scanStatus: JsonlScanStatus = + scan.limitReason === null + ? { status: 'complete' } + : { status: 'limited', reason: scan.limitReason }; + return { + lines: scan.lines, + diagnostics: scan.diagnostics, + cursor: { + device, + inode, + offset: scan.offset, + lineNumber: scan.lineNumber, + generation: (cursor?.generation ?? 0) + (reset ? 1 : 0), + headDigest: digests.headDigest, + boundaryDigest: digests.boundaryDigest, + pending: scan.pending, + }, + fileSize, + reset, + scanStatus, + scannedBytes: scan.scannedBytes, + scannedLines: scan.scannedLines, + }; + } finally { + await file.close(); + } +} + +/** + * Validate one cursor identity without consuming JSONL content. + * + * @param path - JSONL path to validate. + * @param cursor - Existing cursor identity and committed boundary. + * @returns True when the cursor must be discarded. + */ +export async function jsonlCursorNeedsReset( + path: string, + cursor: JsonlCursor +): Promise { + let file: FileHandle; + try { + file = await open(path, 'r'); + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) return true; + throw error; + } + try { + const stats = await file.stat(); + return shouldResetCursor( + file, + stats.size, + String(stats.dev), + String(stats.ino), + cursor + ); + } finally { + await file.close(); + } +} + +function resolveScanLimits(options: ReadJsonlDeltaOptions): JsonlScanLimits { + const maxLineBytes = options.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES; + const maxScanBytes = options.maxScanBytes ?? DEFAULT_MAX_SCAN_BYTES; + const maxScanLines = options.maxScanLines ?? DEFAULT_MAX_SCAN_LINES; + assertPositiveLimit(maxLineBytes, 'maxLineBytes'); + assertPositiveLimit(maxScanBytes, 'maxScanBytes'); + assertPositiveLimit(maxScanLines, 'maxScanLines'); + if (maxScanBytes < maxLineBytes + 1) { + throw new RangeError('maxScanBytes must be >= maxLineBytes + 1'); + } + return { maxLineBytes, maxScanBytes, maxScanLines }; +} + +function assertPositiveLimit(value: number, name: string): void { + if (!Number.isSafeInteger(value) || value < 1) { + throw new RangeError(`${name} must be a positive safe integer`); + } +} + +async function shouldResetCursor( + file: FileHandle, + fileSize: number, + device: string, + inode: string, + cursor: JsonlCursor | null +): Promise { + if (cursor === null) return false; + if (cursor.device !== device || cursor.inode !== inode) return true; + if (fileSize < cursor.offset) return true; + const digests = await digestCommittedBoundary(file, cursor.offset); + return ( + digests.headDigest !== cursor.headDigest || + digests.boundaryDigest !== cursor.boundaryDigest + ); +} + +async function digestCommittedBoundary( + file: FileHandle, + offset: number +): Promise<{ readonly headDigest: string; readonly boundaryDigest: string }> { + const headLength = Math.min(offset, DIGEST_WINDOW_BYTES); + const boundaryStart = Math.max(0, offset - DIGEST_WINDOW_BYTES); + const [head, boundary] = await Promise.all([ + readRange(file, 0, headLength), + readRange(file, boundaryStart, offset - boundaryStart), + ]); + return { + headDigest: createHash('sha256').update(head).digest('hex'), + boundaryDigest: createHash('sha256').update(boundary).digest('hex'), + }; +} + +async function readRange( + file: FileHandle, + position: number, + length: number +): Promise { + if (length === 0) return Buffer.alloc(0); + const buffer = Buffer.allocUnsafe(length); + let totalRead = 0; + while (totalRead < length) { + const { bytesRead } = await file.read( + buffer, + totalRead, + length - totalRead, + position + totalRead + ); + if (bytesRead === 0) break; + totalRead += bytesRead; + } + return buffer.subarray(0, totalRead); +} + +function missingDelta(cursor: JsonlCursor | null): JsonlDelta { + return { + lines: [], + diagnostics: [], + cursor, + fileSize: null, + reset: false, + scanStatus: { status: 'complete' }, + scannedBytes: 0, + scannedLines: 0, + }; +} + +function hasErrorCode(error: unknown, code: string): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === code + ); +} diff --git a/src/internal/lease-lock-types.ts b/src/internal/lease-lock-types.ts new file mode 100644 index 0000000..4c8ee34 --- /dev/null +++ b/src/internal/lease-lock-types.ts @@ -0,0 +1,119 @@ +import type { z } from 'zod'; + +export interface LeaseTokenData { + readonly version: 2; + readonly ownerId: string; + readonly leaseId: string; + readonly pid: number; +} + +export interface LeaseTokenContext { + readonly ownerId: string; + readonly leaseId: string; + readonly pid: number; + readonly now: number; +} + +export interface LeaseLockHooks { + readonly onAfterCanonicalMkdirBeforeToken?: () => void | Promise; + readonly onAfterExpiredTokensClassified?: () => void | Promise; + readonly onAfterExpiredTokensUnlinkedBeforeRmdir?: () => void | Promise; + readonly onAfterReleaseTokensUnlinkedBeforeRmdir?: () => void | Promise; +} + +export type LegacyLeaseTokenData = + | { readonly nonce: string } + | { + readonly token: string; + readonly pid: number; + readonly createdAt: number; + }; + +export interface ExpiredLeaseToken { + readonly path: string; + readonly mtimeMs: number; + readonly token: LeaseTokenData | LegacyLeaseTokenData; +} + +export interface LeaseLockOptions extends LeaseLockHooks { + readonly staleMs: number; + readonly now?: () => number; + readonly ownerId?: string; + readonly pid?: number; + readonly tokenFields?: ( + context: LeaseTokenContext + ) => Readonly>; + readonly tokenSchema?: z.ZodType; + /** Adapter policy that may keep an age-expired token live. */ + readonly canReclaimExpiredToken?: ( + captured: ExpiredLeaseToken + ) => boolean | Promise; +} + +/** + * Handle for one owner on a canonical lock directory. + * + * `renew` publishes the successor token before unlinking this owner's + * previous token. `release` unlinks this owner's created tokens and + * non-recursively `rmdir`s. The canonical lock directory is never renamed or recursively removed; live tokens are never unlinked by another owner. + * + * @param lockPath - Canonical lock directory this handle occupies. + * @param ownerId - Owner UUID written into token names and bodies. + * @param leaseId - Current lease UUID; changes on a successful `renew`. + * @param tokenPath - Current token file path. + * @param createdTokenPaths - Every token pathname this handle created. + * @returns The handle object; `renew`, `assertHeld`, and `release` return `Promise`. + * @throws {LeaseLockLostError} From `renew` or `assertHeld` when the + * lease is no longer held. + */ +export interface Lease { + readonly lockPath: string; + readonly ownerId: string; + readonly leaseId: string; + readonly tokenPath: string; + /** Every token pathname created by this handle, including retired tokens. */ + readonly createdTokenPaths: readonly string[]; + renew(): Promise; + assertHeld(): Promise; + release(): Promise; +} + +/** + * Alias of {@link Lease}. Produced by the sync-filesystem acquire path. + * Handle methods stay async because schedule hooks may be async. + * + * The canonical lock directory is never renamed or recursively removed; live tokens are never unlinked by another owner. + * + * @returns The same shape as {@link Lease}. + */ +export type SyncLease = Lease; + +/** + * Thrown when acquire cannot take the canonical lock directory because a + * live occupant remains, or the directory identity changed mid-claim. + * + * The canonical lock directory is never renamed or recursively removed; live tokens are never unlinked by another owner. + * + * @param lockPath - Canonical lock path included in the message. + */ +export class LeaseLockBusyError extends Error { + constructor(lockPath: string) { + super(`Lease lock is busy: '${lockPath}'`); + this.name = 'LeaseLockBusyError'; + } +} + +/** + * Thrown by `Lease.renew` or `Lease.assertHeld` when this handle no + * longer holds a live token. + * + * The canonical lock directory is never renamed or recursively removed; live tokens are never unlinked by another owner. + * + * @param lockPath - Canonical lock path included in the message. + */ +export class LeaseLockLostError extends Error { + constructor(lockPath: string) { + super(`Lease lock is no longer held: '${lockPath}'`); + this.name = 'LeaseLockLostError'; + } +} diff --git a/src/internal/lease-lock.ts b/src/internal/lease-lock.ts new file mode 100644 index 0000000..d510b81 --- /dev/null +++ b/src/internal/lease-lock.ts @@ -0,0 +1,861 @@ +import type { Buffer } from 'node:buffer'; +import { randomUUID } from 'node:crypto'; +import { + closeSync, + fstatSync, + fsyncSync, + linkSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + renameSync, + rmdirSync, + statSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; +import { + link, + lstat, + mkdir, + open, + readdir, + rename, + rmdir, + stat, + unlink, +} from 'node:fs/promises'; +import { join } from 'node:path'; +import { z } from 'zod'; + +import { + LeaseLockBusyError, + LeaseLockLostError, + type ExpiredLeaseToken, + type Lease, + type LeaseLockHooks, + type LeaseLockOptions, + type LeaseTokenContext, + type LeaseTokenData, + type LegacyLeaseTokenData, + type SyncLease, +} from './lease-lock-types.js'; + +export type { + ExpiredLeaseToken, + Lease, + LeaseLockHooks, + LeaseLockOptions, + LeaseTokenContext, + LeaseTokenData, + LegacyLeaseTokenData, + SyncLease, +}; +export { LeaseLockBusyError, LeaseLockLostError }; + +const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const TOKEN_NAME_PATTERN = + /^owner\.([0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\.([0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/i; + +export const leaseTokenSchema = z + .object({ + version: z.literal(2), + ownerId: z.string().regex(UUID_PATTERN), + leaseId: z.string().regex(UUID_PATTERN), + pid: z.number().int().positive(), + }) + .strict(); + +const legacyTokenSchema = z.union([ + z.object({ nonce: z.string().min(1) }).strict(), + z + .object({ + token: z.string().min(1), + pid: z.number().int().positive(), + createdAt: z.number(), + }) + .strict(), +]); + +type Entry = { readonly name: string; readonly isFile: boolean }; +type Stats = { + readonly dev: number; + readonly ino: number; + readonly mtimeMs: number; + readonly isDirectory: boolean; + readonly isFile: boolean; +}; +type CapturedFile = { + readonly raw: Buffer; + readonly stats: Stats; +}; +type ExpiredFile = { + readonly path: string; + readonly isLegacy: boolean; +}; + +interface FileOps { + mkdir(path: string): Promise; + stat(path: string): Promise; + lstat(path: string): Promise; + readdir(path: string): Promise; + readAndStat(path: string): Promise; + writeToken(path: string, contents: string): Promise; + rename(from: string, to: string): Promise; + link(existingPath: string, newPath: string): Promise; + unlink(path: string): Promise; + rmdir(path: string): Promise; +} + +const asyncOps: FileOps = { + mkdir: async path => mkdir(path, { mode: 0o700 }), + stat: async path => toStats(await stat(path)), + lstat: async path => toStats(await lstat(path)), + readdir: async path => + (await readdir(path, { withFileTypes: true })).map(entry => ({ + name: entry.name, + isFile: entry.isFile(), + })), + readAndStat: async path => { + const handle = await open(path, 'r'); + try { + const stats = toStats(await handle.stat()); + const raw = await handle.readFile(); + return { raw, stats }; + } finally { + await handle.close(); + } + }, + writeToken: async (path, contents) => { + const handle = await open(path, 'wx', 0o600); + try { + await handle.writeFile(contents, 'utf8'); + await handle.sync(); + return toStats(await handle.stat()); + } catch (error: unknown) { + await handle.close(); + await unlink(path).catch(unlinkError => { + if (!hasCode(unlinkError, 'ENOENT')) throw unlinkError; + }); + throw error; + } finally { + await handle.close().catch(() => undefined); + } + }, + rename, + link, + unlink, + rmdir, +}; + +const syncOps: FileOps = { + mkdir: path => resolved(() => mkdirSync(path, { mode: 0o700 })), + stat: path => resolved(() => toStats(statSync(path))), + lstat: path => resolved(() => toStats(lstatSync(path))), + readdir: path => + resolved(() => + readdirSync(path, { withFileTypes: true }).map(entry => ({ + name: entry.name, + isFile: entry.isFile(), + })) + ), + readAndStat: path => + resolved(() => { + const fd = openSync(path, 'r'); + try { + const stats = toStats(fstatSync(fd)); + const raw = readFileSync(fd); + return { raw, stats }; + } finally { + closeSync(fd); + } + }), + writeToken: (path, contents) => + resolved(() => { + const fd = openSync(path, 'wx', 0o600); + try { + writeFileSync(fd, contents, 'utf8'); + fsyncSync(fd); + return toStats(fstatSync(fd)); + } catch (error: unknown) { + closeSync(fd); + try { + unlinkSync(path); + } catch (unlinkError: unknown) { + if (!hasCode(unlinkError, 'ENOENT')) throw unlinkError; + } + throw error; + } finally { + try { + closeSync(fd); + } catch (error: unknown) { + if (!hasCode(error, 'EBADF')) throw error; + } + } + }), + rename: (from, to) => resolved(() => renameSync(from, to)), + link: (existingPath, newPath) => + resolved(() => linkSync(existingPath, newPath)), + unlink: path => resolved(() => unlinkSync(path)), + rmdir: path => resolved(() => rmdirSync(path)), +}; + +/** + * Acquire an exclusive token lease on `lockPath` using async filesystem calls. + * + * Creates the canonical lock directory if absent, then publishes an + * `owner..` token. On `EEXIST`, entries are classified + * and only captured mtime-expired tokens are unlinked before a + * non-recursive `rmdir`. Fixed-name legacy objects are atomically renamed + * to unique claim paths, then reclassified from the claimed object itself. + * A fresh claim is restored with no-replace `link`, or fenced if another + * owner has occupied the canonical name; an unclaimed pathname is never + * unlinked. Expiry is `now - token.mtimeMs > staleMs`. + * The canonical lock directory is never renamed or recursively removed; + * a displaced fresh legacy owner may lose its lease fail-safe, but can + * never overlap ownership with the reclaimer. + * + * @param lockPath - Canonical lock directory path. + * @param options - Stale age, clock, owner identity, token schema, and + * optional reclaim / schedule hooks. + * @returns A {@link Lease} whose `release` unlinks this owner's tokens + * and `rmdir`s when the directory is empty. + * @throws {RangeError} When `options.staleMs` is not a non-negative finite number. + * @throws {LeaseLockBusyError} When a live occupant remains after reclaim + * attempts, or the directory identity changes before the token is bound. + * @throws {Error} When the published token path is not a regular file. + */ +export function acquireLeaseLock( + lockPath: string, + options: LeaseLockOptions +): Promise { + return acquire(lockPath, options, asyncOps); +} + +/** + * Acquire an exclusive token lease on `lockPath` using synchronous + * filesystem calls. Schedule hooks may still be async, so the function + * returns a Promise. + * + * The canonical lock directory is never renamed or recursively removed; + * fixed-name legacy occupants use the claim-reclassify-restore-or-fence + * protocol described by {@link acquireLeaseLock}. + * + * @param lockPath - Canonical lock directory path. + * @param options - Stale age, clock, owner identity, token schema, and + * optional reclaim / schedule hooks. + * @returns A {@link SyncLease} (same shape as {@link Lease}). + * @throws {RangeError} When `options.staleMs` is not a non-negative finite number. + * @throws {LeaseLockBusyError} When a live occupant remains after reclaim + * attempts, or the directory identity changes before the token is bound. + * @throws {Error} When the published token path is not a regular file. + */ +export function acquireLeaseLockSync( + lockPath: string, + options: LeaseLockOptions +): Promise { + return acquire(lockPath, options, syncOps); +} + +/** + * Acquire a lease, run `action`, and release in `finally`. + * + * Call `lease.renew()` or `lease.assertHeld()` at commit points; there is + * no heartbeat. The canonical lock directory is never renamed or recursively + * removed; fixed-name legacy occupants use the + * claim-reclassify-restore-or-fence protocol described by + * {@link acquireLeaseLock}. + * + * @param lockPath - Canonical lock directory path. + * @param action - Critical section. Receives the {@link Lease}. + * @param options - Stale age, clock, owner identity, token schema, and + * optional reclaim / schedule hooks. + * @returns The value returned by `action`. + * @throws {RangeError} When `options.staleMs` is not a non-negative finite number. + * @throws {LeaseLockBusyError} When a live occupant remains after reclaim + * attempts, or the directory identity changes before the token is bound. + * @throws {LeaseLockLostError} When `action` calls `renew` or `assertHeld` + * after the lease is released or the token is no longer held. + * @throws {Error} When the published token path is not a regular file. + */ +export async function withLeaseLock( + lockPath: string, + action: (lease: Lease) => T | Promise, + options: LeaseLockOptions +): Promise { + const lease = await acquireLeaseLock(lockPath, options); + try { + return await action(lease); + } finally { + await lease.release(); + } +} + +/** + * Same as {@link withLeaseLock}, using {@link acquireLeaseLockSync}. + * + * The canonical lock directory is never renamed or recursively removed; + * fixed-name legacy occupants use the claim-reclassify-restore-or-fence + * protocol described by {@link acquireLeaseLock}. + * + * @param lockPath - Canonical lock directory path. + * @param action - Critical section. Receives the {@link SyncLease}. + * @param options - Stale age, clock, owner identity, token schema, and + * optional reclaim / schedule hooks. + * @returns The value returned by `action`. + * @throws {RangeError} When `options.staleMs` is not a non-negative finite number. + * @throws {LeaseLockBusyError} When a live occupant remains after reclaim + * attempts, or the directory identity changes before the token is bound. + * @throws {LeaseLockLostError} When `action` calls `renew` or `assertHeld` + * after the lease is released or the token is no longer held. + * @throws {Error} When the published token path is not a regular file. + */ +export async function withLeaseLockSync( + lockPath: string, + action: (lease: SyncLease) => T | Promise, + options: LeaseLockOptions +): Promise { + const lease = await acquireLeaseLockSync(lockPath, options); + try { + return await action(lease); + } finally { + await lease.release(); + } +} + +async function acquire( + lockPath: string, + options: LeaseLockOptions, + ops: FileOps +): Promise { + validateOptions(options); + for (let attempt = 0; attempt < 3; attempt += 1) { + let directory: Stats; + try { + await ops.mkdir(lockPath); + directory = await ops.stat(lockPath); + } catch (error: unknown) { + if (!hasCode(error, 'EEXIST')) throw error; + if (await reclaim(lockPath, options, ops)) continue; + throw new LeaseLockBusyError(lockPath); + } + + try { + await options.onAfterCanonicalMkdirBeforeToken?.(); + return await createLease(lockPath, directory, options, ops); + } catch (error: unknown) { + if (!(error instanceof LeaseLockBusyError)) { + await removeEmptyOwnedDirectory(lockPath, directory, ops); + } + throw error; + } + } + throw new LeaseLockBusyError(lockPath); +} + +async function createLease( + lockPath: string, + directory: Stats, + options: LeaseLockOptions, + ops: FileOps +): Promise { + const ownerId = options.ownerId ?? randomUUID(); + const created = new Set(); + let leaseId = randomUUID(); + let tokenPath = tokenName(lockPath, ownerId, leaseId); + try { + await publishToken(tokenPath, ownerId, leaseId, options, ops); + } catch (error: unknown) { + if (hasCode(error, 'ENOENT')) throw new LeaseLockBusyError(lockPath); + throw error; + } + created.add(tokenPath); + + if ( + !(await ownsPublishedDirectory( + lockPath, + directory, + created, + tokenPath, + ops + )) + ) { + await discardPublishedToken(tokenPath, ops); + throw new LeaseLockBusyError(lockPath); + } + + let released = false; + const handle: Lease = { + lockPath, + ownerId, + get leaseId() { + return leaseId; + }, + get tokenPath() { + return tokenPath; + }, + get createdTokenPaths() { + return [...created]; + }, + async renew() { + if ( + released || + !(await tokenIsHeld( + lockPath, + directory, + tokenPath, + ownerId, + leaseId, + options, + ops + )) + ) { + throw new LeaseLockLostError(lockPath); + } + const previousPath = tokenPath; + const nextLeaseId = randomUUID(); + const nextPath = tokenName(lockPath, ownerId, nextLeaseId); + await publishToken(nextPath, ownerId, nextLeaseId, options, ops); + created.add(nextPath); + if ( + !(await ownsPublishedDirectory( + lockPath, + directory, + created, + nextPath, + ops + )) + ) { + await discardPublishedToken(nextPath, ops); + throw new LeaseLockLostError(lockPath); + } + leaseId = nextLeaseId; + tokenPath = nextPath; + await unlinkIfPresent(previousPath, ops); + }, + async assertHeld() { + if ( + released || + !(await tokenIsHeld( + lockPath, + directory, + tokenPath, + ownerId, + leaseId, + options, + ops + )) + ) { + throw new LeaseLockLostError(lockPath); + } + }, + async release() { + if (released) return; + released = true; + let firstError: unknown; + for (const path of created) { + try { + await unlinkIfPresent(path, ops); + } catch (error: unknown) { + firstError ??= error; + } + } + await options.onAfterReleaseTokensUnlinkedBeforeRmdir?.(); + try { + await safeRmdir(lockPath, ops); + } catch (error: unknown) { + firstError ??= error; + } + if (firstError !== undefined) throw firstError; + }, + }; + return handle; +} + +async function publishToken( + path: string, + ownerId: string, + leaseId: string, + options: LeaseLockOptions, + ops: FileOps +): Promise { + const now = (options.now ?? Date.now)(); + const pid = options.pid ?? process.pid; + const candidate: unknown = { + version: 2, + ownerId, + leaseId, + pid, + ...options.tokenFields?.({ ownerId, leaseId, pid, now }), + }; + const parsed = tokenSchema(options).parse(candidate); + const stats = await ops.writeToken(path, JSON.stringify(parsed)); + if (!stats.isFile) + throw new Error(`Lease token is not a regular file: '${path}'`); +} + +async function reclaim( + lockPath: string, + options: LeaseLockOptions, + ops: FileOps +): Promise { + let initial: Stats; + try { + initial = await ops.lstat(lockPath); + } catch (error: unknown) { + return hasCode(error, 'ENOENT'); + } + const now = (options.now ?? Date.now)(); + if (!initial.isDirectory) { + let captured: CapturedFile; + try { + captured = await ops.readAndStat(lockPath); + } catch (error: unknown) { + if (hasCode(error, 'ENOENT')) return true; + return false; + } + if ( + !captured.stats.isFile || + now - captured.stats.mtimeMs <= options.staleMs + ) { + return false; + } + await options.onAfterExpiredTokensClassified?.(); + const claim = await claimLegacyFile(lockPath, now, options, ops); + if (claim === 'retained') return false; + if (claim === 'removed') { + await options.onAfterExpiredTokensUnlinkedBeforeRmdir?.(); + } + return true; + } + + let entries: readonly Entry[]; + try { + entries = await ops.readdir(lockPath); + } catch (error: unknown) { + if (hasCode(error, 'ENOENT')) return true; + return false; + } + const expired: ExpiredFile[] = []; + let hasLive = false; + for (const entry of entries) { + if (!entry.isFile) return false; + const match = TOKEN_NAME_PATTERN.exec(entry.name); + const isLegacy = entry.name === 'owner.json'; + if (match === null && !isLegacy) return false; + let captured: CapturedFile; + try { + captured = await ops.readAndStat(join(lockPath, entry.name)); + } catch (error: unknown) { + if (hasCode(error, 'ENOENT')) continue; + return false; + } + if (!captured.stats.isFile) return false; + const parsed = parseJson(captured.raw); + const validation = isLegacy + ? legacyTokenSchema.safeParse(parsed) + : tokenSchema(options).safeParse(parsed); + if (!validation.success) return false; + const capturedPath = join(lockPath, entry.name); + const capturedToken = validation.data; + if ( + match !== null && + ('ownerId' in capturedToken === false || + capturedToken.ownerId !== match[1] || + capturedToken.leaseId !== match[2]) + ) { + return false; + } + const ageExpired = now - captured.stats.mtimeMs > options.staleMs; + if ( + ageExpired && + (options.canReclaimExpiredToken === undefined || + (await options.canReclaimExpiredToken({ + path: capturedPath, + mtimeMs: captured.stats.mtimeMs, + token: capturedToken, + }))) + ) { + expired.push({ path: capturedPath, isLegacy }); + } else { + hasLive = true; + } + } + + await options.onAfterExpiredTokensClassified?.(); + // dev+ino may be reused after replacement, but reclaim remains fail-closed: + // non-recursive rmdir cannot remove a replacement containing a fresh token. + // Removing a replacement that is still tokenless only interrupts a stalled + // publisher, whose subsequent ENOENT write is treated as contention. + if (!(await sameDirectory(lockPath, initial, ops))) return false; + if (entries.length === 0) { + let current: Stats; + try { + current = await ops.stat(lockPath); + } catch (error: unknown) { + if (hasCode(error, 'ENOENT')) return true; + throw error; + } + if (now - current.mtimeMs <= options.staleMs) return false; + } + // Claim fixed-name legacy entries before touching unique v2 tokens. If a + // replacement is fresh, restoring or fencing it aborts this reclaim pass + // without deleting any other classified object. + for (const captured of expired) { + if (!captured.isLegacy) continue; + const claim = await claimLegacyOwner( + captured.path, + now, + options.staleMs, + ops + ); + if (claim === 'retained') return false; + } + for (const captured of expired) { + if (!captured.isLegacy) await unlinkIfPresent(captured.path, ops); + } + if (expired.length > 0) { + await options.onAfterExpiredTokensUnlinkedBeforeRmdir?.(); + } + if (hasLive) return false; + return safeRmdir(lockPath, ops); +} + +async function tokenIsHeld( + lockPath: string, + directory: Stats, + path: string, + ownerId: string, + leaseId: string, + options: LeaseLockOptions, + ops: FileOps +): Promise { + if (!(await sameDirectory(lockPath, directory, ops))) return false; + try { + const captured = await ops.readAndStat(path); + const parsed = tokenSchema(options).safeParse(parseJson(captured.raw)); + return ( + captured.stats.isFile && + parsed.success && + parsed.data.ownerId === ownerId && + parsed.data.leaseId === leaseId && + (options.now ?? Date.now)() - captured.stats.mtimeMs <= options.staleMs + ); + } catch { + return false; + } +} + +function tokenSchema(options: LeaseLockOptions): z.ZodType { + return options.tokenSchema ?? leaseTokenSchema; +} + +function tokenName(lockPath: string, ownerId: string, leaseId: string): string { + return join(lockPath, `owner.${ownerId}.${leaseId}`); +} + +// If two stalled publishers reach one recreated directory, each sees the +// other's token and fails, or the first verified publisher wins and the later +// one fails. Cleanup may leave an orphan token, which is safer than admitting +// two owners and remains reclaimable by the normal stale-token path. +async function ownsPublishedDirectory( + path: string, + expected: Stats, + createdTokenPaths: ReadonlySet, + requiredTokenPath: string, + ops: FileOps +): Promise { + if (!(await sameDirectory(path, expected, ops))) return false; + try { + const entries = await ops.readdir(path); + const createdNames = new Set( + [...createdTokenPaths].map(createdPath => + createdPath.slice(path.length + 1) + ) + ); + const entryNames = entries.map(entry => entry.name); + return ( + entryNames.includes(requiredTokenPath.slice(path.length + 1)) && + entryNames.every(name => createdNames.has(name)) + ); + } catch { + return false; + } +} + +async function sameDirectory( + path: string, + expected: Stats, + ops: FileOps +): Promise { + try { + const current = await ops.stat(path); + return ( + current.isDirectory && + current.dev === expected.dev && + current.ino === expected.ino + ); + } catch { + return false; + } +} + +async function removeEmptyOwnedDirectory( + path: string, + expected: Stats, + ops: FileOps +): Promise { + if (await sameDirectory(path, expected, ops)) await safeRmdir(path, ops); +} + +async function discardPublishedToken( + path: string, + ops: FileOps +): Promise { + try { + await unlinkIfPresent(path, ops); + } catch { + // Ownership is already disproven, so the contention/lost result must win. + // An unremovable orphan fails closed and ages into normal stale reclaim. + } +} + +async function unlinkIfPresent(path: string, ops: FileOps): Promise { + try { + await ops.unlink(path); + } catch (error: unknown) { + if (!hasCode(error, 'ENOENT')) throw error; + } +} + +async function safeRmdir(path: string, ops: FileOps): Promise { + try { + await ops.rmdir(path); + return true; + } catch (error: unknown) { + if ( + hasCode(error, 'ENOENT') || + hasCode(error, 'ENOTEMPTY') || + hasCode(error, 'EEXIST') + ) { + return hasCode(error, 'ENOENT'); + } + throw error; + } +} + +function validateOptions(options: LeaseLockOptions): void { + if (!Number.isFinite(options.staleMs) || options.staleMs < 0) { + throw new RangeError('staleMs must be a non-negative finite number'); + } +} + +type LegacyClaimResult = 'gone' | 'removed' | 'retained'; + +async function claimLegacyFile( + originalPath: string, + now: number, + options: LeaseLockOptions, + ops: FileOps +): Promise { + return claimAndReclassify(originalPath, ops, async claimed => + Boolean( + claimed.stats.isFile && now - claimed.stats.mtimeMs > options.staleMs + ) + ); +} + +async function claimLegacyOwner( + originalPath: string, + now: number, + staleMs: number, + ops: FileOps +): Promise { + return claimAndReclassify(originalPath, ops, async claimed => { + if (!claimed.stats.isFile) return false; + const parsed = legacyTokenSchema.safeParse(parseJson(claimed.raw)); + return parsed.success && now - claimed.stats.mtimeMs > staleMs; + }); +} + +// Atomic rename removes the selected pathname before inspection, so later +// unlink can only affect the unique claim. A non-stale replacement is restored +// without replacement via link+unlink. EEXIST means a new owner already holds +// the canonical name: deleting the claim fences the displaced legacy owner. +// Its subsequent lease checks fail safe; the reclaimer never acquires in this +// pass, so displacement cannot produce double ownership. +async function claimAndReclassify( + originalPath: string, + ops: FileOps, + isStale: (claimed: CapturedFile) => Promise +): Promise { + const claimPath = `${originalPath}.claim.${randomUUID()}`; + try { + await ops.rename(originalPath, claimPath); + } catch (error: unknown) { + if (hasCode(error, 'ENOENT')) return 'gone'; + throw error; + } + + const claimed = await ops.readAndStat(claimPath); + if (await isStale(claimed)) { + await ops.unlink(claimPath); + return 'removed'; + } + + try { + await ops.link(claimPath, originalPath); + } catch (error: unknown) { + if (!hasCode(error, 'EEXIST')) throw error; + await ops.unlink(claimPath); + return 'retained'; + } + await ops.unlink(claimPath); + return 'retained'; +} + +function parseJson(raw: Buffer): unknown { + try { + return JSON.parse(raw.toString('utf8')) as unknown; + } catch { + return undefined; + } +} + +function toStats(stats: { + readonly dev: number; + readonly ino: number; + readonly mtimeMs: number; + isDirectory(): boolean; + isFile(): boolean; +}): Stats { + return { + dev: stats.dev, + ino: stats.ino, + mtimeMs: stats.mtimeMs, + isDirectory: stats.isDirectory(), + isFile: stats.isFile(), + }; +} + +function hasCode(error: unknown, code: string): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === code + ); +} + +function resolved(fn: () => T): Promise { + try { + return Promise.resolve(fn()); + } catch (error: unknown) { + return Promise.reject( + error instanceof Error ? error : new Error(String(error)) + ); + } +} diff --git a/src/internal/marker-lock.ts b/src/internal/marker-lock.ts new file mode 100644 index 0000000..7cde4dd --- /dev/null +++ b/src/internal/marker-lock.ts @@ -0,0 +1,216 @@ +import { mkdir, lstat, readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +import { + LeaseLockBusyError, + withLeaseLock, + type ExpiredLeaseToken, + type Lease, + type LegacyLeaseTokenData, +} from './lease-lock.js'; + +/** Lock directories older than this age are treated as abandoned. */ +export const DEFAULT_STALE_MARKER_LOCK_MS = 30_000; + +/** + * Identity captured for stale-lock reclamation hooks. + * + * `nonce` is taken from a legacy `owner.json` `{ nonce }` token when present. + * Version-2 tokens use `owner..` and report `nonce: null`. + */ +export type MarkerLockIdentity = { + readonly dev: number; + readonly ino: number; + readonly nonce: string | null; + readonly mtimeMs: number; +}; + +/** + * Options for {@link withMarkerLock}. + * + * `lockedLabel` is interpolated as `