diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..f1e34c4 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,101 @@ +name: Publish to GitHub Packages + +on: + push: + tags: + - 'v*' + +# Concurrency: one publish at a time per tag. +concurrency: + group: publish-${{ github.ref }} + cancel-in-progress: false + +# Workflow-level minimal permissions. `packages: write` is what +# `npm publish --registry=https://npm.pkg.github.com` actually needs; +# `contents: read` lets actions/checkout fetch the tagged commit. +permissions: + contents: read + packages: write + id-token: write + +jobs: + publish: + name: Publish @padosoft/agentic-qa-kit to GitHub Packages + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout tagged commit + uses: actions/checkout@v4 + with: + # The tag itself, not the default branch, so the published + # artifact matches the release notes. + ref: ${{ github.ref }} + fetch-depth: 1 + + - name: Pin Bun version + uses: oven-sh/setup-bun@v2 + with: + bun-version-file: .bun-version + + - name: Setup Node 22 (publisher) + uses: actions/setup-node@v4 + with: + node-version: '22' + # Configure npm for GitHub Packages auth — actions/setup-node + # writes an .npmrc with the right token + registry. + registry-url: 'https://npm.pkg.github.com' + scope: '@padosoft' + always-auth: true + + - name: Verify tag matches packages/kit version + # The publish workflow is tag-triggered. The kit's package.json + # version MUST match the tag (modulo a leading 'v') so the + # published tarball's metadata aligns with the release. + run: | + TAG="${GITHUB_REF##*/}" + TAG_VERSION="${TAG#v}" + PKG_VERSION="$(node -e "console.log(require('./packages/kit/package.json').version)")" + echo "tag=$TAG_VERSION pkg=$PKG_VERSION" + if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then + echo "::error::Tag $TAG_VERSION does not match packages/kit/package.json version $PKG_VERSION" + exit 1 + fi + + - name: Install workspaces + run: bun install --frozen-lockfile + + - name: Build the whole monorepo (so kit's bundle has all deps available) + run: bun run build + + - name: Verify built bundle exists + # The build step writes dist/cli.cjs + dist/cli.bundle.meta.json + # via packages/kit/scripts/build-bundle.mjs. Fail fast if not. + run: | + if [ ! -f packages/kit/dist/cli.cjs ]; then + echo "::error::packages/kit/dist/cli.cjs missing — build-bundle.mjs did not run" + exit 1 + fi + if [ ! -f packages/kit/dist/cli.bundle.meta.json ]; then + echo "::error::publish bundle meta missing — partial build" + exit 1 + fi + ls -lh packages/kit/dist/cli.cjs + cat packages/kit/dist/cli.bundle.meta.json + + - name: Rewrite name + workspace:* deps for publish + # publish-prep.mjs swaps @aqa/kit → @padosoft/agentic-qa-kit and + # pins every workspace:* dep to the kit's current version. The + # rewrite is local to this CI checkout and never committed back. + run: node packages/kit/scripts/publish-prep.mjs + + - name: npm publish (GitHub Packages) + working-directory: packages/kit + env: + # actions/setup-node@v4 wires NODE_AUTH_TOKEN into the .npmrc + # it generated; npm picks it up automatically when publishing + # to a scope-bound registry. + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # --provenance + --access public + the GH-Packages registry is + # the complete publish contract. The kit's package.json already + # carries publishConfig.access=public + publishConfig.registry. + run: npm publish --provenance --access public diff --git a/CHANGELOG.md b/CHANGELOG.md index f20eed6..f9d6f14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.9.0] — 2026-05-21 — Junior quick-start truthing + +The v1.x roadmap is fully closed and the kit ships to GitHub Packages as `@padosoft/agentic-qa-kit`. This release closes the four gaps an external junior would have hit if they tried to follow the README quick-start in v1.8: + +### Added + +- **`aqa install-agent-files --targets …`** CLI verb (PR #52). Cables the existing `renderForTargets()` from `@aqa/adapters` into a real command. Generates `CLAUDE.md` + `AGENTS.md` + `GEMINI.md` + `.github/copilot-instructions.md` plus per-agent skills under `.claude/`, `.agents/`, `.gemini/`, `.github/`. Flags: `--targets `, `--project-name `, `--force`, `--dry-run`. Unknown target fails fast without writing anything. Existing files preserved unless `--force`. +- **`aqa report [--run-id ] [--format md|json|both]`** CLI verb (PR #53). Renders `events.jsonl` + `findings.jsonl` from a run into `report.md` (auditor-friendly) and `report.json` (stable shape consumed by the admin UI). Defaults to the latest run by file mtime so hash-suffixed `--seed` ids work alongside ISO-prefixed ones. Strict on bad inputs: missing artifacts, malformed JSONL, traversal in `--run-id`, symlinked run dirs all error fast. +- **`aqa admin [--port ] [--host ]`** CLI verb (PR #54). Boots the admin SPA + `makeApi()` in a single Node process on `http://127.0.0.1:5173`. The bundled SPA ships inside the kit tarball; the in-memory store is seeded from `.aqa/runs//{events,findings}.jsonl` so the admin shows real local data out of the box. Path-traversal-safe static serving with SPA fallback for client-side routes. +- **`@aqa/pack-author`** new workspace package (PR #54). Extracted `runPackNew` from `@aqa/kit` to break the kit↔server build cycle that emerged when kit started depending on server (via the new `aqa admin` command). `@aqa/server`'s `POST /api/packs/scaffold` and `@aqa/kit`'s `aqa pack new` both consume it. Kit keeps a 5-line re-export shim so existing in-kit imports work unchanged. +- **GitHub Packages publish pipeline** (PR #55). New `.github/workflows/publish.yml` runs on every `v*` tag and publishes `@padosoft/agentic-qa-kit` to `https://npm.pkg.github.com` with `--provenance`. The kit publishes as a single bundled `dist/cli.cjs` (~460 KB) via esbuild — every `@aqa/*` workspace dep + every npm dep is inlined; only Node built-ins stay external. `packages/kit/scripts/publish-prep.mjs` swaps `@aqa/kit` → `@padosoft/agentic-qa-kit` and pins `workspace:*` deps to the kit's version at publish time only (the workspace keeps its internal name so other packages can keep referencing it). +- **README + `docs/getting-started.md` rewritten** to match the actually-shipped CLI surface. Adds the `.npmrc` snippet for GH Packages auth, the 10-step quick-start, the `aqa admin` boot path, and a single-command `bun run e2e:ecosystem` pointer for monorepo contributors. + +### Changed + +- **kit `package.json` `name` field policy.** The workspace name stays `@aqa/kit` so other monorepo packages can reference it. The published artifact's `name` is set at publish time from the new `aqa.publishName` declaration (`@padosoft/agentic-qa-kit`). This dual-naming keeps internal imports stable while satisfying GH Packages' ` === ` requirement. +- **Bundle format.** Kit now ships as `dist/cli.cjs` (CJS-in-.cjs) instead of separate per-file ESM modules. The `.cjs` extension overrides the package-level `"type": "module"` so Node loads it as CJS and bundled deps that internally `require('process')` resolve cleanly. +- **`packages/server/src/api.ts`** imports `runPackNew` from `@aqa/pack-author` (was `@aqa/kit`). +- **`packages/server/src/index.ts`** re-exports `ApiHandler`, `ApiMethod`, `ApiRequest`, `ApiResponse` so kit can consume them type-only. + +### Fixed + +- **`doctor.ts` hint no longer says `(Task 4)`** — the verb exists now, so the suggestion is the full command a junior can paste. +- **Slugifier caps at 64 chars** (Slug schema max). Previously a long project directory name would slip through `aqa init` / `aqa install-agent-files` and trip `aqa validate` later. Caps then re-strips trailing dashes so the truncated slug stays schema-conformant. +- **`KNOWN_TARGETS` derived from `@aqa/adapters.adapters`** instead of being a hardcoded duplicate — adding a new adapter (e.g. `opencode`) auto-extends `--targets`. + +## [1.8.3] — 2026-05-20 — Live ecosystem e2e + roadmap closure sync + +PR #51 — dedicated ecosystem Playwright smoke (`packages/admin/test/e2e/ecosystem-live.e2e.ts`) with a single-command stack bootstrap (`scripts/ecosystem-stack.mjs`). Boots `examples/bun-api`, runs a real `aqa run --profile smoke`, serves live `/api/*` from `@aqa/server.makeApi` + `MemoryStore`, drives the admin against the live backend, asserts `finding_emitted` is visible from `/api/audit` and chain verification returns `CHAIN OK`. Command: `bun run e2e:ecosystem`. + +## [1.8.2] — 2026-05-20 — Ecosystem smoke e2e hardening + +PR #50 — `scripts/e2e-cli.mjs` no longer stops at version/help/doctor/validate only: boots a local HTTP `/healthz` target, seeds a schema-valid local smoke pack/profile, executes `aqa run --profile smoke` with the real HTTP probe runner, and asserts run artifacts are emitted under `.aqa/runs//`. + +## [1.8.1] — 2026-05-20 — Audit chain canonical reconciliation + +PR #49 — aligned `@aqa/compliance.verifyEventChain` with `@aqa/runner.EventChainWriter`: hash recomputation excludes `prev_hash` from canonical body, and first-record `prev_hash: null` is canonical instead of expecting all-zero literal. + ## [1.3.0] — 2026-05-18 ### Added — quality polish (no new packages) diff --git a/README.md b/README.md index 12b546e..d73b1ba 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Coding agents (Claude Code, Codex CLI, Gemini CLI, GitHub Copilot CLI) are great ## Quick start (junior-friendly) -> **Status note:** the kit reached **v1.0 GA** (24-task roadmap complete) and is now at **v1.1**. The 18 workspace packages (`@aqa/schemas`, `@aqa/kit`, `@aqa/runner`, `@aqa/reporter`, `@aqa/server`, `@aqa/admin`, `@aqa/compliance`, `@aqa/methodology`, …) ship from this monorepo. Detailed walk-through: [`docs/getting-started.md`](docs/getting-started.md). +> **Status note:** the kit reached **v1.0 GA** (24-task roadmap complete) and is now at **v1.9**. The `@padosoft/agentic-qa-kit` CLI ships as a single bundled tarball from GitHub Packages. Detailed walk-through: [`docs/getting-started.md`](docs/getting-started.md). ### 1. Install Bun @@ -85,65 +85,111 @@ curl -fsSL https://bun.sh/install | bash powershell -c "irm bun.sh/install.ps1 | iex" ``` -### 2. Install the kit in your project +### 2. Tell your project where to find the kit (GitHub Packages auth) + +GitHub Packages requires authentication even for public packages. One-time setup per machine — create a PAT with `read:packages` scope at [github.com/settings/tokens](https://github.com/settings/tokens), then add it to a per-project `.npmrc`: + +```ini +# .npmrc — at the root of your project +@padosoft:registry=https://npm.pkg.github.com +//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN} +``` + +Export the token in your shell (or your CI secrets): + +```bash +export GITHUB_TOKEN=ghp_XXXXXXXXXXXXXXXXXXXX +``` + +### 3. Install the kit in your project ```bash cd /path/to/your/project -bun add -d agentic-qa-kit +bun add -d @padosoft/agentic-qa-kit ``` -> _If you don't have a project yet, clone `examples/bun-api` from this repo (available in v0.1.0)._ +> _If you don't have a project yet, clone `examples/bun-api` from this repo as a starting point._ -### 3. Initialize the AQA workspace +### 4. Initialize the AQA workspace + verify ```bash -bunx aqa init +bunx aqa init # scaffold .aqa/{project,risk-map,profiles}.yaml + testing.md +bunx aqa doctor # green/yellow/red checklist of kit health +bunx aqa validate # schema-check every .aqa/* file against @aqa/schemas ``` -Detects your stack and creates `.aqa/` with `testing.md`, `risk-map.yaml`, `profiles.yaml`, and scenarios for the packs your project matches. +`init` detects your stack (Bun/Node, framework, DB, SUT type) and creates a `.aqa/` directory anchored to the packs your project matches. -### 4. Install agent-specific files (pick one or many) +### 5. Install agent-specific files (one or many) ```bash bunx aqa install-agent-files --targets claude,codex,gemini,copilot ``` -This generates `CLAUDE.md` + `.claude/skills/aqa-*`, `AGENTS.md` + `.agents/skills/`, `GEMINI.md` + `.gemini/skills/`, `.github/copilot-instructions.md` + `.github/skills/`. +Generates `CLAUDE.md` + `.claude/skills/aqa-*`, `AGENTS.md` + `.agents/skills/`, `GEMINI.md` + `.gemini/skills/`, and `.github/copilot-instructions.md` + `.github/skills/`. Existing files are preserved unless you pass `--force`. Add `--dry-run` to see what would change first. -### 5. Run your first agentic QA pass +### 6. Edit `.aqa/risk-map.yaml` (declare what must never break) + +Replace the placeholder risk with the one that actually matters for your project. **The risk map is the heart of the kit — generic risks produce generic findings.** + +```yaml +- id: r-token-replay + category: auth + title: Tokens remain valid past rotation + severity: critical + likelihood: possible + invariants: + - id: inv-token-rotation + statement: Old tokens become invalid within 60 seconds of rotation. +``` + +### 7. Run your first agentic QA pass ```bash bunx aqa run --profile smoke ``` -A 10-minute, non-destructive sweep. When it finishes: +A fast, non-destructive sweep. Each run is written to `.aqa/runs//` with `events.jsonl`, `findings.jsonl`, and 3-level replay artifacts (`repro.sh`, `repro.curl`, `repro.playwright.ts`). + +### 8. Render the report ```bash -bunx aqa report +bunx aqa report # latest run, Markdown + JSON +bunx aqa report --run-id # explicit run +bunx aqa report --format md # just report.md ``` -You'll see findings like: +Writes `report.md` and `report.json` inside the same run directory. You'll see findings like: ``` AQA-2026-0001 [P1] Cross-tenant data leak (verified, 3/3 deterministic replay) AQA-2026-0002 [P3] Missing rate limit on /api/search ``` -### 6. Open the admin panel +### 9. Boot the admin panel (single command) ```bash -bun --filter @aqa/admin dev +bunx aqa admin ``` -Then open the local URL shown by Vite (normally `http://127.0.0.1:5173`) and inspect runs, findings, replay artifacts, and audit chain state. +Opens `http://127.0.0.1:5173`. The admin SPA + API server boot in one process, seeded from your local `.aqa/runs/`. Inspect runs, findings, replay artifacts, and verify the hash-chained audit log in-browser. `Ctrl-C` to stop. + +| Flag | Effect | +|---|---| +| `--port ` | listen on a specific port (default 5173) | +| `--host ` | bind host (default `127.0.0.1`; use `0.0.0.0` to expose on LAN) | -### 7. Reproduce from generated artifacts +### 10. Reproduce from generated artifacts ```bash ls .aqa/runs// +# events.jsonl findings.jsonl report.md report.json +# repro.sh repro.curl repro.playwright.ts ``` -Each run stores replay artifacts (`repro.sh`, `repro.curl`, `repro.playwright.ts`) so you can reproduce findings deterministically and confirm fixes. +Each finding ships with a deterministic replay artifact so you can reproduce it, hand it to a teammate, or attach it to a PR. + +> **Want the whole ecosystem in one go?** From a clone of `padosoft/agentic-qa-kit`, run `bun run e2e:ecosystem`. It boots `examples/bun-api`, runs a real `aqa run --profile smoke` against it, and opens the admin against the live data. Single command, end-to-end smoke. ## The mental model in 7 words @@ -156,12 +202,13 @@ Every concept in AQA is one of these seven things or a tool that operates on the ## How you use it 1. `aqa init`: detect your repo and scaffold `.aqa/`. -2. Edit `risk-map.yaml`: declare what must never break. -3. Install agent files: Claude/Codex/Gemini/Copilot instructions + skills. +2. `aqa install-agent-files --targets …`: write Claude/Codex/Gemini/Copilot instructions + skills. +3. Edit `risk-map.yaml`: declare what must never break. 4. `aqa run --profile smoke`: execute scenarios with probes + oracles. -5. Open admin: `bun --filter @aqa/admin dev`. -6. Inspect findings, replay deterministically, verify audit chain. -7. Iterate risks + scenarios until `release-gate` is green. +5. `aqa report`: render `report.md` + `report.json` from the latest run. +6. `aqa admin`: boot the SPA + API on `127.0.0.1:5173`, seeded from local runs. +7. Inspect findings, replay deterministically, verify audit chain. +8. Iterate risks + scenarios until `release-gate` is green. ## Multi-agent @@ -219,10 +266,12 @@ Full diagram: [`docs/architecture/reference.md`](docs/architecture/reference.md) | `v1.5` | **Admin design integration — shipped** | 30-screen hi-fi prototype bundled, Playwright E2E gate, theme + palette + Findings kanban | | `v1.6` | **`aqa run` + bundled packs — shipped** | Three-tier pack discovery, atomic run-dir, applies_when filtering, agent-mode rejection until driver lands | | `v1.7` | **Pack authoring + admin CRUD — shipped** | `PACK-AUTHORING.md`, `aqa pack new`, admin Create-pack/Import-manifest wizards, full Profile/Risk/Scenario CRUD (Delete/Edit/Clone), Agents wired to `/api/agents`, Operations + Admin pages wired to `/api/audit` / `/api/cost/summary` / `/api/queue` / `/api/notifications` / `/api/tokens` / `/api/orgs`, scenario YAML editor, schema-conforming mock-id migration, `Agent` schema, `agents:read`/`agents:edit` permissions, atomic `Store.createProfile/createScenario` | +| `v1.8` | **Live ecosystem e2e — shipped** | Real HTTP probe runner, release-gate finding enforcement, single-command ecosystem stack (`bun run e2e:ecosystem`), Playwright admin-against-live-API smoke, audit-chain canonical reconciliation | +| `v1.9` | **Junior quick-start truthing — shipped** | `aqa install-agent-files` + `aqa report` + `aqa admin` CLI verbs (previously documented but unwired), `@aqa/pack-author` extracted to break kit↔server build cycle, esbuild bundled `dist/cli.cjs`, GitHub Packages publish workflow on `v*` tags, README quick-start rewritten to match the actually-shipped CLI surface | ## Status -**GA (`v1.0` shipped, `v1.7` current).** The full 24-task roadmap is closed: +**GA (`v1.0` shipped, `v1.9` current).** The full 24-task roadmap is closed: schemas, CLI (`@aqa/kit`), 5 baseline packs, multi-agent adapters (Claude/Codex/Gemini/Copilot), runner with hash-chained audit, reporter with 3-level replay, admin panel, server + runner fleet, on-prem LLM diff --git a/bun.lock b/bun.lock index 16bba94..b7d49e4 100644 --- a/bun.lock +++ b/bun.lock @@ -84,17 +84,26 @@ }, "packages/kit": { "name": "@aqa/kit", - "version": "0.0.1", + "version": "1.9.0", "bin": { - "aqa": "./dist/cli/aqa.js", + "aqa": "./dist/cli.cjs", }, "dependencies": { + "@aqa/adapters": "workspace:*", + "@aqa/auth": "workspace:*", + "@aqa/pack-author": "workspace:*", "@aqa/pack-loader": "workspace:*", + "@aqa/reporter": "workspace:*", "@aqa/runner": "workspace:*", "@aqa/schemas": "workspace:*", + "@aqa/server": "workspace:*", + "@aqa/store": "workspace:*", "kleur": "^4.1.5", "yaml": "^2.6.0", }, + "devDependencies": { + "esbuild": "^0.24.0", + }, }, "packages/llm-adapters": { "name": "@aqa/llm-adapters", @@ -107,6 +116,14 @@ "@aqa/schemas": "workspace:*", }, }, + "packages/pack-author": { + "name": "@aqa/pack-author", + "version": "0.0.1", + "dependencies": { + "@aqa/schemas": "workspace:*", + "yaml": "^2.6.0", + }, + }, "packages/pack-loader": { "name": "@aqa/pack-loader", "version": "0.0.1", @@ -157,7 +174,7 @@ "version": "0.0.1", "dependencies": { "@aqa/auth": "workspace:*", - "@aqa/kit": "workspace:*", + "@aqa/pack-author": "workspace:*", "@aqa/schemas": "workspace:*", "@aqa/store": "workspace:*", "yaml": "^2.9.0", @@ -216,6 +233,8 @@ "@aqa/pack-api-core": ["@aqa/pack-api-core@workspace:packs/api-core"], + "@aqa/pack-author": ["@aqa/pack-author@workspace:packages/pack-author"], + "@aqa/pack-core": ["@aqa/pack-core@workspace:packs/core"], "@aqa/pack-llm-agent": ["@aqa/pack-llm-agent@workspace:packs/llm-agent"], @@ -296,57 +315,57 @@ "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@1.9.4", "", { "os": "win32", "cpu": "x64" }, "sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA=="], - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.24.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA=="], - "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], + "@esbuild/android-arm": ["@esbuild/android-arm@0.24.2", "", { "os": "android", "cpu": "arm" }, "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q=="], - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="], + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.24.2", "", { "os": "android", "cpu": "arm64" }, "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg=="], - "@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="], + "@esbuild/android-x64": ["@esbuild/android-x64@0.24.2", "", { "os": "android", "cpu": "x64" }, "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw=="], - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="], + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.24.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA=="], - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="], + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.24.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA=="], - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="], + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.24.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg=="], - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="], + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.24.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q=="], - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="], + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.24.2", "", { "os": "linux", "cpu": "arm" }, "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA=="], - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="], + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.24.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg=="], - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="], + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.24.2", "", { "os": "linux", "cpu": "ia32" }, "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw=="], - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="], + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.24.2", "", { "os": "linux", "cpu": "none" }, "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ=="], - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="], + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.24.2", "", { "os": "linux", "cpu": "none" }, "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw=="], - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="], + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.24.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw=="], - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="], + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.24.2", "", { "os": "linux", "cpu": "none" }, "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q=="], - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="], + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.24.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw=="], - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="], + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.24.2", "", { "os": "linux", "cpu": "x64" }, "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q=="], - "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="], + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.24.2", "", { "os": "none", "cpu": "arm64" }, "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw=="], - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="], + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.24.2", "", { "os": "none", "cpu": "x64" }, "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw=="], - "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="], + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.24.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A=="], - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="], + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.24.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA=="], "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="], - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="], + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.24.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig=="], - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="], + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.24.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ=="], - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="], + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.24.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA=="], - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.24.2", "", { "os": "win32", "cpu": "x64" }, "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg=="], "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], @@ -454,7 +473,7 @@ "electron-to-chromium": ["electron-to-chromium@1.5.357", "", {}, "sha512-NHlTIQDK8fmVwHwuIzmXYEJ1Ewq3D9wDNc0cWXxDGysP6Pb21giwGNkxiTifyKy/4SoPuN5l6GLP1W9Sv7zB2g=="], - "esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + "esbuild": ["esbuild@0.24.2", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.24.2", "@esbuild/android-arm": "0.24.2", "@esbuild/android-arm64": "0.24.2", "@esbuild/android-x64": "0.24.2", "@esbuild/darwin-arm64": "0.24.2", "@esbuild/darwin-x64": "0.24.2", "@esbuild/freebsd-arm64": "0.24.2", "@esbuild/freebsd-x64": "0.24.2", "@esbuild/linux-arm": "0.24.2", "@esbuild/linux-arm64": "0.24.2", "@esbuild/linux-ia32": "0.24.2", "@esbuild/linux-loong64": "0.24.2", "@esbuild/linux-mips64el": "0.24.2", "@esbuild/linux-ppc64": "0.24.2", "@esbuild/linux-riscv64": "0.24.2", "@esbuild/linux-s390x": "0.24.2", "@esbuild/linux-x64": "0.24.2", "@esbuild/netbsd-arm64": "0.24.2", "@esbuild/netbsd-x64": "0.24.2", "@esbuild/openbsd-arm64": "0.24.2", "@esbuild/openbsd-x64": "0.24.2", "@esbuild/sunos-x64": "0.24.2", "@esbuild/win32-arm64": "0.24.2", "@esbuild/win32-ia32": "0.24.2", "@esbuild/win32-x64": "0.24.2" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA=="], "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], @@ -557,5 +576,57 @@ "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], "playwright/fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + + "vite/esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + + "vite/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], + + "vite/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], + + "vite/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="], + + "vite/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="], + + "vite/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="], + + "vite/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="], + + "vite/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="], + + "vite/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="], + + "vite/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="], + + "vite/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="], + + "vite/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="], + + "vite/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="], + + "vite/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="], + + "vite/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="], + + "vite/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="], + + "vite/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="], + + "vite/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="], + + "vite/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="], + + "vite/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="], + + "vite/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="], + + "vite/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="], + + "vite/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="], + + "vite/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="], + + "vite/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="], + + "vite/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], } } diff --git a/docs/LESSON.md b/docs/LESSON.md index da1c410..07c3f29 100644 --- a/docs/LESSON.md +++ b/docs/LESSON.md @@ -21,6 +21,20 @@ - Topic / context — what was learned + why it matters. Reference files/commits when useful. ``` +## 2026-05-21 — v1.9 junior-quickstart-truthing macro + +- **Workspace-internal cycle breaks topo-sort builds even when TS handles them.** `@aqa/server` depended on `@aqa/kit` (for `runPackNew`). When `aqa admin` started needing `@aqa/server` (for `makeApi()`) and we added `@aqa/server` to kit's deps, locally everything looked fine because each prior build had already produced `kit/dist/`. CI from a fresh checkout hit `TS2307: cannot find module '@aqa/kit'` when building server — the topo-sort warned about the cycle but had to break it arbitrarily, and the loser saw the other side's dist as not-yet-built. **Fix**: extract the shared module (`runPackNew` → `@aqa/pack-author`). Both sides depend on the new package; the cycle goes away from the dep graph entirely. Cheaper than any `peerDependency`/dynamic-import workaround for the build phase. +- **CJS-in-`.cjs` is the safe shape for esbuild-bundled CLIs in `"type": "module"` packages.** ESM output bundled by esbuild rewrites bundled-dep `require('process')` into a `__require` helper that throws `Dynamic require of "process" is not supported`. Fixing it via banner `import { createRequire } from 'node:module'; const require = createRequire(import.meta.url)` doesn't help — the `__require` wrapper esbuild emits doesn't consult the user-level `require` binding. Switching to `format: 'cjs'` keeps Node's native `require` and the wrapper resolves cleanly. The output must be `.cjs` (not `.js`) so the package-level `"type": "module"` doesn't make Node parse it as ESM. +- **Externalise Node built-ins in BOTH prefix forms in esbuild.** `'fs'` AND `'node:fs'`. Some bundled CJS deps (yaml, ajv) call `require('process')` without the prefix; without listing the unprefixed names esbuild silently rewrites them as `Dynamic require` stubs. Easy to miss because the prefixed form covers all the TS/ESM call sites and tests pass — until a dep that uses bare requires is exercised at runtime. +- **Esbuild banner + entry shebang = double shebang = SyntaxError.** tsc preserves the source `#!/usr/bin/env node` in `dist/cli/aqa.js` (when the source has it). esbuild bundling preserves any shebang it sees at the entry's top. Adding `banner: { js: '#!/usr/bin/env node' }` results in two shebang lines in the output — Node parses the first as a hashbang and the second as JS, throwing `SyntaxError: Invalid or unexpected token`. Either don't use banner OR strip the entry's shebang first. Defensive post-process dedup (read output, drop subsequent `#!` lines) is cheap and survives future config drift. +- **GitHub Packages requires ` === `.** A repo under `padosoft/agentic-qa-kit` can only publish `@padosoft/*` packages. Internal workspace names that aren't in that scope must be rewritten at publish time. Two-name solution: workspace stays `@aqa/kit` so other monorepo packages can reference it via `workspace:*`; a `publish-prep.mjs` script reads a `aqa.publishName` field and swaps the `name` in `packages/kit/package.json` only inside the CI checkout. The rewrite is never committed back. +- **CRITICAL: strip `@aqa/*` deps from the published manifest when bundling.** If you publish `@padosoft/agentic-qa-kit` to GH Packages with `@aqa/runner` etc. still in `dependencies`, `bun add @padosoft/agentic-qa-kit` fails with `404 — @aqa/runner not found on registry` because those `@aqa/*` packages don't exist on any registry (they're inlined into `dist/cli.cjs` by esbuild). The publish-prep script must delete every `@aqa/*` entry from `dependencies`/`devDependencies`/`peerDependencies` after the name swap. +- **`package.json` LICENSE not auto-promoted from repo root.** If `packages/kit/package.json` declares `files: ['..., LICENSE']`, the tarball includes a `LICENSE` only if `packages/kit/LICENSE` exists on disk. The root-level `LICENSE` is NOT magically copied. Always cp / ship a sub-package LICENSE for any publishable package. +- **Copilot pattern: parallel PRs amortise review-round latency.** With 5 sequential PRs each taking 2-3 Copilot rounds, a sequential cascade is 10-15 round-trips on the critical path. Opening all 5 in parallel collapses that to max(rounds) = 3 rounds wall-clock. Cost: conflict resolution at merge cascade time on shared files (`packages/kit/package.json`, `packages/kit/src/cli/aqa.ts`). For this macro the conflicts were all union-merges (combine test scripts, combine deps, combine `case` statements in switch) — cheap. +- **"Docs reference things that don't exist on this branch" is expected and OK on a docs-only sub-task.** Copilot reviewing PR #56 in isolation correctly flagged that `aqa install-agent-files` / `aqa report` / `aqa admin` were referenced in README but not implemented on that branch's tree. The fix is the macro merge — once 1+2+3 land in macro, PR #56's claims become true. Don't address each "doc-vs-code mismatch" comment by reverting docs; address it by sequencing the merges correctly. +- **`bun pr merge --auto` flag isn't honored if repo-level auto-merge is disabled.** It returns `GraphQL: Auto merge is not allowed for this repository (enablePullRequestAutoMerge)` but the merge can still happen if CI is already green at command time. For reliable auto-merge enable it in repo settings; otherwise just poll and merge manually when status flips to CLEAN. +- **PR's `mergeStateStatus` lags the CI rollup by a few seconds.** Right after a force-push or rebase, a query may return `UNKNOWN` for a few seconds before flipping to `UNSTABLE` (CI in progress) → `CLEAN` (mergeable) or `DIRTY` (conflicts). A polling loop should treat `UNKNOWN` as "wait" not "merge". + ## 2026-05-20 - **E2E CLI smoke should be self-contained and network-real, not command-only.** A stable pattern is: bootstrap a temp sandbox with `aqa init`, overwrite `.aqa/project.yaml` + `profiles.yaml` with schema-valid minimal config, create a local `packs/pack-local-smoke` scenario, boot a local HTTP server (`/healthz`), run `aqa run --profile smoke`, and assert `.aqa/runs//events.jsonl` + `findings.jsonl` exist. This catches orchestration regressions without external services and without adding root-level test dependencies. diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 475646a..90979dc 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -9,6 +9,18 @@ - Each bullet states **what changed**, **why**, and **what's next** where relevant. - After a session interruption, the last bullet of the latest day is the resume point. +## 2026-05-21 + +- **v1.9 macro closed — junior quick-start truthing.** Five sub-task PRs merged into `task/v1.9-junior-quickstart-truthing`: + - **PR #52 — `aqa install-agent-files` CLI verb.** Cables `renderForTargets()` (`@aqa/adapters`) into a real command. `--targets `, `--project-name `, `--force`, `--dry-run`. 14 tests. 2 Copilot iter passes (array-form trim + Windows trailing-space in test temp dir, help text path `.github/copilot-instructions.md`, extracted `lastPathSegment`+`slugify` to `cli-utils.ts`, then iter 2: slugify cap to 64 chars + `KNOWN_TARGETS` derived from `adapters` registry). + - **PR #53 — `aqa report` CLI verb.** Cables `@aqa/reporter` (md+json) reading `events.jsonl`+`findings.jsonl`. `--run-id`, `--format md|json|both`. 24 tests. 3 Copilot iter passes covering: state derivation from `run_finished` payload counters (not "any-finished=succeeded"), mtime-based latest run (vs lexical, broken for `--seed` hashes), missing-artifact fail-fast, `--run-id` LongSlug regex (mirrors `@aqa/schemas` SlugPattern + 256-char cap), `readJsonl` rejects non-object lines (null/array/string/number), symlink protection on run dir AND per-file (`report.md`/`report.json`), reconstructed Run revalidated via `Run.Run.safeParse`. + - **PR #54 — `aqa admin` CLI verb.** Boots `node:http` server in-process serving bundled `dist/admin/` SPA + delegating `/api/*` to `makeApi()`. `--port`, `--host`, `/api/healthz` always-200. Path-traversal-safe static serving, SPA fallback for client routes. Seeds `MemoryStore` from `.aqa/runs/`. New `packages/pack-author/` package extracted to break the kit↔server build cycle. 7 admin tests + 2 pack-author tests. 3 Copilot iter passes (stale-cycle comment, `--host 0.0.0.0` security warning, pack-author README + named-export comment). + - **PR #55 — GitHub Packages publish pipeline.** esbuild bundles every workspace+npm dep into `dist/cli.cjs` (~570 KB CJS-in-.cjs). `scripts/publish-prep.mjs` swaps `@aqa/kit` → `@padosoft/agentic-qa-kit` AND strips `@aqa/*` deps (inlined in bundle). `.github/workflows/publish.yml` runs on `v*` tag, `npm publish --provenance --access public` to `https://npm.pkg.github.com`. `LICENSE` copied into `packages/kit/`. 4 publish/bundle tests + POSIX exec-bit assertion. 2 Copilot iter passes (CRITICAL `@aqa/*` strip, stale doc comments, LICENSE missing). + - **PR #56 — docs refresh.** README + `docs/getting-started.md` rewritten 1:1 with shipped verbs. Adds GH Packages auth `.npmrc` snippet (P1 junior trap), 10-step quick-start, `aqa admin` single-command boot, `bun run e2e:ecosystem` pointer. CHANGELOG `[1.9.0]` entry + backfilled `[1.8.1]`/`[1.8.2]`/`[1.8.3]`. +- **Strategy: 5 PRs in parallel** to amortise Copilot review latency. Each merged in cascade: 52 → 53 → 54 → 55 → 56, with conflict resolution at each step on `packages/kit/package.json` (deps + test scripts) and `packages/kit/src/cli/aqa.ts` (case statements + VALUE_FLAGS). 12 packages, 270 tests pass locally after the cascade. +- **Bundle health snapshot:** `node packages/kit/dist/cli.cjs --version` + `--help` both work end-to-end; all 7 verbs (init / doctor / validate / install-agent-files / run / report / admin / pack new) listed. Bundle size 571 KB (admin SPA + makeApi inlined via dynamic import). +- **Next:** macro PR `task/v1.9-junior-quickstart-truthing` → `main`, then `git tag v1.9.0` + GitHub Release. The tag triggers `.github/workflows/publish.yml` which publishes `@padosoft/agentic-qa-kit@1.9.0` to GitHub Packages. + ## 2026-05-20 - **v1.x roadmap closure completed.** Added a dedicated ecosystem Playwright smoke (`packages/admin/test/e2e/ecosystem-live.e2e.ts`) with a single-command stack bootstrap (`scripts/ecosystem-stack.mjs`) that boots `examples/bun-api`, runs a real `aqa run --profile smoke`, serves live `/api/*` from `@aqa/server.makeApi` + `MemoryStore`, and drives the admin against that live backend (`VITE_AQA_SERVER_URL`). The test asserts `finding_emitted` is visible from live `/api/audit` data and that chain verification returns `CHAIN OK`. Command: `bun run e2e:ecosystem`. diff --git a/docs/getting-started.md b/docs/getting-started.md index 6330a47..6df4daa 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -14,14 +14,36 @@ > Windows users: PowerShell 7+ is the supported shell. WSL also works. -## 1. Install the kit in your project (3 min) +## 1. Authenticate to GitHub Packages (2 min, one-time) + +The kit is published as `@padosoft/agentic-qa-kit` on GitHub Packages. Public packages on GH Packages still require auth — create a personal access token (PAT) once and tell `bun`/`npm` how to use it. + +1. Create a PAT at with scope **`read:packages`** (that's the only one you need to install). +2. Add this `.npmrc` to your project root: + +```ini +@padosoft:registry=https://npm.pkg.github.com +//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN} +``` + +3. Export the token in your shell (`~/.zshrc`, `~/.bashrc`, or the equivalent on Windows): + +```bash +export GITHUB_TOKEN=ghp_XXXXXXXXXXXXXXXXXXXX +``` + +> For CI, set `GITHUB_TOKEN` as a workflow/runner secret — never commit it. + +## 2. Install the kit in your project (1 min) ```bash cd path/to/your/project -bun add -D agentic-qa-kit +bun add -D @padosoft/agentic-qa-kit ``` -## 2. Bootstrap `.aqa/` (1 min) +The CLI is the only thing that gets installed — a single bundled `cli.cjs` (~460 KB) with all `@aqa/*` workspace deps inlined. The admin SPA and the 5 bundled packs ride along inside the same tarball. + +## 3. Bootstrap `.aqa/` (1 min) ```bash bunx aqa init @@ -37,25 +59,29 @@ This writes four files (non-destructive — existing files are skipped): └── testing.md # human-readable rationale for the QA conventions ``` -Open each file and tailor them to your SUT. **The risk map is the heart of -the kit — generic risks produce generic findings.** +Open each file and tailor them to your SUT. **The risk map is the heart of the kit — generic risks produce generic findings.** -## 3. Verify the install (1 min) +## 4. Verify the install (1 min) ```bash bunx aqa doctor # ✓/⚠/✗ checklist bunx aqa validate # schema-check .aqa/* against @aqa/schemas (CI-safe) ``` -## 4. Install agent instruction files (2 min) +## 5. Install agent instruction files (2 min) ```bash bunx aqa install-agent-files --targets claude,codex,gemini,copilot ``` -This generates agent-specific instruction files and skills in your repo. +This generates `CLAUDE.md`, `AGENTS.md`, `GEMINI.md`, `.github/copilot-instructions.md` plus per-agent skills under `.claude/skills/`, `.agents/skills/`, `.gemini/skills/`, and `.github/skills/`. + +Flags worth knowing: +- `--force` — overwrite existing files (default: skip). +- `--dry-run` — preview what would change without touching disk. +- `--project-name ` — override the slug embedded in the headers (default: directory name, slugified, capped at 64 chars). -## 5. Define one real risk (3 min) +## 6. Define one real risk (3 min) Replace the placeholder in `.aqa/risk-map.yaml`: @@ -70,37 +96,47 @@ Replace the placeholder in `.aqa/risk-map.yaml`: statement: Old tokens become invalid within 60 seconds of rotation. ``` -A good invariant is **one sentence**, **falsifiable**, and **independent of -implementation**. +A good invariant is **one sentence**, **falsifiable**, and **independent of implementation**. -## 6. Run the smoke profile (3 min) +## 7. Run the smoke profile (3 min) ```bash bunx aqa run --profile smoke ``` -Optional immediate report: +Each run writes `events.jsonl`, `findings.jsonl`, and per-finding replay artifacts (`repro.sh`, `repro.curl`, `repro.playwright.ts`) under `.aqa/runs//`. + +## 8. Render the report (10 sec) ```bash -bunx aqa report +bunx aqa report # latest run, both formats +bunx aqa report --run-id # explicit run +bunx aqa report --format md # just report.md ``` -Then open the admin panel: +Output lands inside the same run directory as `report.md` (auditor-friendly) and `report.json` (machine-readable, same shape the admin UI consumes). + +## 9. Boot the admin (10 sec) ```bash -bun --filter @aqa/admin dev +bunx aqa admin ``` +Opens `http://127.0.0.1:5173`. The SPA + API run in one process and the in-memory store is auto-seeded from `.aqa/runs/`. Browse runs, drill into findings, replay deterministically, verify the hash-chained audit log. `Ctrl-C` to stop. + +| Flag | Effect | +|---|---| +| `--port ` | listen on a specific port (default 5173; 0 = OS-assigned) | +| `--host ` | bind host (default `127.0.0.1`; use `0.0.0.0` to expose on LAN) | + ## Where to go next -- **`docs/methodology/agentic-qa.md`** — the Risk × Invariant × Probe × Oracle - methodology, in long form. -- **`docs/ecosystem-explained.md`** — every concept in the kit, with a worked - example. +- **`docs/methodology/agentic-qa.md`** — the Risk × Invariant × Probe × Oracle methodology, in long form. +- **`docs/ecosystem-explained.md`** — every concept in the kit, with a worked example. - **`docs/architecture/reference.md`** — the component map and data flow. +- **`docs/PACK-AUTHORING.md`** — write your own pack (`aqa pack new `). - **`docs/design/admin-panel-template.md`** — the full admin UI spec. - **`docs/RULES.md`** — the hard rules every contribution must obey. - **`docs/adr/`** — architecture decisions (start with ADR-001). -When you hit something the docs don't cover, file an issue. The kit is -junior-friendly **on purpose**. +When you hit something the docs don't cover, file an issue. The kit is junior-friendly **on purpose**. diff --git a/packages/kit/LICENSE b/packages/kit/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/packages/kit/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/kit/package.json b/packages/kit/package.json index ebbdc1e..843c399 100644 --- a/packages/kit/package.json +++ b/packages/kit/package.json @@ -1,36 +1,58 @@ { "name": "@aqa/kit", - "version": "0.0.1", - "description": "agentic-qa-kit — `aqa` CLI for the multi-agent QA operating system.", + "version": "1.9.0", + "description": "agentic-qa-kit — `aqa` CLI for the multi-agent QA operating system. Multi-agent (Claude/Codex/Gemini/Copilot), Bun-first, enterprise-ready.", "type": "module", "license": "Apache-2.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", + "homepage": "https://github.com/padosoft/agentic-qa-kit", + "repository": { + "type": "git", + "url": "https://github.com/padosoft/agentic-qa-kit.git", + "directory": "packages/kit" + }, + "bugs": { + "url": "https://github.com/padosoft/agentic-qa-kit/issues" + }, + "author": "Padosoft ", + "keywords": ["qa", "testing", "agent", "claude", "codex", "gemini", "copilot", "bun"], + "main": "./dist/cli.cjs", "exports": { ".": { - "types": "./dist/index.d.ts", - "default": "./dist/index.js" + "default": "./dist/cli.cjs" } }, "bin": { - "aqa": "./dist/cli/aqa.js" + "aqa": "./dist/cli.cjs" }, - "files": ["dist", "README.md"], + "files": ["dist/cli.cjs", "dist/cli.cjs.map", "dist/packs", "dist/admin", "README.md", "LICENSE"], "scripts": { - "build": "tsc -p tsconfig.json && node scripts/bundle-packs.mjs", + "build": "tsc -p tsconfig.json && node scripts/bundle-packs.mjs && node scripts/bundle-admin.mjs && node scripts/build-bundle.mjs", "typecheck": "tsc -p tsconfig.json --noEmit", - "pretest": "tsc -p tsconfig.json && node scripts/bundle-packs.mjs", - "test": "node --experimental-strip-types --no-warnings=ExperimentalWarning --test test/cli.test.ts test/profiler.test.ts test/run-cmd.test.ts test/pack-new.test.ts", + "pretest": "tsc -p tsconfig.json && node scripts/bundle-packs.mjs && node scripts/bundle-admin.mjs", + "test": "node --experimental-strip-types --no-warnings=ExperimentalWarning --test test/cli.test.ts test/profiler.test.ts test/run-cmd.test.ts test/pack-new.test.ts test/install-agent-files-cmd.test.ts test/report-cmd.test.ts test/admin-cmd.test.ts test/build-bundle.test.ts", "clean": "node --input-type=module -e \"import { rmSync } from 'node:fs'; for (const p of ['dist','.tsbuildinfo']) { try { rmSync(p, { recursive: true, force: true }); } catch {} }\"" }, "dependencies": { + "@aqa/adapters": "workspace:*", + "@aqa/auth": "workspace:*", + "@aqa/pack-author": "workspace:*", "@aqa/pack-loader": "workspace:*", + "@aqa/reporter": "workspace:*", "@aqa/runner": "workspace:*", "@aqa/schemas": "workspace:*", + "@aqa/server": "workspace:*", + "@aqa/store": "workspace:*", "kleur": "^4.1.5", "yaml": "^2.6.0" }, + "devDependencies": { + "esbuild": "^0.24.0" + }, "publishConfig": { - "access": "public" + "access": "public", + "registry": "https://npm.pkg.github.com" + }, + "aqa": { + "publishName": "@padosoft/agentic-qa-kit" } } diff --git a/packages/kit/scripts/build-bundle.mjs b/packages/kit/scripts/build-bundle.mjs new file mode 100644 index 0000000..2e15620 --- /dev/null +++ b/packages/kit/scripts/build-bundle.mjs @@ -0,0 +1,191 @@ +#!/usr/bin/env node +/** + * Bundles the `aqa` CLI into a single CommonJS file at `dist/cli.cjs`. + * + * Why bundle: `@padosoft/agentic-qa-kit` ships to GitHub Packages as a + * single tarball. Internal workspace deps (@aqa/runner, @aqa/schemas, + * @aqa/pack-loader, …) are NOT separately published to any registry — + * they only exist inside this monorepo's bun workspace. If we left them + * in `dependencies`, a downstream `bun add @padosoft/agentic-qa-kit` + * would fail because npm/bun can't resolve `@aqa/*` against the registry. + * + * esbuild walks the import graph, inlines every @aqa/* + every regular + * npm dep (yaml, kleur, zod, …) into one file, and externalises only + * Node built-ins (`node:fs`, `node:http`, …). The resulting `dist/cli.cjs` + * is the only JS artifact shipped in the npm tarball — alongside the + * static `dist/packs/` directory (bundled by bundle-packs.mjs) that the + * CLI loads at runtime via fs paths. The admin SPA static directory + * (`dist/admin/`) is bundled by a sibling `bundle-admin.mjs` added in + * the macro-task v1.9 `aqa admin` sub-task; this PR alone ships only + * the CLI bundler. + * + * Why CJS-in-.cjs instead of ESM-in-.js: some bundled CJS deps (yaml, + * ajv) call `require('process')` etc. inside their compiled output. + * esbuild's ESM bundler rewrites those into a `__require` helper that + * throws at runtime; CJS output keeps Node's native `require`. The + * `.cjs` extension makes Node load this file as CJS even though the + * kit's package.json sets `"type": "module"`. + * + * Sourcemap is emitted so `node --enable-source-maps` and crash stacks + * point back at the original TypeScript source files (under each + * `packages//src/`) even after the bundle inlines them. + */ +import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { build } from 'esbuild'; + +const here = dirname(fileURLToPath(import.meta.url)); +const kitRoot = resolve(here, '..'); +const entryPoint = join(kitRoot, 'dist', 'cli', 'aqa.js'); +// `.cjs` extension is mandatory because the kit's package.json sets +// `"type": "module"`, which makes every plain `.js` be parsed as ESM. +// The bundler emits CJS to keep bundled-dep `require()` calls working +// (see below), and a `.cjs` extension tells Node to load this file as +// CJS regardless of the surrounding type:module setting. +const outFile = join(kitRoot, 'dist', 'cli.cjs'); + +if (!existsSync(entryPoint)) { + console.error( + `[build-bundle] missing entry point ${entryPoint} — run \`tsc -p tsconfig.json\` first (this script is invoked after tsc in the kit build pipeline)`, + ); + process.exit(1); +} + +await build({ + entryPoints: [entryPoint], + outfile: outFile, + bundle: true, + platform: 'node', + target: 'node22', + // CJS output: some bundled npm deps (yaml's internals, ajv) embed + // `require('process')` etc. in their CJS source. esbuild's ESM + // bundling rewrites those to a `__require` helper that throws + // ("Dynamic require of X is not supported"). CJS output preserves + // Node's native `require` so the wrappers resolve cleanly. The + // tarball's `bin: dist/cli.cjs` runs identically as CJS — bin + // scripts don't care about ESM vs CJS as long as the shebang is + // intact. + format: 'cjs', + sourcemap: true, + // Node 22+ built-ins MUST stay external — bundling them breaks at + // runtime. `bun:*` modules are also external so a Node 22 install + // doesn't try to resolve them (the kit's bun-specific paths gracefully + // detect runtime via `globalThis.Bun` checks). + // Externalise Node built-ins in BOTH prefix forms — `node:fs` and + // `fs`. Some bundled CJS deps (yaml, ajv, …) `require('process')` or + // `require('crypto')` without the prefix; without listing the + // unprefixed names esbuild rewrites those as `Dynamic require ...` + // stubs that throw at runtime. The prefixed forms cover what TS/ESM + // code targets directly. + external: [ + 'bun', + 'bun:sqlite', + 'bun:test', + ...[ + 'assert', + 'async_hooks', + 'buffer', + 'child_process', + 'console', + 'crypto', + 'dgram', + 'diagnostics_channel', + 'dns', + 'events', + 'fs', + 'fs/promises', + 'http', + 'http2', + 'https', + 'inspector', + 'module', + 'net', + 'os', + 'path', + 'path/posix', + 'path/win32', + 'perf_hooks', + 'process', + 'punycode', + 'querystring', + 'readline', + 'repl', + 'stream', + 'stream/promises', + 'stream/web', + 'string_decoder', + 'sys', + 'test', + 'timers', + 'timers/promises', + 'tls', + 'trace_events', + 'tty', + 'url', + 'util', + 'util/types', + 'v8', + 'vm', + 'worker_threads', + 'zlib', + ].flatMap((m) => [m, `node:${m}`]), + ], + // No banner: tsc preserves the source shebang in dist/cli/aqa.js, and + // esbuild keeps any shebang it sees at the top of the entry. A banner + // here would emit a second `#!/usr/bin/env node` and Node rejects two + // shebangs in a row with `SyntaxError: Invalid or unexpected token`. + // Don't minify: the bundle stays grep-able and readable in stack + // traces. The tarball size penalty vs minified is small (~30%) and + // not worth the debugging cost for a CLI. + minify: false, + legalComments: 'none', + logLevel: 'info', +}); + +// Defensive shebang dedup. Today this script does NOT set an esbuild +// banner (the entry file's own `#!/usr/bin/env node`, preserved by +// tsc, is enough). If a future iteration re-introduces a banner with +// a shebang — or if esbuild ever decides to add one itself — the +// output would land with two shebang lines and Node would reject the +// second as a SyntaxError. Keeping this idempotent post-process means +// the bundle stays runnable through that kind of edit without anyone +// noticing in CI. +{ + const text = readFileSync(outFile, 'utf8'); + const lines = text.split('\n'); + if (lines[0]?.startsWith('#!')) { + const cleaned = [lines[0]]; + for (let i = 1; i < lines.length; i++) { + const line = lines[i] ?? ''; + if (line.startsWith('#!')) continue; + cleaned.push(line); + } + writeFileSync(outFile, cleaned.join('\n'), 'utf8'); + } +} + +// Make the bundled file executable on POSIX so `bunx @padosoft/agentic-qa-kit` +// can chmod-then-spawn without an extra step. No-op on Windows. +if (process.platform !== 'win32') { + try { + const { chmodSync } = await import('node:fs'); + chmodSync(outFile, 0o755); + } catch (e) { + console.warn( + `[build-bundle] could not chmod ${outFile}: ${e instanceof Error ? e.message : String(e)}`, + ); + } +} + +const st = statSync(outFile); +console.info(`[build-bundle] wrote ${outFile} (${(st.size / 1024).toFixed(1)} KB)`); + +// Write a sentinel that the test/CI can use to verify the bundle exists +// and is non-empty without invoking node on it (which would require the +// whole runtime + all bundled paths to be loadable). +writeFileSync( + join(kitRoot, 'dist', 'cli.bundle.meta.json'), + `${JSON.stringify({ bytes: st.size, generated_at: new Date().toISOString() }, null, 2)}\n`, + 'utf8', +); diff --git a/packages/kit/scripts/bundle-admin.mjs b/packages/kit/scripts/bundle-admin.mjs new file mode 100644 index 0000000..2bb1612 --- /dev/null +++ b/packages/kit/scripts/bundle-admin.mjs @@ -0,0 +1,39 @@ +#!/usr/bin/env node +/** + * Copies the prebuilt admin SPA (under /packages/admin/dist) into + * `packages/kit/dist/admin/` so it ships inside the `@aqa/kit` npm + * tarball. + * + * Why this exists: `aqa admin` boots an in-process HTTP server that + * serves the SPA static files alongside the @aqa/server API handlers. + * Without bundling, a junior who installed only `@aqa/kit` would have + * no admin SPA to serve and `aqa admin` would 404 on `/`. Topological + * `bun run build` runs `@aqa/admin`'s build before `@aqa/kit`'s, so the + * source dist is guaranteed to be present at bundle time. + * + * If the admin dist is missing (e.g. a downstream contributor only built + * `@aqa/kit` in isolation), we emit a warning and continue — `aqa admin` + * will then report a clear error at boot, not at build. + */ +import { cpSync, existsSync, mkdirSync, rmSync, statSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const kitRoot = resolve(here, '..'); +const repoRoot = resolve(kitRoot, '..', '..'); +const sourceAdminDist = join(repoRoot, 'packages', 'admin', 'dist'); +const destAdminDir = join(kitRoot, 'dist', 'admin'); + +if (!existsSync(sourceAdminDist) || !statSync(sourceAdminDist).isDirectory()) { + console.warn( + `[bundle-admin] no admin dist at ${sourceAdminDist} — \`aqa admin\` will not work until @aqa/admin is built`, + ); + process.exit(0); +} + +if (existsSync(destAdminDir)) rmSync(destAdminDir, { recursive: true, force: true }); +mkdirSync(destAdminDir, { recursive: true }); +cpSync(sourceAdminDist, destAdminDir, { recursive: true }); + +console.info(`[bundle-admin] copied admin SPA into ${destAdminDir}`); diff --git a/packages/kit/scripts/publish-prep.mjs b/packages/kit/scripts/publish-prep.mjs new file mode 100644 index 0000000..c181c69 --- /dev/null +++ b/packages/kit/scripts/publish-prep.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +/** + * Prepares `packages/kit/` for publishing to GitHub Packages. + * + * Why this exists: internally the package is named `@aqa/kit` so the + * monorepo's other workspaces (server, store, …) can reference it via + * `workspace:*`. The published artifact must be named + * `@padosoft/agentic-qa-kit` because GitHub Packages requires + * ` === ` (= `padosoft`). Renaming the workspace + * permanently would break every internal import path; rewriting only + * the published tarball is a one-line transform here. + * + * What it does: + * 1. Reads `packages/kit/package.json`. + * 2. Substitutes `name` with the value declared in `aqa.publishName` + * (currently `@padosoft/agentic-qa-kit`). + * 3. STRIPS every `@aqa/*` dependency entirely — those packages are + * bundled into `dist/cli.cjs` by esbuild and are NOT separately + * published, so leaving them in `dependencies` would make + * `bun add @padosoft/agentic-qa-kit` fail at install time + * ("404 — @aqa/runner not found on registry"). Reported by + * Copilot iter 1 on PR #55. + * 4. Replaces every remaining `workspace:*` dep value with the + * package's current version (so the published manifest is + * self-consistent and an npm consumer doesn't see the bun-only + * protocol). + * 5. Writes the prepared manifest back over `packages/kit/package.json` + * so `npm publish` in the same directory uses the rewritten file. + * + * Idempotent: running twice is a no-op as long as `aqa.publishName` is + * preserved. The publish workflow runs this immediately before `npm + * publish` and does NOT commit the change back — the rewrite is for + * the tarball only. + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const kitRoot = resolve(here, '..'); +const pkgPath = join(kitRoot, 'package.json'); + +const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')); +const publishName = pkg.aqa?.publishName; +if (typeof publishName !== 'string' || publishName.length === 0) { + console.error( + '[publish-prep] no aqa.publishName declared in packages/kit/package.json — refusing to publish', + ); + process.exit(2); +} + +if (pkg.name === publishName) { + console.info( + `[publish-prep] package already named ${publishName}; only rewriting workspace:* deps`, + ); +} else { + console.info(`[publish-prep] rewriting name: ${pkg.name} → ${publishName}`); + pkg.name = publishName; +} + +const version = pkg.version; +if (typeof version !== 'string' || !/^\d+\.\d+\.\d+/.test(version)) { + console.error(`[publish-prep] invalid version "${version}"; expected semver`); + process.exit(2); +} + +function rewriteDeps(group) { + const deps = pkg[group]; + if (!deps || typeof deps !== 'object') return { stripped: 0, pinned: 0 }; + let stripped = 0; + let pinned = 0; + for (const name of Object.keys(deps)) { + // 1. Drop @aqa/* — these are inlined in the bundle, never published. + if (name.startsWith('@aqa/')) { + delete deps[name]; + stripped += 1; + continue; + } + // 2. Pin any remaining `workspace:*` deps to the kit's version. + const spec = deps[name]; + if (typeof spec === 'string' && spec.startsWith('workspace:')) { + deps[name] = version; + pinned += 1; + } + } + if (stripped > 0) + console.info(`[publish-prep] stripped ${stripped} @aqa/* ${group} (bundled into dist/cli.cjs)`); + if (pinned > 0) console.info(`[publish-prep] pinned ${pinned} ${group} workspace:* → ${version}`); + return { stripped, pinned }; +} +rewriteDeps('dependencies'); +rewriteDeps('devDependencies'); +rewriteDeps('peerDependencies'); +rewriteDeps('optionalDependencies'); + +writeFileSync(pkgPath, `${JSON.stringify(pkg, null, 2)}\n`, 'utf8'); +console.info(`[publish-prep] wrote ${pkgPath}`); diff --git a/packages/kit/src/cli-utils.ts b/packages/kit/src/cli-utils.ts new file mode 100644 index 0000000..aae0477 --- /dev/null +++ b/packages/kit/src/cli-utils.ts @@ -0,0 +1,37 @@ +/** + * Shared helpers used across CLI commands. + * + * `lastPathSegment` + `slugify` were originally duplicated between + * `init.ts` and `install-agent-files.ts`. Both commands derive the + * project name from cwd when the user doesn't pass one, so they must + * produce identical results — extracting them here removes the drift + * risk if the Slug rules ever change. + */ + +export function lastPathSegment(root: string): string { + const parts = root.replace(/[\\/]+$/, '').split(/[\\/]/); + return parts[parts.length - 1] ?? 'project'; +} + +/** + * `@aqa/schemas` Slug has `.max(64)`. The slugifier feeds this value into + * `Project.name` (project.yaml + risk-map.yaml) where the schema would + * otherwise reject anything longer at `aqa validate` time. Cap to 64 *after* + * normalization and re-trim trailing dashes so the truncated tail doesn't + * become an illegal `-`-terminated slug. + */ +const SLUG_MAX_LEN = 64; + +export function slugify(raw: string): string { + const normalized = + raw + .toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^-+|-+$/g, '') || 'project'; + if (normalized.length <= SLUG_MAX_LEN) return normalized; + // Re-strip leading/trailing dashes after the slice — a cap that lands + // inside a `-` run would leave the slug ending in `-`, which the schema + // rejects. + return normalized.slice(0, SLUG_MAX_LEN).replace(/-+$/, '') || 'project'; +} diff --git a/packages/kit/src/cli/aqa.ts b/packages/kit/src/cli/aqa.ts index 2e3f3ef..c14c40d 100644 --- a/packages/kit/src/cli/aqa.ts +++ b/packages/kit/src/cli/aqa.ts @@ -1,8 +1,11 @@ #!/usr/bin/env node import { bold, cyan, dim, green, red, yellow } from 'kleur/colors'; +import { runAdmin } from '../commands/admin.js'; import { type CheckStatus, runDoctor } from '../commands/doctor.js'; import { runInit } from '../commands/init.js'; +import { runInstallAgentFiles } from '../commands/install-agent-files.js'; import { runPackNew } from '../commands/pack-new.js'; +import { runReport } from '../commands/report.js'; import { runRun } from '../commands/run.js'; import { runValidate } from '../commands/validate.js'; @@ -22,7 +25,20 @@ interface ParsedArgs { values: Map; } -const VALUE_FLAGS = new Set(['profile', 'seed', 'sut-type', 'description', 'author', 'license']); +const VALUE_FLAGS = new Set([ + 'profile', + 'seed', + 'sut-type', + 'description', + 'author', + 'license', + 'targets', + 'project-name', + 'run-id', + 'format', + 'port', + 'host', +]); function parseArgs(argv: string[]): ParsedArgs { const out: ParsedArgs = { command: null, positionals: [], flags: new Set(), values: new Map() }; @@ -77,19 +93,33 @@ ${bold('Usage')} aqa [options] ${bold('Commands')} - init [name] Scaffold .aqa/{project,risk-map,profiles}.yaml + testing.md - doctor Report kit health (runtime, .aqa, agent docs, validation) - validate Validate .aqa/* against @aqa/schemas - run [--profile

] Execute scenarios for the given profile; write events + findings - pack new Scaffold a new pack at /packs// (see the pack authoring guide: - https://github.com/padosoft/agentic-qa-kit/blob/main/docs/PACK-AUTHORING.md - — this path is only present in the source repo, not in the npm tarball) + init [name] Scaffold .aqa/{project,risk-map,profiles}.yaml + testing.md + doctor Report kit health (runtime, .aqa, agent docs, validation) + validate Validate .aqa/* against @aqa/schemas + install-agent-files --targets … Write CLAUDE.md / AGENTS.md / GEMINI.md / .github/copilot-instructions.md + plus per-agent skills under .claude/ .agents/ .gemini/ .github/ + run [--profile

] Execute scenarios for the given profile; write events + findings + report [--run-id ] Render the latest (or specified) run as report.md + report.json + admin [--port N] Boot the admin SPA + API on http://127.0.0.1:5173, seeded from .aqa/runs/ + pack new Scaffold a new pack at /packs// (see the pack authoring + guide: https://github.com/padosoft/agentic-qa-kit/blob/main/docs/PACK-AUTHORING.md + — this path is only present in the source repo, not in the npm tarball) ${bold('Common options')} - --force (init / pack new) overwrite existing files/directory - --dry-run (init) don't write to disk; print what would happen + --force (init / install-agent-files / pack new) overwrite existing files/directory + --dry-run (init / install-agent-files) don't write to disk; print what would happen --profile (run) profile key from .aqa/profiles.yaml --seed (run) deterministic run_id seed — useful for replay + --targets (install-agent-files) comma-separated targets: claude,codex,gemini,copilot + --project-name (install-agent-files) override the slug embedded in instruction files + --run-id (report) target a specific run; default = latest + --format (report) md | json | both (default: both) + --port (admin) HTTP port to listen on (default 5173; 0 = OS-assigned) + --host (admin) bind host (default 127.0.0.1 — recommended) + WARNING: \`aqa admin\` runs WITHOUT real authentication. + Binding to 0.0.0.0 exposes the in-memory store and + makeApi() to any host on the same network — only do + this on a trusted dev VM or isolated CI runner. --sut-type (pack new) api | web | cli | lib | agent | pipeline --description (pack new) one-line summary written into the manifest --author (pack new) manifest author field @@ -158,6 +188,49 @@ async function main(): Promise { } return result.ok ? 0 : 1; } + case 'install-agent-files': { + printHeader('install-agent-files'); + if (args.flags.has('targets') && !args.values.has('targets')) { + console.error(red('aqa install-agent-files: --targets requires a value')); + return 1; + } + if (args.flags.has('project-name') && !args.values.has('project-name')) { + console.error(red('aqa install-agent-files: --project-name requires a value')); + return 1; + } + const targetsRaw = args.values.get('targets'); + if (targetsRaw === undefined) { + console.error( + red('aqa install-agent-files: --targets is required (e.g. --targets claude,codex)'), + ); + return 1; + } + const installOpts: Parameters[0] = { + root: cwd, + targets: targetsRaw, + }; + if (args.values.has('project-name')) { + installOpts.projectName = args.values.get('project-name') ?? ''; + } + if (args.flags.has('force')) installOpts.overwrite = true; + if (args.flags.has('dry-run')) installOpts.dryRun = true; + const result = runInstallAgentFiles(installOpts); + if (!result.ok) { + console.error(red(` ✗ ${result.error}`)); + return 1; + } + console.info(dim(`targets: ${result.targets.join(', ')}`)); + for (const f of result.files) { + const marker = { + created: green('+'), + overwritten: yellow('~'), + 'skipped-exists': dim('·'), + 'dry-run': cyan('?'), + }[f.result]; + console.info(` ${marker} ${f.path} ${dim(`[${f.target}/${f.result}]`)}`); + } + return 0; + } case 'run': { printHeader('run'); // A flag passed without a value (e.g. `aqa run --profile`) is treated as @@ -199,6 +272,84 @@ async function main(): Promise { } return 0; } + case 'report': { + printHeader('report'); + if (args.flags.has('run-id') && !args.values.has('run-id')) { + console.error(red('aqa report: --run-id requires a value')); + return 1; + } + if (args.flags.has('format') && !args.values.has('format')) { + console.error(red('aqa report: --format requires a value')); + return 1; + } + const reportOpts: Parameters[0] = { root: cwd }; + if (args.values.has('run-id')) reportOpts.runId = args.values.get('run-id') ?? ''; + if (args.values.has('format')) { + const fmt = args.values.get('format') ?? ''; + if (fmt !== 'md' && fmt !== 'json' && fmt !== 'both') { + console.error(red(`aqa report: --format must be md | json | both, got "${fmt}"`)); + return 1; + } + reportOpts.format = fmt; + } + const result = runReport(reportOpts); + if (!result.ok) { + console.error(red(` ✗ ${result.error}`)); + return 1; + } + console.info(` ${green('✓')} ${bold(result.runId)}`); + console.info(` ${dim('runDir: ')}${result.runDir}`); + console.info(` ${dim('findings: ')}${result.findingsCount}`); + for (const f of result.files) console.info(` ${green('+')} ${f}`); + return 0; + } + case 'admin': { + printHeader('admin'); + if (args.flags.has('port') && !args.values.has('port')) { + console.error(red('aqa admin: --port requires a value')); + return 1; + } + if (args.flags.has('host') && !args.values.has('host')) { + console.error(red('aqa admin: --host requires a value')); + return 1; + } + const adminOpts: Parameters[0] = { root: cwd }; + if (args.values.has('port')) { + const raw = args.values.get('port') ?? ''; + const n = Number(raw); + if (!Number.isInteger(n) || n < 0 || n > 65535) { + console.error(red(`aqa admin: --port must be an integer 0..65535, got "${raw}"`)); + return 1; + } + adminOpts.port = n; + } + if (args.values.has('host')) adminOpts.host = args.values.get('host') ?? ''; + const result = await runAdmin(adminOpts); + if (!result.ok) { + console.error(red(` ✗ ${result.error}`)); + return 1; + } + console.info(` ${green('✓')} admin listening at ${bold(result.url)}`); + console.info(` ${dim('healthz: ')}${result.url}/api/healthz`); + console.info(` ${dim('Stop: ')}Ctrl-C`); + // Keep the process alive until interrupted. The server holds an open + // socket but that alone isn't enough to keep node from exiting once + // there's no other pending work — register a SIGINT/SIGTERM listener + // that closes cleanly before exiting. + const stop = async (): Promise => { + await result.close(); + process.exit(0); + }; + process.on('SIGINT', () => { + void stop(); + }); + process.on('SIGTERM', () => { + void stop(); + }); + // Block forever — until a signal triggers stop(). + await new Promise(() => {}); + return 0; + } case 'pack': { // Subcommand router for `aqa pack `. const sub = args.positionals[0]; diff --git a/packages/kit/src/commands/admin.ts b/packages/kit/src/commands/admin.ts new file mode 100644 index 0000000..c9f4c88 --- /dev/null +++ b/packages/kit/src/commands/admin.ts @@ -0,0 +1,501 @@ +/** + * `aqa admin` — boots the admin SPA + API in a single local process. + * + * The kit ships a prebuilt copy of the admin SPA inside its own + * `dist/admin/` (see scripts/bundle-admin.mjs). At boot we wire + * `@aqa/server.makeApi()` against an in-memory store seeded with the + * project's local runs, then serve the SPA over the same `node:http` + * server. Routes: + * + * GET /api/healthz → `{ ok: true }` (kit-owned, not in makeApi) + * * /api/* → delegated to makeApi() handlers + * * everything else → static file served from dist/admin/, + * index.html fallback for SPA routing. + * + * The store seed mirrors `scripts/ecosystem-stack.mjs` — we walk + * `.aqa/runs//{events,findings}.jsonl` and feed each entry to the + * memory store so the admin shows real local runs out of the box, not + * an empty list. Per-run reconstruction uses the same logic as + * `aqa report`, kept local here to avoid a circular dep on report.ts + * (report.ts owns the Markdown rendering; this file owns the boot). + */ + +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'node:http'; +import { extname, join, normalize, sep } from 'node:path'; +import { Event, Finding, Run } from '@aqa/schemas'; +import type { ApiContext, ApiHandler } from '@aqa/server'; +import type { StoreProvider } from '@aqa/store'; + +export interface AdminOptions { + root: string; + port?: number; + host?: string; + /** + * Override the directory the SPA is served from. Default is the + * `dist/admin/` co-located with the running kit's dist. Tests use + * this to point at a fixture without copying it. + */ + adminDistDir?: string; + /** + * Override the directory scanned for run artifacts. Default is + * `${root}/.aqa/runs/`. + */ + runsRoot?: string; +} + +export interface AdminHandle { + /** Resolved port — useful when caller asked for port=0 (auto). */ + port: number; + /** Resolved host. */ + host: string; + url: string; + /** Stops the server and frees its socket. Idempotent. */ + close: () => Promise; +} + +export interface AdminErr { + ok: false; + error: string; +} + +export type AdminBootResult = ({ ok: true } & AdminHandle) | AdminErr; + +const DEFAULT_PORT = 5173; +const DEFAULT_HOST = '127.0.0.1'; + +const CONTENT_TYPES: Record = { + '.html': 'text/html; charset=utf-8', + '.js': 'application/javascript; charset=utf-8', + '.mjs': 'application/javascript; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.ico': 'image/x-icon', + '.woff': 'font/woff', + '.woff2': 'font/woff2', + '.map': 'application/json; charset=utf-8', +}; + +export async function runAdmin(opts: AdminOptions): Promise { + const port = opts.port ?? DEFAULT_PORT; + const host = opts.host ?? DEFAULT_HOST; + if (!Number.isInteger(port) || port < 0 || port > 65535) { + return { ok: false, error: `admin: --port must be an integer 0..65535, got ${port}` }; + } + + const adminDistDir = opts.adminDistDir ?? defaultAdminDistDir(); + if (!existsSync(adminDistDir) || !statSync(adminDistDir).isDirectory()) { + return { + ok: false, + error: `admin: bundled SPA not found at ${adminDistDir} — reinstall @aqa/kit or run \`bun run build\` from the monorepo root`, + }; + } + const indexHtmlPath = join(adminDistDir, 'index.html'); + if (!existsSync(indexHtmlPath)) { + return { + ok: false, + error: `admin: ${indexHtmlPath} is missing — bundled SPA is incomplete`, + }; + } + + // Dynamic imports keep the bundle slim: makeApi() + RunnerQueue + + // MemoryStore are only needed when `aqa admin` is actually invoked. + // (The kit↔server static cycle that motivated this pattern in an + // earlier iteration was resolved by extracting `runPackNew` into + // `@aqa/pack-author`; the dynamic import is now an optimisation, not + // a workaround.) + const { makeApi, RunnerQueue } = await import('@aqa/server'); + const { MemoryStore } = await import('@aqa/store'); + + const store = new MemoryStore(); + const seedReport = await seedStoreFromRuns( + store, + opts.runsRoot ?? join(opts.root, '.aqa', 'runs'), + ); + + const api = makeApi(); + const ctx = { + store, + queue: new RunnerQueue(), + authenticate: async () => ({ + id: 'usr-local', + email: 'local@aqa.test', + display_name: 'Local', + // 'admin' role short-circuits permission checks in @aqa/auth. + roles: ['admin' as const], + }), + projectRoot: opts.root, + }; + + const server = createServer((req, res) => { + handleRequest(req, res, { api, ctx, adminDistDir, indexHtmlPath }).catch((err: unknown) => { + try { + res.statusCode = 500; + res.setHeader('content-type', 'application/json'); + res.end( + JSON.stringify({ + error: err instanceof Error ? err.message : `internal error: ${String(err)}`, + }), + ); + } catch { + // headers already sent + } + }); + }); + + await new Promise((resolve, reject) => { + const onError = (e: Error): void => { + server.off('listening', onListen); + reject(e); + }; + const onListen = (): void => { + server.off('error', onError); + resolve(); + }; + server.once('error', onError); + server.once('listening', onListen); + server.listen(port, host); + }).catch((e: Error) => { + throw e; + }); + + const address = server.address(); + const resolvedPort = typeof address === 'object' && address !== null ? address.port : port; + + // Log seed summary so the junior sees their local runs were detected. + if (seedReport.runs > 0) { + console.info( + `[admin] seeded ${seedReport.runs} local run(s), ${seedReport.events} event(s), ${seedReport.findings} finding(s) into the in-memory store`, + ); + } else { + console.info( + '[admin] no local runs found under .aqa/runs/ — admin will start empty; run `aqa run --profile smoke` first', + ); + } + + return { + ok: true, + port: resolvedPort, + host, + url: `http://${host}:${resolvedPort}`, + close: () => closeServer(server), + }; +} + +function closeServer(server: Server): Promise { + return new Promise((resolve) => { + server.close(() => resolve()); + }); +} + +function defaultAdminDistDir(): string { + // This file compiles to dist/commands/admin.js; the bundled SPA sits + // at dist/admin/. Resolve relative to the current ESM URL so it works + // both in the source tree and inside an npm-installed tarball. + // Using import.meta.url keeps this self-contained — no env var, no + // build-time string substitution. + const url = new URL('../admin/', import.meta.url); + // pathname is URL-encoded; on Windows it begins with `/C:/...` which + // node treats as a valid path when normalized. + let p = decodeURIComponent(url.pathname); + if (process.platform === 'win32' && /^\/[A-Za-z]:\//.test(p)) { + p = p.slice(1); + } + return normalize(p.replace(/\/+$/, '')); +} + +interface SeedReport { + runs: number; + events: number; + findings: number; +} + +async function seedStoreFromRuns(store: StoreProvider, runsRoot: string): Promise { + const report: SeedReport = { runs: 0, events: 0, findings: 0 }; + if (!existsSync(runsRoot) || !statSync(runsRoot).isDirectory()) return report; + + let entries: string[]; + try { + entries = readdirSync(runsRoot); + } catch { + return report; + } + + for (const name of entries) { + const runDir = join(runsRoot, name); + let runStat: ReturnType | undefined; + try { + runStat = statSync(runDir); + } catch { + continue; + } + if (!runStat || !runStat.isDirectory()) continue; + + const eventsPath = join(runDir, 'events.jsonl'); + const findingsPath = join(runDir, 'findings.jsonl'); + if (!existsSync(eventsPath)) continue; + + const events = readJsonlSafe(eventsPath); + const findings = existsSync(findingsPath) ? readJsonlSafe(findingsPath) : []; + + const runStarted = events.find((e) => e.kind === 'run_started'); + const runFinished = events.find((e) => e.kind === 'run_finished'); + const profile = readPayloadString(runStarted, 'profile') ?? 'unknown'; + const project = readPayloadString(runStarted, 'project') ?? 'unknown'; + const startedAt = + (typeof runStarted?.ts === 'string' && runStarted.ts) || new Date(0).toISOString(); + const finishedAt = typeof runFinished?.ts === 'string' ? runFinished.ts : undefined; + const runDraft = { + schema_version: '1' as const, + id: name, + started_at: startedAt, + ...(finishedAt ? { finished_at: finishedAt } : {}), + state: (runFinished ? 'succeeded' : 'running') as Run.RunState, + project, + profile, + execution_mode: 'orchestrator' as const, + config_snapshot: { + profile, + execution_mode: 'orchestrator' as const, + packs: [], + config_hash: '0'.repeat(64), + }, + totals: { + scenarios: readPayloadNumber(runFinished, 'scenarios_run') ?? 0, + findings: readPayloadNumber(runFinished, 'findings') ?? findings.length, + probes: 0, + llm_tokens_in: 0, + llm_tokens_out: 0, + llm_cost_usd: 0, + }, + artifact_dir: runDir, + }; + const parsedRun = Run.Run.safeParse(runDraft); + if (!parsedRun.success) { + console.warn( + `[admin] skipped run ${name}: invalid Run shape — ${parsedRun.error.message.split('\n')[0]}`, + ); + continue; + } + + try { + await store.saveRun(parsedRun.data); + let acceptedEvents = 0; + for (const raw of events) { + const ev = Event.Event.safeParse(raw); + if (!ev.success) continue; + await store.appendEvent(ev.data); + acceptedEvents += 1; + } + let acceptedFindings = 0; + for (const raw of findings) { + const f = Finding.Finding.safeParse(raw); + if (!f.success) continue; + await store.appendFinding(f.data); + acceptedFindings += 1; + } + report.runs += 1; + report.events += acceptedEvents; + report.findings += acceptedFindings; + } catch (e) { + console.warn(`[admin] skipped run ${name}: ${e instanceof Error ? e.message : String(e)}`); + } + } + + return report; +} + +function readJsonlSafe(path: string): Array> { + let text: string; + try { + text = readFileSync(path, 'utf8'); + } catch { + return []; + } + const out: Array> = []; + for (const raw of text.split('\n')) { + const line = raw.trim(); + if (!line) continue; + try { + const obj = JSON.parse(line) as unknown; + if (obj && typeof obj === 'object' && !Array.isArray(obj)) { + out.push(obj as Record); + } + } catch { + // tolerate malformed lines so a single broken run doesn't take the + // whole admin down at boot. report.ts is strict; admin is lenient. + } + } + return out; +} + +function readPayloadString( + obj: Record | undefined, + key: string, +): string | undefined { + const payload = obj?.payload; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return undefined; + const v = (payload as Record)[key]; + return typeof v === 'string' ? v : undefined; +} + +function readPayloadNumber( + obj: Record | undefined, + key: string, +): number | undefined { + const payload = obj?.payload; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return undefined; + const v = (payload as Record)[key]; + return typeof v === 'number' && Number.isFinite(v) ? v : undefined; +} + +interface HandleCtx { + api: readonly ApiHandler[]; + ctx: ApiContext; + adminDistDir: string; + indexHtmlPath: string; +} + +async function handleRequest( + req: IncomingMessage, + res: ServerResponse, + hctx: HandleCtx, +): Promise { + res.setHeader('access-control-allow-origin', '*'); + res.setHeader('access-control-allow-methods', 'GET,POST,PUT,DELETE,OPTIONS'); + res.setHeader('access-control-allow-headers', 'content-type,x-aqa-org,x-aqa-project'); + + const method = (req.method ?? 'GET').toUpperCase(); + if (method === 'OPTIONS') { + res.statusCode = 204; + res.end(); + return; + } + + const url = new URL(req.url ?? '/', 'http://localhost'); + + // Kit-owned healthz: trivial, always-200. Lets the test (and any + // junior smoke check) confirm the server is up without depending on + // makeApi's auth surface or store state. + if (url.pathname === '/api/healthz') { + res.statusCode = 200; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ ok: true })); + return; + } + + if (url.pathname.startsWith('/api/')) { + await delegateToApi({ req, res, url, method, hctx }); + return; + } + + serveStatic({ req, res, url, hctx }); +} + +async function delegateToApi(args: { + req: IncomingMessage; + res: ServerResponse; + url: URL; + method: string; + hctx: HandleCtx; +}): Promise { + const { req, res, url, method, hctx } = args; + let matched: { + route: HandleCtx['api'][number]; + params: Record; + } | null = null; + for (const r of hctx.api) { + if (r.method !== method) continue; + const params = routeMatch(r.path, url.pathname); + if (params) { + matched = { route: r, params }; + break; + } + } + if (!matched) { + res.statusCode = 404; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ error: 'route not found' })); + return; + } + + let body: unknown; + if (method === 'POST' || method === 'PUT' || method === 'DELETE') { + const chunks: Buffer[] = []; + for await (const c of req) chunks.push(c as Buffer); + const raw = Buffer.concat(chunks).toString('utf8').trim(); + body = raw ? JSON.parse(raw) : undefined; + } + const headers: Record = {}; + for (const [k, v] of Object.entries(req.headers)) { + headers[k] = Array.isArray(v) ? v.join(',') : String(v ?? ''); + } + const params: Record = { + ...Object.fromEntries(url.searchParams.entries()), + ...matched.params, + }; + const out = await matched.route.handle({ headers, params, body }, hctx.ctx); + res.statusCode = out.status; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify(out.body)); +} + +function routeMatch(template: string, pathname: string): Record | null { + const t = template.split('/').filter(Boolean); + const p = pathname.split('/').filter(Boolean); + if (t.length !== p.length) return null; + const params: Record = {}; + for (let i = 0; i < t.length; i += 1) { + const ti = t[i] as string; + const pi = p[i] as string; + if (ti.startsWith(':')) { + params[ti.slice(1)] = decodeURIComponent(pi); + } else if (ti !== pi) { + return null; + } + } + return params; +} + +function serveStatic(args: { + req: IncomingMessage; + res: ServerResponse; + url: URL; + hctx: HandleCtx; +}): void { + const { res, url, hctx } = args; + // Map `/` → `index.html`; everything else is resolved under adminDistDir. + // Reject any candidate that resolves outside adminDistDir (path traversal). + const rel = url.pathname === '/' ? 'index.html' : decodeURIComponent(url.pathname.slice(1)); + const candidate = normalize(join(hctx.adminDistDir, rel)); + if (!candidate.startsWith(hctx.adminDistDir + sep) && candidate !== hctx.adminDistDir) { + res.statusCode = 403; + res.end('forbidden'); + return; + } + + if (existsSync(candidate) && statSync(candidate).isFile()) { + const ext = extname(candidate).toLowerCase(); + res.statusCode = 200; + res.setHeader('content-type', CONTENT_TYPES[ext] ?? 'application/octet-stream'); + res.end(readFileSync(candidate)); + return; + } + + // SPA fallback: unknown paths return index.html so client-side routing + // takes over. Don't fallback on /api/* (already handled) or asset + // paths under /assets/ (which should 404 if truly missing). + if (url.pathname.startsWith('/assets/')) { + res.statusCode = 404; + res.end('not found'); + return; + } + res.statusCode = 200; + res.setHeader('content-type', CONTENT_TYPES['.html'] ?? 'text/html'); + res.end(readFileSync(hctx.indexHtmlPath)); +} diff --git a/packages/kit/src/commands/doctor.ts b/packages/kit/src/commands/doctor.ts index 048634f..5b793c2 100644 --- a/packages/kit/src/commands/doctor.ts +++ b/packages/kit/src/commands/doctor.ts @@ -96,7 +96,7 @@ export function runDoctor(opts: DoctorOptions): DoctorResult { : 'no agent instruction files', suggestion: docsPresent ? undefined - : 'Run `aqa install-agent-files` (Task 4) to scaffold agent-specific instructions.', + : 'Run `aqa install-agent-files --targets claude,codex,gemini,copilot` to scaffold agent-specific instructions.', }); return { profile, checks, worst: worstOf(checks) }; diff --git a/packages/kit/src/commands/init.ts b/packages/kit/src/commands/init.ts index b36666b..948f933 100644 --- a/packages/kit/src/commands/init.ts +++ b/packages/kit/src/commands/init.ts @@ -1,5 +1,6 @@ import { join } from 'node:path'; import { stringify as yamlStringify } from 'yaml'; +import { lastPathSegment, slugify } from '../cli-utils.js'; import { type WriteResult, writeFileSafe } from '../fs-utils.js'; import { type ProjectProfile, profileRepo } from '../profiler.js'; @@ -161,18 +162,3 @@ export function runInit(opts: InitOptions): InitResult { })); return { profile, files }; } - -function lastPathSegment(root: string): string { - const parts = root.replace(/[\\/]+$/, '').split(/[\\/]/); - return parts[parts.length - 1] ?? 'project'; -} - -function slugify(raw: string): string { - return ( - raw - .toLowerCase() - .replace(/[^a-z0-9-]+/g, '-') - .replace(/-{2,}/g, '-') - .replace(/^-+|-+$/g, '') || 'project' - ); -} diff --git a/packages/kit/src/commands/install-agent-files.ts b/packages/kit/src/commands/install-agent-files.ts new file mode 100644 index 0000000..ad574c2 --- /dev/null +++ b/packages/kit/src/commands/install-agent-files.ts @@ -0,0 +1,99 @@ +import { join } from 'node:path'; +import { type AdapterTarget, adapters, renderForTargets } from '@aqa/adapters'; +import { lastPathSegment, slugify } from '../cli-utils.js'; +import { type WriteResult, writeFileSafe } from '../fs-utils.js'; + +// Derive the canonical target list from the adapter registry so adding a +// new adapter (e.g. opencode) automatically extends `--targets` without +// requiring a synced edit here. Sorted alphabetically for stable error +// message ordering. +const KNOWN_TARGETS: readonly AdapterTarget[] = Object.freeze( + [...adapters.map((a) => a.target)].sort(), +); + +export interface InstallAgentFilesOptions { + root: string; + /** Comma-separated string ("claude,codex") or already-split array. */ + targets: string | readonly string[]; + /** Optional override; if omitted, derived from the last segment of `root`. */ + projectName?: string; + overwrite?: boolean; + dryRun?: boolean; +} + +export interface InstallAgentFilesOk { + ok: true; + targets: readonly AdapterTarget[]; + files: Array<{ path: string; target: AdapterTarget; result: WriteResult }>; +} + +export interface InstallAgentFilesErr { + ok: false; + error: string; +} + +export type InstallAgentFilesResult = InstallAgentFilesOk | InstallAgentFilesErr; + +export function runInstallAgentFiles(opts: InstallAgentFilesOptions): InstallAgentFilesResult { + const parsed = parseTargets(opts.targets); + if (!parsed.ok) return { ok: false, error: parsed.error }; + const targets = parsed.targets; + if (targets.length === 0) { + return { ok: false, error: 'install-agent-files: --targets must list at least one target' }; + } + + const projectName = slugify(opts.projectName ?? lastPathSegment(opts.root)); + const rendered = renderForTargets(targets, { projectName, root: opts.root }); + + const writeOpts = { overwrite: opts.overwrite, dryRun: opts.dryRun }; + const files: InstallAgentFilesOk['files'] = []; + for (const target of targets) { + for (const file of rendered.byTarget[target] ?? []) { + const absPath = join(opts.root, file.path); + const result = writeFileSafe(absPath, file.contents, writeOpts); + files.push({ path: file.path, target, result }); + } + } + + return { ok: true, targets, files }; +} + +interface ParsedTargets { + ok: true; + targets: readonly AdapterTarget[]; +} +interface ParsedTargetsErr { + ok: false; + error: string; +} +function parseTargets(input: string | readonly string[]): ParsedTargets | ParsedTargetsErr { + // Both code paths normalize the same way: split-if-string, then trim + drop + // empties from each token. Without unified normalization, an array form + // like `['claude ', '', 'codex']` would surface a confusing + // "unknown target ' '" error even though the user clearly meant + // `['claude', 'codex']`. + const tokens: string[] = Array.isArray(input) + ? input.map((s) => String(s)) + : String(input).split(','); + const raw = tokens.map((s) => s.trim()).filter((s) => s.length > 0); + // Preserve user-given order but de-dupe so the same target isn't written twice. + const seen = new Set(); + const targets: AdapterTarget[] = []; + for (const t of raw) { + const norm = t.toLowerCase(); + if (seen.has(norm)) continue; + seen.add(norm); + if (!isKnownTarget(norm)) { + return { + ok: false, + error: `install-agent-files: unknown target "${t}" — expected one of ${KNOWN_TARGETS.join(', ')}`, + }; + } + targets.push(norm); + } + return { ok: true, targets }; +} + +function isKnownTarget(s: string): s is AdapterTarget { + return (KNOWN_TARGETS as readonly string[]).includes(s); +} diff --git a/packages/kit/src/commands/pack-new.ts b/packages/kit/src/commands/pack-new.ts index d9a2a4d..828bbcc 100644 --- a/packages/kit/src/commands/pack-new.ts +++ b/packages/kit/src/commands/pack-new.ts @@ -1,483 +1,11 @@ -/** - * `aqa pack new ` — scaffold a runnable pack on disk. - * - * The output is the smallest schema-valid pack that `aqa run` will execute - * cleanly against the no-network probe stub: one risk, one scenario whose - * `http_status` oracle expects 200 (which the stub returns by default). - * That gives community authors a green starting point — they then replace - * the placeholder probe URL + add real scenarios. - * - * The CLI accepts `--sut-type api|web|cli|lib|agent|pipeline`. The scaffold - * adapts `applies_when.sut_type` and the example scenario's URL accordingly. - */ - -import { - existsSync, - lstatSync, - mkdirSync, - readFileSync, - renameSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { resolve } from 'node:path'; -import { PackManifest, RiskMap, Scenario } from '@aqa/schemas'; -import { parse as yamlParse, stringify as yamlStringify } from 'yaml'; - -export interface PackNewOptions { - root: string; - slug: string; - sutType: string; - /** Overwrite an existing target directory. Defaults to false. */ - force?: boolean; - description?: string; - author?: string; - license?: string; -} - -/** - * Structured error code returned alongside `error` on failure. Lets - * callers distinguish causes programmatically without regex-matching - * the human-readable error string: - * - * - `EEXIST` — the target pack directory already exists and `force` - * was not set. The right HTTP mapping is 409 Conflict. - * - `EINVAL` — the input was invalid (bad slug, unsupported - * sut-type, slug too long, malformed manifest). 400. - * - `EIO` — filesystem operation failed (permission, disk full, - * cross-device rename). 500. - * - * Always present on failure; callers can safely switch on it. The - * `error` field carries the human-readable detail. - */ -export type PackNewErrorCode = 'EEXIST' | 'EINVAL' | 'EIO'; - -export interface PackNewResult { - ok: boolean; - /** Absolute path to the scaffolded pack directory. Present only on success. */ - packDir?: string; - /** Files created, relative to `packDir`. Present only on success. */ - files?: string[]; - error?: string; - /** Structured cause; see {@link PackNewErrorCode}. Present only on failure. */ - code?: PackNewErrorCode; -} - -const VALID_SUT_TYPES = new Set(['api', 'web', 'cli', 'lib', 'agent', 'pipeline']); -const SLUG_PATTERN = /^[a-z0-9](?:-?[a-z0-9])*$/; -// Derived IDs prepend at most `inv-` (4 chars) and append `-starter` (8 chars). -// `Slug` schema allows up to 64 chars, so the user-supplied slug fragment must -// stay within (64 - 12) = 52 chars or `Scenario.parse` / `RiskMap.parse` will -// later reject the scaffolded files. -const MAX_SLUG_LEN = 52; - -/** - * Build a failure result. `code` defaults to `EINVAL` because the vast - * majority of failures are user-input or schema-validation issues; the - * two non-default sites (`EEXIST` for an existing packDir without - * --force, and `EIO` for actual filesystem operation failures) pass - * the right code explicitly. - */ -function makeError(error: string, code: PackNewErrorCode = 'EINVAL'): PackNewResult { - return { ok: false, error, code }; -} - -/** Starter URL each SUT-type's example probe points at. Stub returns 200 either way. */ -function exampleUrlFor(sutType: string): string { - switch (sutType) { - case 'api': - return '/healthz'; - case 'web': - return '/'; - case 'cli': - return '/usage'; - case 'agent': - return '/agent/ping'; - default: - return '/'; - } -} - -export function runPackNew(opts: PackNewOptions): PackNewResult { - if (!opts.slug || opts.slug.trim() === '') { - return makeError('slug is required'); - } - if (!SLUG_PATTERN.test(opts.slug)) { - return makeError( - `slug "${opts.slug}" must be lowercase alphanumeric with single dashes (matches /^[a-z0-9](?:-?[a-z0-9])*$/)`, - ); - } - if (opts.slug.length > MAX_SLUG_LEN) { - return makeError( - `slug "${opts.slug}" is ${opts.slug.length} chars; max ${MAX_SLUG_LEN} (the scaffold generates derived IDs "scn--starter", "r--starter", and "inv--starter" — the worst-case overhead is 4+8=12 chars, and the underlying Slug schema caps every id at 64)`, - ); - } - if (!VALID_SUT_TYPES.has(opts.sutType)) { - return makeError( - `unsupported sut-type "${opts.sutType}" — must be one of: ${[...VALID_SUT_TYPES].join(', ')}`, - ); - } - - // Scaffold into `/packs//` (not `//`) so the - // pack ends up in a location `aqa run`'s `defaultPacksRoot()` actually - // discovers. Otherwise the user would scaffold a pack, hit `aqa run`, - // and see "0 scenarios" with no clue why. `resolve` always returns an - // absolute path even if `opts.root` was relative; the SLUG_PATTERN - // check above already rejects any slug containing `/` or `\`, so the - // prior `isAbsolute(opts.slug)` branch was unreachable. - const packDir = resolve(opts.root, 'packs', opts.slug); - // Also check the `packs/` parent — a symlinked parent would let - // `mkdirSync(packDir, { recursive: true })` follow the link and write - // outside the project root. Both the parent and the leaf target are - // refused if they're symlinks. - const packsParent = resolve(opts.root, 'packs'); - if (existsSync(packsParent)) { - let parentStat: ReturnType; - try { - parentStat = lstatSync(packsParent); - } catch (e) { - return makeError( - `cannot stat ${packsParent}: ${e instanceof Error ? e.message : String(e)}`, - 'EIO', - ); - } - if (parentStat.isSymbolicLink()) { - return makeError( - `parent directory ${packsParent} is a symlink — refusing to scaffold (would follow the link and write outside the project root)`, - ); - } - // Reject anything that isn't a directory (regular file, socket, - // device) up-front with a clear message — otherwise we'd fail later - // in `mkdirSync` with a generic ENOTDIR that doesn't pinpoint the - // wrong path. - if (!parentStat.isDirectory()) { - return makeError( - `${packsParent} exists but is not a directory — refusing to scaffold (move/remove the file first, then re-run)`, - ); - } - } - // Whether packDir already exists (as anything that's *not* a symlink — - // we explicitly reject symlinks above, but the path could still be a - // regular file rather than a directory; either way `rmSync({recursive, - // force})` will clear it) and therefore needs to be removed before we - // recreate it. We compute this up-front so we can refuse fast (no - // --force + path exists, or symlink at any time) but defer the actual - // destructive rmSync until *after* all schema validation passes. - // Otherwise a scaffold that fails schema validation would still have - // nuked the user's existing pack, with nothing recreated to take its - // place. - let existingPackDirNeedsRm = false; - if (existsSync(packDir)) { - // Use lstat (not stat) so symlinks don't transparently pass the - // directory check — following them with `mkdirSync` later would let a - // malicious or accidental symlink overwrite files outside packDir. - // Failure to stat is treated as a hard refusal: we can't confirm the - // target is safe to overwrite, so we don't. - let isSymlink: boolean; - try { - isSymlink = lstatSync(packDir).isSymbolicLink(); - } catch (e) { - return makeError( - `cannot stat pack directory ${packDir}: ${e instanceof Error ? e.message : String(e)} — refusing to scaffold (cannot confirm path is not a symlink)`, - 'EIO', - ); - } - if (isSymlink) { - return makeError( - `pack directory ${packDir} is a symlink — refusing to scaffold into it (would follow the link and write outside the pack root)`, - ); - } - if (!opts.force) { - return makeError( - `pack directory ${packDir} already exists; pass --force to overwrite`, - 'EEXIST', - ); - } - existingPackDirNeedsRm = true; - } - - const description = opts.description ?? 'Pack scaffolded by aqa pack new'; - const author = opts.author ?? 'You'; - const license = opts.license ?? 'Apache-2.0'; - const exampleUrl = exampleUrlFor(opts.sutType); - - // Build the manifest object and validate against the canonical schema - // before we touch the filesystem. Catches typos in the starter content - // before they end up on disk. - const manifest = { - schema_version: '1' as const, - name: opts.slug, - version: '0.1.0', - description, - author, - license, - applies_when: { - sut_type: [opts.sutType], - }, - templates: [], - scenarios: ['scenarios/starter.yaml'], - risks: ['risks/starter.yaml'], - oracles: [], - probes: [], - }; - const validated = PackManifest.PackManifest.safeParse(manifest); - if (!validated.success) { - return makeError( - `generated manifest failed schema validation (this is a bug in aqa pack new — please report): ${validated.error.message}`, - ); - } - - const scenarioObj = { - schema_version: '1' as const, - id: `scn-${opts.slug}-starter`, - title: `Starter scenario for ${opts.slug} — replace with a real test`, - risk_refs: [`r-${opts.slug}-starter`], - invariant_refs: [`inv-${opts.slug}-starter`], - preconditions: [], - steps: [ - { - id: 'probe-starter', - kind: 'http' as const, - with: { method: 'GET', url: exampleUrl }, - }, - ], - oracles: [ - { - id: 'o-starter-ok', - kind: 'http_status' as const, - // The no-network probe stub returns status=200, so this passes - // out of the box. When you wire a real probe runner the oracle - // will be checked against the actual server response. - with: { expected: 200 }, - }, - ], - tags: [opts.sutType, 'starter'], - }; - const scenarioValid = Scenario.Scenario.safeParse(scenarioObj); - if (!scenarioValid.success) { - return makeError( - `generated scenario failed schema validation (likely a too-long slug): ${scenarioValid.error.message}`, - ); - } - const scenarioYaml = yamlStringify(scenarioObj); - - const riskObj = { - schema_version: '1' as const, - project: opts.slug, - risks: [ - { - id: `r-${opts.slug}-starter`, - category: 'integrity' as const, - title: 'Starter risk — replace with a real one', - severity: 'medium' as const, - likelihood: 'possible' as const, - invariants: [ - { - id: `inv-${opts.slug}-starter`, - statement: - 'Replace this with the real invariant your scenarios prove. The current statement is a placeholder so the pack passes schema validation.', - }, - ], - }, - ], - }; - const riskValid = RiskMap.RiskMap.safeParse(riskObj); - if (!riskValid.success) { - return makeError( - `generated risk map failed schema validation (likely a too-long slug): ${riskValid.error.message}`, - ); - } - const riskYaml = yamlStringify(riskObj); - - const readmeMd = `# ${opts.slug} - -Scaffolded by \`aqa pack new\`. Replace this with a real description. - -## Files - -- \`pack.yaml\` — the manifest. Update \`name\`, \`description\`, and \`applies_when\` to match your project. -- \`scenarios/starter.yaml\` — example scenario. Edit the probe URL + oracle to match real behavior. -- \`risks/starter.yaml\` — risk declaration. Replace the placeholder \`r-${opts.slug}-starter\` with the real risk you're proving. -- \`package.json\` — only used if you publish to npm. The scaffold sets \`name: "${opts.slug}"\` (unscoped) so vendor/copy distribution works as-is. **Before \`npm publish\` you must change \`name\` to a scope you own** (e.g. \`"@your-scope/${opts.slug}"\`), or the publish will fail with "name already taken" / "you do not have permission". \`pack.yaml.name\` (the discovery key) stays as the unscoped slug — those two names are independent. - -## Run it - -Drop this pack under \`/packs/${opts.slug}/\` and reference it from \`.aqa/profiles.yaml\`. To distribute it across projects: publish under your own npm scope (\`@your-scope/${opts.slug}\`) and have consumers either (a) vendor/copy/extract the published tarball into their \`/packs/\` directory, or (b) install it normally via \`npm install\` and add an alias into the \`@aqa\` scope for auto-discovery (\`"@aqa/${opts.slug}": "npm:@your-scope/${opts.slug}"\` in their \`package.json\` — packs under \`/node_modules/@aqa/*\` are auto-discovered). The snippet below is the smallest schema-valid form — both top-level \`schema_version\` and per-profile fields (\`schema_version\`, \`execution_mode\`) are required by \`@aqa/schemas/ProfilesFile\`: - -\`\`\`yaml -schema_version: "1" -profiles: - smoke: - schema_version: "1" - name: smoke - execution_mode: orchestrator - packs: ["${opts.slug}"] - tags: ["${opts.sutType}", "starter"] -\`\`\` - -Then \`aqa run --profile smoke\` will pick it up. - -See the [pack authoring guide](https://github.com/padosoft/agentic-qa-kit/blob/main/docs/PACK-AUTHORING.md) for the full reference. -`; - - // All the content is built and validated. Wrap writes in try/catch so - // any FS failure (permission, file-at-path, partial-write) returns a - // structured error instead of throwing past the CLI's top handler. - // - // Atomic-ish `--force`: when overwriting, we rename the existing pack - // out of the way to a sibling backup path BEFORE writing the new one. - // If the write phase fails (permissions, disk full, partial write), we - // remove the half-written new pack and rename the backup back into - // place — so the user is left with their original pack intact rather - // than neither pack. On success, the backup is removed. The rename - // stays inside the same parent directory, so it's atomic on any sane - // filesystem (no cross-device move). - // - // Symlink safety inside `packDir`: we already rejected `packDir` itself - // as a symlink up top. The rename here moves the *entire* old tree out - // of the way as one atomic operation, so any symlinks living inside - // the old packDir (e.g. `/scenarios` as a symlink) follow the - // rename and never get touched. The fresh `mkdirSync(packDir)` below - // creates a brand-new empty directory, so `mkdirSync(packDir/scenarios)` - // and `writeFileSync(...)` cannot follow any leftover symlink — there - // are none to follow. Stale files from the previous pack are likewise - // impossible: the new packDir starts empty. - let backupDir: string | null = null; - if (existingPackDirNeedsRm) { - backupDir = `${packDir}.aqa-backup-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; - try { - renameSync(packDir, backupDir); - } catch (e) { - return makeError( - `cannot rename existing pack directory ${packDir} → ${backupDir} (needed to make --force non-destructive): ${e instanceof Error ? e.message : String(e)}`, - 'EIO', - ); - } - } - try { - mkdirSync(packDir, { recursive: true }); - mkdirSync(resolve(packDir, 'scenarios'), { recursive: true }); - mkdirSync(resolve(packDir, 'risks'), { recursive: true }); - writeFileSync(resolve(packDir, 'pack.yaml'), yamlStringify(manifest), 'utf8'); - writeFileSync(resolve(packDir, 'scenarios', 'starter.yaml'), scenarioYaml, 'utf8'); - writeFileSync(resolve(packDir, 'risks', 'starter.yaml'), riskYaml, 'utf8'); - writeFileSync(resolve(packDir, 'README.md'), readmeMd, 'utf8'); - writeFileSync( - resolve(packDir, 'package.json'), - JSON.stringify( - { - name: opts.slug, - version: '0.1.0', - description, - license, - author, - // `files` lists what to include in the published tarball. - // We only list directories the scaffold actually creates; if - // the author later adds custom `oracles/` or `probes/`, they - // should extend this array themselves. Listing non-existent - // directories here makes some tooling warn on `npm pack`. - files: ['pack.yaml', 'scenarios', 'risks', 'README.md'], - // No `private: true`. The pack must remain publishable from - // this scaffold: setting `private: true` would make npm refuse - // the publish outright. Note that an author will still need to - // change `name` to a scope they own (e.g. `@your-scope/`) - // before `npm publish` — the README explains this — but - // leaving `private: true` would block them at a less-obvious - // step. For vendor/copy distribution into `/packs/`, - // the package.json is irrelevant either way. - }, - null, - 2, - ), - 'utf8', - ); - } catch (e) { - return rollbackAndError( - packDir, - backupDir, - `cannot write pack files to ${packDir}: ${e instanceof Error ? e.message : String(e)}`, - ); - } - - // Re-validate by parsing what we wrote — catches any serializer divergence. - try { - const roundTrip = PackManifest.PackManifest.safeParse( - yamlParse(readFileSync(resolve(packDir, 'pack.yaml'), 'utf8')), - ); - if (!roundTrip.success) { - return rollbackAndError( - packDir, - backupDir, - `scaffolded pack.yaml failed round-trip validation: ${roundTrip.error.message}`, - ); - } - } catch (e) { - return rollbackAndError( - packDir, - backupDir, - `cannot re-read scaffolded pack.yaml at ${packDir}: ${e instanceof Error ? e.message : String(e)}`, - ); - } - - // Scaffold succeeded — drop the backup. A failure to clean up the - // backup is intentionally silent: the new pack is in place and fully - // usable, and surfacing a partial success would force every caller - // (CLI, tests, future programmatic users) to handle a `warnings` - // shape on the happy path. The worst-case outcome is a stale - // `.aqa-backup-*` sibling that a user can rm manually; the - // randomly-suffixed name makes it obviously not part of the pack. - if (backupDir !== null) { - try { - rmSync(backupDir, { recursive: true, force: true }); - } catch { - // best-effort; new pack is valid either way - } - } - - return { - ok: true, - packDir, - files: [ - 'pack.yaml', - 'scenarios/starter.yaml', - 'risks/starter.yaml', - 'README.md', - 'package.json', - ], - }; -} - -/** - * Restore the user's original pack (renamed to `backupDir`) and return a - * structured error. Called on every failure path after the rename has - * happened, so an interrupted scaffold leaves the working tree in the - * same state it was in before `aqa pack new --force` was invoked. - * - * If `backupDir` is null (no existing pack to begin with), we just clear - * any half-written content at `packDir` and surface the error. - */ -function rollbackAndError( - packDir: string, - backupDir: string | null, - message: string, -): PackNewResult { - try { - rmSync(packDir, { recursive: true, force: true }); - } catch { - // best-effort — surface the original error regardless - } - if (backupDir !== null) { - try { - renameSync(backupDir, packDir); - } catch (restoreErr) { - // We failed to restore. Don't lose the data silently — point the - // user at the backup path so they can recover manually. - return makeError( - `${message} (rollback FAILED — your original pack is at ${backupDir}, please restore it manually: ${restoreErr instanceof Error ? restoreErr.message : String(restoreErr)})`, - 'EIO', - ); - } - } - // Write/round-trip phase failures are all EIO (FS or serializer issues). - return makeError(message, 'EIO'); -} +// `runPackNew` lives in @aqa/pack-author (extracted in v1.9 to break the +// @aqa/kit ↔ @aqa/server build cycle: server's POST /api/packs/scaffold +// needs the same scaffolding logic and used to import it from @aqa/kit, +// which forbade kit from depending on @aqa/server). This file is a +// re-export so existing in-kit imports (CLI, tests) keep working. +export { + runPackNew, + type PackNewErrorCode, + type PackNewOptions, + type PackNewResult, +} from '@aqa/pack-author'; diff --git a/packages/kit/src/commands/report.ts b/packages/kit/src/commands/report.ts new file mode 100644 index 0000000..1a036c6 --- /dev/null +++ b/packages/kit/src/commands/report.ts @@ -0,0 +1,443 @@ +/** + * `aqa report` — renders the on-disk run artifacts (`events.jsonl` + + * `findings.jsonl`) into a Markdown summary and a stable JSON view for + * downstream dashboards. + * + * Loose contract (intentional, see also `runRun`): + * - `aqa run` writes events + findings, NOT a `run.json` (the canonical + * Run shape is reconstructed from the audit chain). `aqa report` does + * that reconstruction here from the first `run_started` and last + * `run_finished` events so reports remain replayable from the audit + * trail alone, without coupling reporter output to a new sidecar file. + * - Reports are written into the same run directory so a junior can hand + * a single `runDir` to a teammate and get the whole story (events, + * findings, replay artifacts, plus the rendered report). + */ + +import { existsSync, lstatSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { renderJson, renderMarkdown } from '@aqa/reporter'; +import { Finding, Run } from '@aqa/schemas'; + +export type ReportFormat = 'md' | 'json' | 'both'; + +// Mirrors @aqa/schemas LongSlug exactly: SlugPattern + max length 256. +// Previous v1.9 iteration used a looser /^[a-z0-9-]{1,80}$/ that both +// capped at 80 instead of 256 AND allowed leading/trailing dashes plus +// `--` runs — so a malformed --run-id could pass this guard yet later +// fail at a Finding.parse() site. Pattern + length checked separately +// so the error message can distinguish "bad characters" from "too long" +// (helpful for hash-suffixed run ids on the edge of the cap). +const RUN_ID_RE = /^[a-z0-9](?:-?[a-z0-9])*$/; +const RUN_ID_MAX_LEN = 256; + +export interface ReportOptions { + root: string; + /** Run id (== directory name under `.aqa/runs/`). Omit to use the latest run. */ + runId?: string; + /** Default 'both'. Controls which artifacts are written. */ + format?: ReportFormat; +} + +export interface ReportOk { + ok: true; + runId: string; + runDir: string; + files: string[]; + findingsCount: number; +} + +export interface ReportErr { + ok: false; + error: string; +} + +export type ReportResult = ReportOk | ReportErr; + +export function runReport(opts: ReportOptions): ReportResult { + const format: ReportFormat = opts.format ?? 'both'; + if (format !== 'md' && format !== 'json' && format !== 'both') { + return { + ok: false, + error: `report: --format must be md | json | both, got "${String(format)}"`, + }; + } + const runsRoot = join(opts.root, '.aqa', 'runs'); + if (!existsSync(runsRoot)) { + return { + ok: false, + error: `report: no runs directory at ${runsRoot} — run \`aqa run --profile smoke\` first`, + }; + } + const runIdResolved = opts.runId ?? latestRunId(runsRoot); + if (!runIdResolved) { + return { + ok: false, + error: `report: no runs found under ${runsRoot} — run \`aqa run --profile smoke\` first`, + }; + } + // Defense-in-depth: a `--run-id` of `../../../etc/passwd` (or similar) + // would otherwise resolve outside `.aqa/runs/`. The schema treats run + // IDs as LongSlug; mirror that constraint at the CLI boundary so a + // typo or malicious input can't drive writes anywhere outside the + // intended directory. + if (runIdResolved.length > RUN_ID_MAX_LEN) { + return { + ok: false, + error: `report: invalid run id — exceeds ${RUN_ID_MAX_LEN}-char LongSlug cap (got ${runIdResolved.length})`, + }; + } + if (!RUN_ID_RE.test(runIdResolved)) { + return { + ok: false, + error: `report: invalid run id "${runIdResolved}" — must match ${RUN_ID_RE.source}`, + }; + } + const runDir = join(runsRoot, runIdResolved); + if (!safeIsDir(runDir)) { + return { ok: false, error: `report: run directory not found: ${runDir}` }; + } + // Refuse symlinked run dirs. A previous run (or an attacker with FS + // write under .aqa/runs/) could leave a symlink pointing outside the + // project; `report.md` / `report.json` writes would then land + // wherever the link points. lstatSync (not statSync) so the test + // doesn't transparently follow the link. + try { + if (lstatSync(runDir).isSymbolicLink()) { + return { + ok: false, + error: `report: refusing to write into symlinked run directory ${runDir}`, + }; + } + } catch (e) { + return { + ok: false, + error: `report: cannot stat run directory ${runDir}: ${e instanceof Error ? e.message : String(e)}`, + }; + } + + const eventsPath = join(runDir, 'events.jsonl'); + // Missing artifacts → fail-fast. A silent empty report on a corrupted + // run dir hides the real problem (the run never wrote its audit trail). + if (!existsSync(eventsPath)) { + return { + ok: false, + error: `report: events.jsonl is missing at ${eventsPath} — run is incomplete or corrupted`, + }; + } + let events: ReadonlyArray>; + try { + events = readJsonl(eventsPath); + } catch (e) { + return { + ok: false, + error: `report: cannot read events.jsonl: ${e instanceof Error ? e.message : String(e)}`, + }; + } + + const findingsPath = join(runDir, 'findings.jsonl'); + if (!existsSync(findingsPath)) { + return { + ok: false, + error: `report: findings.jsonl is missing at ${findingsPath} — run is incomplete or corrupted`, + }; + } + let findings: readonly Finding.Finding[]; + try { + findings = readJsonl(findingsPath).map((raw, idx) => { + const parsed = Finding.Finding.safeParse(raw); + if (!parsed.success) { + throw new Error(`findings.jsonl line ${idx + 1}: ${parsed.error.message}`); + } + return parsed.data; + }); + } catch (e) { + return { + ok: false, + error: `report: cannot read findings.jsonl: ${e instanceof Error ? e.message : String(e)}`, + }; + } + + const runDraft = reconstructRun({ + runId: runIdResolved, + runDir, + events, + findingsCount: findings.length, + }); + // Validate the reconstructed Run against the canonical schema before + // handing it to the renderers. reconstructRun could in theory produce + // a Run.parse-incompatible shape (e.g. terminal state with missing + // finished_at if the audit chain itself is malformed) — surfacing that + // as a structured error keeps `report.json` consumers safe instead of + // shipping a JSON the admin UI silently rejects later. + const runParsed = Run.Run.safeParse(runDraft); + if (!runParsed.success) { + return { + ok: false, + error: `report: reconstructed run failed schema validation (audit chain is malformed): ${runParsed.error.message.split('\n')[0]}`, + }; + } + const run = runParsed.data; + + const written: string[] = []; + // Writes can fail (read-only FS, disk full, permission). Return a + // structured error rather than letting the exception escape into the + // CLI's top-level unhandled-error path so callers get a clean message + // plus an exit code derived from the structured result. + // Per-file symlink check: even with a non-symlinked run dir, a + // pre-existing `report.md`/`report.json` symlink would be followed + // by writeFileSync and let an attacker (or a prior run) redirect the + // writes outside the project. lstat each target before writing. + try { + if (format === 'md' || format === 'both') { + const mdPath = join(runDir, 'report.md'); + if (existsSync(mdPath) && lstatSync(mdPath).isSymbolicLink()) { + return { + ok: false, + error: `report: refusing to overwrite symlinked report file ${mdPath}`, + }; + } + writeFileSync(mdPath, renderMarkdown({ run, findings }), 'utf8'); + written.push(mdPath); + } + if (format === 'json' || format === 'both') { + const jsonPath = join(runDir, 'report.json'); + if (existsSync(jsonPath) && lstatSync(jsonPath).isSymbolicLink()) { + return { + ok: false, + error: `report: refusing to overwrite symlinked report file ${jsonPath}`, + }; + } + writeFileSync( + jsonPath, + `${JSON.stringify(renderJson({ run, findings }), null, 2)}\n`, + 'utf8', + ); + written.push(jsonPath); + } + } catch (e) { + return { + ok: false, + error: `report: cannot write report file: ${e instanceof Error ? e.message : String(e)}`, + }; + } + + return { + ok: true, + runId: runIdResolved, + runDir, + files: written, + findingsCount: findings.length, + }; +} + +function latestRunId(runsRoot: string): string | undefined { + let entries: string[]; + try { + entries = readdirSync(runsRoot); + } catch { + return undefined; + } + // Pick by file mtime, not lexical name. `aqa run --seed` produces + // hash-based IDs (`run-`) that don't sort by recency. mtime is + // monotonic enough for "the most recent run" semantics — lexical name + // is only used as a deterministic tie-breaker (same-millisecond runs). + // + // Filter: only consider directories that look like actual runs + // (presence of events.jsonl, the canonical run-start marker). Without + // this, an unrelated subdirectory under `.aqa/runs/` — or a symlink + // whose target's mtime is newer than any real run — would be selected + // by mtime and either fail with a confusing error or accidentally + // generate a report for the wrong directory. Also reject symlinks at + // the dir-entry level for the same reason as the symlink check in + // runReport: writes into a symlinked dir leak outside the project. + const candidates: Array<{ name: string; mtimeMs: number }> = []; + for (const name of entries) { + if (!RUN_ID_RE.test(name) || name.length > RUN_ID_MAX_LEN) continue; + const dir = join(runsRoot, name); + try { + const lst = lstatSync(dir); + if (lst.isSymbolicLink()) continue; + if (!lst.isDirectory()) continue; + if (!existsSync(join(dir, 'events.jsonl'))) continue; + const st = statSync(dir); + candidates.push({ name, mtimeMs: st.mtimeMs }); + } catch { + // ignore entries we can't stat (broken symlinks, races) + } + } + candidates.sort((a, b) => { + if (b.mtimeMs !== a.mtimeMs) return b.mtimeMs - a.mtimeMs; + return b.name.localeCompare(a.name); + }); + return candidates[0]?.name; +} + +function safeIsDir(p: string): boolean { + try { + return statSync(p).isDirectory(); + } catch { + return false; + } +} + +function readJsonl(path: string): Array> { + // Caller has already confirmed the file exists — this helper only + // returns [] for an empty file (legitimate "zero events" case), + // never for a missing one. + // Strict: a non-empty line that parses as valid JSON but isn't a + // plain object (e.g. `null`, `[]`, `"x"`, `42`) is rejected. Silently + // dropping such lines would turn a corrupted file into a seemingly + // successful report. + const text = readFileSync(path, 'utf8'); + const out: Array> = []; + let lineNo = 0; + for (const raw of text.split('\n')) { + lineNo += 1; + const line = raw.trim(); + if (!line) continue; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch (e) { + throw new Error(`${path} line ${lineNo}: ${e instanceof Error ? e.message : String(e)}`); + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + // typeof null === 'object', so explicit null branch. + const got = parsed === null ? 'null' : Array.isArray(parsed) ? 'array' : typeof parsed; + throw new Error(`${path} line ${lineNo}: expected a JSON object, got ${got}`); + } + out.push(parsed as Record); + } + return out; +} + +interface ReconstructInput { + runId: string; + runDir: string; + events: ReadonlyArray>; + findingsCount: number; +} + +function reconstructRun(input: ReconstructInput): Run.Run { + // Best-effort reconstruction from the audit chain — see file header. The + // reporter only reads a small subset of Run fields, but we still build the + // full schema-conformant object so the JSON report stays valid for the + // admin UI and any external dashboard. + const { runId, runDir, events, findingsCount } = input; + const started = pickEvent(events, 'run_started'); + const finished = pickEvent(events, 'run_finished'); + + const startedAt = readString(started, 'ts') ?? new Date(0).toISOString(); + const finishedAt = readString(finished, 'ts'); + const profile = readPayloadString(started, 'profile') ?? 'unknown'; + const project = readPayloadString(started, 'project') ?? 'unknown'; + const scenariosRun = readPayloadNumber(finished, 'scenarios_run') ?? 0; + const totalsFindings = readPayloadNumber(finished, 'findings') ?? findingsCount; + + const state: Run.Run['state'] = deriveState(finished, scenariosRun); + + const run: Run.Run = { + schema_version: '1', + id: runId, + started_at: startedAt, + ...(finishedAt ? { finished_at: finishedAt } : {}), + state, + project, + profile, + execution_mode: 'orchestrator', + config_snapshot: { + profile, + execution_mode: 'orchestrator', + packs: [], + // Synthetic placeholder: this report path doesn't recompute the + // config hash from disk (the canonical hash is computed by the + // runner at run-time). The admin UI displays config_hash in + // replay copy, so an all-zeros value will be visible — users + // viewing a CLI-rendered report.json should treat this hash as + // a "not computed" sentinel, not a real digest. Encoded as 64 + // zeros (a valid Sha256 by shape) so the JSON still passes + // Run.parse(). + config_hash: '0'.repeat(64), + }, + totals: { + scenarios: scenariosRun, + findings: totalsFindings, + probes: 0, + llm_tokens_in: 0, + llm_tokens_out: 0, + llm_cost_usd: 0, + }, + artifact_dir: runDir, + }; + return run; +} + +function deriveState( + finished: Record | undefined, + scenariosRun: number, +): Run.Run['state'] { + // `runRun` writes `run_finished` on success AND on most failure paths + // (pack errors, scenario errors, missing scenarios, unsafe paths, runtime + // errors, zero scenarios). Treat any non-zero error counter — or a run + // that completed zero scenarios — as `failed` so the report doesn't + // mislabel broken runs as successes. + if (!finished) return 'running'; + const errorKeys = [ + 'pack_errors', + 'scenario_errors', + 'missing_scenarios', + 'unsafe_paths', + 'runtime_errors', + ] as const; + for (const k of errorKeys) { + const v = readPayloadNumber(finished, k); + if (typeof v === 'number' && v > 0) return 'failed'; + } + if (scenariosRun === 0) return 'failed'; + return 'succeeded'; +} + +function pickEvent( + events: ReadonlyArray>, + kind: string, +): Record | undefined { + // run_started: first match; run_finished: last (a re-run within the same + // dir would have appended a new finalization line). Both kinds appear at + // most once today, but the lookup stays defensive. + if (kind === 'run_finished') { + for (let i = events.length - 1; i >= 0; i--) { + if (events[i]?.kind === kind) return events[i]; + } + return undefined; + } + for (const e of events) { + if (e.kind === kind) return e; + } + return undefined; +} + +function readString(obj: Record | undefined, key: string): string | undefined { + const v = obj?.[key]; + return typeof v === 'string' ? v : undefined; +} + +function readPayloadString( + obj: Record | undefined, + key: string, +): string | undefined { + const payload = obj?.payload; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return undefined; + const v = (payload as Record)[key]; + return typeof v === 'string' ? v : undefined; +} + +function readPayloadNumber( + obj: Record | undefined, + key: string, +): number | undefined { + const payload = obj?.payload; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return undefined; + const v = (payload as Record)[key]; + return typeof v === 'number' && Number.isFinite(v) ? v : undefined; +} diff --git a/packages/kit/test/admin-cmd.test.ts b/packages/kit/test/admin-cmd.test.ts new file mode 100644 index 0000000..b81d1d5 --- /dev/null +++ b/packages/kit/test/admin-cmd.test.ts @@ -0,0 +1,167 @@ +/** + * v1.9 — `aqa admin` CLI verb. + * + * Boots the admin SPA + makeApi() in-process on a random port and + * exercises both the static SPA path (`/`) and the API surface + * (`/api/healthz` + a real makeApi() route). Tests use port 0 so they + * can run in parallel without colliding with the kit's default 5173. + */ + +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { runAdmin } from '../dist/commands/admin.js'; + +function makeTempRoot(): string { + return mkdtempSync(join(tmpdir(), 'aqa-admin-cmd-')); +} + +/** Minimal fake admin dist so the boot path can find index.html. */ +function makeFakeAdminDist(): string { + const dir = mkdtempSync(join(tmpdir(), 'aqa-admin-dist-')); + mkdirSync(join(dir, 'assets'), { recursive: true }); + writeFileSync( + join(dir, 'index.html'), + '

fake
', + 'utf8', + ); + writeFileSync( + join(dir, 'assets', 'app.js'), + '/* fake bundle */ console.log("aqa admin smoke");\n', + 'utf8', + ); + return dir; +} + +async function fetchText( + url: string, + init?: { method?: string; headers?: Record }, +): Promise<{ status: number; text: string; contentType: string }> { + const r = await fetch(url, init); + return { + status: r.status, + text: await r.text(), + contentType: r.headers.get('content-type') ?? '', + }; +} + +describe('aqa admin — boot + smoke', () => { + it('boots, serves index.html on /, and returns 200 on /api/healthz', async () => { + const root = makeTempRoot(); + const adminDistDir = makeFakeAdminDist(); + const boot = await runAdmin({ root, port: 0, host: '127.0.0.1', adminDistDir }); + assert.equal(boot.ok, true, `expected ok, got ${JSON.stringify(boot)}`); + if (!boot.ok) return; + try { + assert.ok(boot.port > 0, 'port=0 must resolve to an OS-assigned port'); + assert.ok(boot.url.startsWith('http://127.0.0.1:')); + + const indexRes = await fetchText(`${boot.url}/`); + assert.equal(indexRes.status, 200); + assert.match(indexRes.contentType, /text\/html/); + assert.match(indexRes.text, /
fake<\/div>/); + + const assetRes = await fetchText(`${boot.url}/assets/app.js`); + assert.equal(assetRes.status, 200); + assert.match(assetRes.contentType, /javascript/); + assert.match(assetRes.text, /aqa admin smoke/); + + const healthRes = await fetchText(`${boot.url}/api/healthz`); + assert.equal(healthRes.status, 200); + const health = JSON.parse(healthRes.text) as { ok: boolean }; + assert.equal(health.ok, true); + } finally { + await boot.close(); + } + }); + + it('serves index.html for unknown non-asset paths (SPA fallback)', async () => { + const root = makeTempRoot(); + const adminDistDir = makeFakeAdminDist(); + const boot = await runAdmin({ root, port: 0, host: '127.0.0.1', adminDistDir }); + assert.equal(boot.ok, true); + if (!boot.ok) return; + try { + const res = await fetchText(`${boot.url}/runs/some-deep/route`); + assert.equal(res.status, 200); + assert.match(res.contentType, /text\/html/); + assert.match(res.text, /
fake<\/div>/); + } finally { + await boot.close(); + } + }); + + it('returns 404 for missing /assets/* without falling back to index.html', async () => { + const root = makeTempRoot(); + const adminDistDir = makeFakeAdminDist(); + const boot = await runAdmin({ root, port: 0, host: '127.0.0.1', adminDistDir }); + assert.equal(boot.ok, true); + if (!boot.ok) return; + try { + const res = await fetchText(`${boot.url}/assets/does-not-exist.js`); + assert.equal(res.status, 404); + } finally { + await boot.close(); + } + }); + + it('serves a real makeApi() route (GET /api/orgs)', async () => { + const root = makeTempRoot(); + const adminDistDir = makeFakeAdminDist(); + const boot = await runAdmin({ root, port: 0, host: '127.0.0.1', adminDistDir }); + assert.equal(boot.ok, true); + if (!boot.ok) return; + try { + const res = await fetchText(`${boot.url}/api/orgs`); + // A bare boot has no orgs seeded, but the route exists and the + // adapter wraps it; status MUST NOT be 404. Status may be 200 + // (empty list) or 401/403 depending on auth wiring — we accept + // any non-404 here because the goal is "route is reachable". + assert.notEqual(res.status, 404, `/api/orgs should be reachable, got ${res.status}`); + } finally { + await boot.close(); + } + }); + + it('rejects --port outside 0..65535', async () => { + const root = makeTempRoot(); + const adminDistDir = makeFakeAdminDist(); + const result = await runAdmin({ root, port: 99999, adminDistDir }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /--port must be an integer/); + }); + + it('errors with a clear message when the bundled SPA is missing', async () => { + const root = makeTempRoot(); + const adminDistDir = join(makeTempRoot(), 'definitely-does-not-exist'); + const result = await runAdmin({ root, port: 0, adminDistDir }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /bundled SPA not found/); + }); + + it('refuses to serve files outside the SPA dist (path traversal)', async () => { + const root = makeTempRoot(); + const adminDistDir = makeFakeAdminDist(); + const boot = await runAdmin({ root, port: 0, host: '127.0.0.1', adminDistDir }); + assert.equal(boot.ok, true); + if (!boot.ok) return; + try { + // node:http normalises the URL but a literal `..` segment that + // survives URL parsing must still be refused by the static handler. + const res = await fetch(`${boot.url}/..%2F..%2Fetc%2Fpasswd`); + assert.ok(res.status === 403 || res.status === 200, `got ${res.status}`); + // If it's a 200, it must be the SPA fallback (index.html), not + // a leaked outside-dist file. + if (res.status === 200) { + const text = await res.text(); + assert.match(text, /
fake<\/div>/); + } + } finally { + await boot.close(); + } + }); +}); diff --git a/packages/kit/test/build-bundle.test.ts b/packages/kit/test/build-bundle.test.ts new file mode 100644 index 0000000..8bef773 --- /dev/null +++ b/packages/kit/test/build-bundle.test.ts @@ -0,0 +1,166 @@ +/** + * v1.9 — bundling + publish-prep smoke. + * + * Verifies the two scripts the publish workflow depends on: + * - scripts/build-bundle.mjs emits dist/cli.cjs (CJS, executable, non-empty). + * - scripts/publish-prep.mjs rewrites name, strips @aqa/* deps, pins + * remaining workspace:* deps. + * + * Bundle assertions only run if the bundle has already been built (the + * test isn't a `bun run build` invocation — that would be circular, + * since `pretest` runs tsc but NOT the bundler). On a fresh checkout + * the bundle assertions skip; the publish-prep tests always run. + */ + +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const kitRoot = resolve(here, '..'); +const bundlePath = join(kitRoot, 'dist', 'cli.cjs'); +const metaPath = join(kitRoot, 'dist', 'cli.bundle.meta.json'); + +describe('build-bundle — dist/cli.cjs (skipped if not built)', () => { + it('emits a non-empty, executable bundle with a shebang', () => { + if (!existsSync(bundlePath)) { + console.warn( + `[build-bundle.test] ${bundlePath} not built — skipping bundle assertions. Run \`bun run build\` first.`, + ); + return; + } + const st = statSync(bundlePath); + assert.ok(st.size > 1024, `bundle suspiciously small: ${st.size} bytes`); + // Bundle for a CLI with 18+ workspace deps + zod + yaml + kleur is + // ~hundreds of KB at minimum. 10 MB is a sanity ceiling — anything + // larger probably means a workspace dist accidentally got inlined + // as a string asset (e.g. the admin SPA HTML/JS). + assert.ok(st.size < 10 * 1024 * 1024, `bundle too large: ${st.size} bytes (>10 MB)`); + + const head = readFileSync(bundlePath, 'utf8').slice(0, 100); + assert.ok( + head.startsWith('#!/usr/bin/env node'), + `bundle must start with shebang, got: ${head.slice(0, 40)}`, + ); + + // POSIX: build-bundle.mjs chmod's the output 0o755 so consumers + // can spawn the bin script without an extra chmod step. Windows + // file modes don't carry POSIX execute bits the same way, so this + // assertion is POSIX-only. + if (process.platform !== 'win32') { + assert.ok( + (st.mode & 0o111) !== 0, + `bundle must be executable (mode 755+); got mode ${(st.mode & 0o777).toString(8)}`, + ); + } + }); + + it('emits a sidecar meta JSON (cli.bundle.meta.json)', () => { + if (!existsSync(metaPath)) { + console.warn(`[build-bundle.test] ${metaPath} not built — skipping meta assertions.`); + return; + } + const meta = JSON.parse(readFileSync(metaPath, 'utf8')) as { + bytes: number; + generated_at: string; + }; + assert.equal(typeof meta.bytes, 'number'); + assert.ok(meta.bytes > 0); + assert.match(meta.generated_at, /^\d{4}-\d{2}-\d{2}T/); + }); +}); + +describe('publish-prep — package.json rewrite', () => { + // Build a fresh pkg-like object in-memory and run the rewrite logic + // against it. The real script writes back to disk; we copy the input + // to a temp dir, point the script at it via cwd, then assert the + // result. This keeps the test hermetic — never modifies the real + // packages/kit/package.json even if assertions fail mid-run. + it('substitutes name, strips @aqa/* deps, pins remaining workspace:* deps', async () => { + const dir = mkdtempSync(join(tmpdir(), 'aqa-publish-prep-')); + const fakePkg = { + name: '@aqa/kit', + version: '1.9.0', + dependencies: { + '@aqa/runner': 'workspace:*', + '@aqa/schemas': 'workspace:*', + 'some-third-party-workspace': 'workspace:^1.0.0', + yaml: '^2.6.0', + }, + devDependencies: { + esbuild: '^0.24.0', + }, + aqa: { publishName: '@padosoft/agentic-qa-kit' }, + }; + // Mimic the script's directory layout: scripts/publish-prep.mjs + // lives at /scripts/, so the script resolves package.json + // as `resolve(here, '..', 'package.json')`. Reproduce that here. + const scriptsDir = join(dir, 'scripts'); + const fakeScript = join(scriptsDir, 'publish-prep.mjs'); + const pkgPath = join(dir, 'package.json'); + const realScript = readFileSync(join(kitRoot, 'scripts', 'publish-prep.mjs'), 'utf8'); + // Materialise the temp tree and run. + const { mkdirSync } = await import('node:fs'); + mkdirSync(scriptsDir, { recursive: true }); + writeFileSync(pkgPath, `${JSON.stringify(fakePkg, null, 2)}\n`, 'utf8'); + writeFileSync(fakeScript, realScript, 'utf8'); + + const { spawnSync } = await import('node:child_process'); + const r = spawnSync(process.execPath, [fakeScript], { encoding: 'utf8' }); + assert.equal( + r.status, + 0, + `publish-prep exited ${r.status}\nstdout:${r.stdout}\nstderr:${r.stderr}`, + ); + + const rewritten = JSON.parse(readFileSync(pkgPath, 'utf8')) as { + name: string; + dependencies: Record; + }; + assert.equal(rewritten.name, '@padosoft/agentic-qa-kit'); + // @aqa/* must be GONE entirely — they're bundled into dist/cli.cjs + // and don't exist on any registry, so leaving them would make `bun + // add @padosoft/agentic-qa-kit` fail with "404 — @aqa/runner not + // found on registry" (Copilot iter 1 P1 on PR #55). + assert.equal( + rewritten.dependencies['@aqa/runner'], + undefined, + '@aqa/* deps must be stripped (they are bundled, not published)', + ); + assert.equal(rewritten.dependencies['@aqa/schemas'], undefined); + // Non-@aqa workspace:* deps still get pinned to the kit version + // (kept in case the kit ever depends on a non-aqa workspace). + assert.equal( + rewritten.dependencies['some-third-party-workspace'], + '1.9.0', + 'non-@aqa workspace:* must be pinned to the kit version', + ); + assert.equal(rewritten.dependencies.yaml, '^2.6.0', 'non-workspace deps must be left alone'); + }); + + it('exits non-zero when aqa.publishName is missing', async () => { + const dir = mkdtempSync(join(tmpdir(), 'aqa-publish-prep-')); + const pkgPath = join(dir, 'package.json'); + writeFileSync( + pkgPath, + `${JSON.stringify({ name: '@aqa/kit', version: '1.9.0' }, null, 2)}\n`, + 'utf8', + ); + const scriptsDir = join(dir, 'scripts'); + const fakeScript = join(scriptsDir, 'publish-prep.mjs'); + const { mkdirSync } = await import('node:fs'); + mkdirSync(scriptsDir, { recursive: true }); + writeFileSync( + fakeScript, + readFileSync(join(kitRoot, 'scripts', 'publish-prep.mjs'), 'utf8'), + 'utf8', + ); + const { spawnSync } = await import('node:child_process'); + const r = spawnSync(process.execPath, [fakeScript], { encoding: 'utf8' }); + assert.notEqual(r.status, 0, 'publish-prep must fail when publishName is missing'); + assert.match(r.stderr ?? '', /publishName/); + }); +}); diff --git a/packages/kit/test/install-agent-files-cmd.test.ts b/packages/kit/test/install-agent-files-cmd.test.ts new file mode 100644 index 0000000..aea8ad5 --- /dev/null +++ b/packages/kit/test/install-agent-files-cmd.test.ts @@ -0,0 +1,219 @@ +/** + * v1.9 — `aqa install-agent-files` CLI verb. + * + * Wraps `@aqa/adapters renderForTargets()` and writes the rendered files via + * `writeFileSafe`. These tests assert the behaviour a junior expects from the + * README quick-start: + * - `--targets` parsing (csv, dedup, unknown rejection, empty rejection) + * - default + override of the project name (Slug-conforming) + * - existing files preserved unless `--force` + * - `--dry-run` never touches disk + */ + +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { runInstallAgentFiles } from '../dist/commands/install-agent-files.js'; + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'aqa-install-agent-files-')); +} + +describe('aqa install-agent-files — happy path', () => { + it('writes CLAUDE.md + at least one claude skill for --targets claude', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: 'claude' }); + assert.equal(result.ok, true, `expected ok, got ${JSON.stringify(result)}`); + if (!result.ok) return; + + assert.deepEqual(result.targets, ['claude']); + assert.ok(existsSync(join(root, 'CLAUDE.md')), 'CLAUDE.md must be written'); + const claudeFiles = result.files.filter((f) => f.target === 'claude'); + assert.ok( + claudeFiles.some((f) => f.path.startsWith('.claude/skills/')), + 'at least one .claude/skills/ file expected', + ); + for (const f of claudeFiles) { + assert.equal(f.result, 'created', `${f.path} should be 'created' on a fresh dir`); + } + }); + + it('writes files for every requested target when given --targets claude,codex,gemini,copilot', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ + root, + targets: 'claude,codex,gemini,copilot', + }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.deepEqual([...result.targets].sort(), ['claude', 'codex', 'copilot', 'gemini']); + assert.ok(existsSync(join(root, 'CLAUDE.md'))); + assert.ok(existsSync(join(root, 'AGENTS.md'))); + assert.ok(existsSync(join(root, 'GEMINI.md'))); + assert.ok(existsSync(join(root, '.github', 'copilot-instructions.md'))); + }); + + it('accepts an array form of targets identical to the csv form', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: ['claude', 'codex'] }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.deepEqual(result.targets, ['claude', 'codex']); + }); + + it('embeds the slugified project name in the instruction file (default = dir name)', () => { + // Prefix has internal spaces but NO trailing space — Windows rejects + // path components ending in a space, which would crash mkdtempSync before + // the test even reached its assertions. + const root = mkdtempSync(join(tmpdir(), 'My Junior Project-')); + const result = runInstallAgentFiles({ root, targets: 'claude' }); + assert.equal(result.ok, true); + const claudeMd = readFileSync(join(root, 'CLAUDE.md'), 'utf8'); + // Slugified: lowercase, '-' separators, no spaces. Just assert the file + // contains a slug-looking token referring to "my-junior-project" prefix + // (mkdtemp adds a trailing random suffix that the slugifier keeps). + assert.match(claudeMd, /my-junior-project/, 'project name must be slugified into CLAUDE.md'); + }); + + it('honors --project-name override over the directory-derived default', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ + root, + targets: 'claude', + projectName: 'My Override', + }); + assert.equal(result.ok, true); + const claudeMd = readFileSync(join(root, 'CLAUDE.md'), 'utf8'); + assert.match(claudeMd, /my-override/, 'project-name override must be slugified into CLAUDE.md'); + }); + + it('caps the slugified project name at 64 chars (Slug.max) — Copilot iter 2', () => { + // @aqa/schemas Slug enforces .max(64). A 70-char project name fed into + // init/install-agent-files would otherwise produce a name that + // `aqa validate` rejects. Cap-then-trim-trailing-dashes keeps the slug + // schema-conformant in the worst case. + const root = makeTempDir(); + const result = runInstallAgentFiles({ + root, + targets: 'claude', + projectName: 'a'.repeat(70), + }); + assert.equal(result.ok, true); + const claudeMd = readFileSync(join(root, 'CLAUDE.md'), 'utf8'); + // The slugified name appears in CLAUDE.md's header. It must be exactly + // 64 'a's, never longer. + const match = claudeMd.match(/`(a+)`/); + assert.ok(match, 'expected the slugified project name in a backticked token'); + assert.ok((match?.[1]?.length ?? 0) <= 64, `slug exceeded 64 chars: ${match?.[1]?.length}`); + }); +}); + +describe('aqa install-agent-files — targets validation', () => { + it('returns error when --targets is empty string', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: '' }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /must list at least one target/); + }); + + it('returns error when --targets is an empty array', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: [] }); + assert.equal(result.ok, false); + }); + + it('rejects an unknown target and writes nothing', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: 'claude,mistral' }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /unknown target "mistral"/); + assert.ok( + !existsSync(join(root, 'CLAUDE.md')), + 'no file should be written on validation error', + ); + assert.ok(!existsSync(join(root, 'AGENTS.md'))); + }); + + it('de-duplicates duplicate targets without erroring', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: 'claude,claude,codex' }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.deepEqual(result.targets, ['claude', 'codex']); + }); + + it('normalizes target casing (CLAUDE → claude)', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: 'CLAUDE,Codex' }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.deepEqual(result.targets, ['claude', 'codex']); + }); + + it('trims whitespace and drops empties in the array form (Copilot iter 1)', () => { + // Regression: previously the array form was used as-is, so a token like + // 'claude ' would be classified as an unknown target. Both CSV and array + // inputs must produce identical normalized results. + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: ['claude ', '', ' codex'] }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.deepEqual(result.targets, ['claude', 'codex']); + }); +}); + +describe('aqa install-agent-files — overwrite semantics', () => { + it('skips existing files by default (skipped-exists)', () => { + const root = makeTempDir(); + // First pass creates CLAUDE.md. + const first = runInstallAgentFiles({ root, targets: 'claude' }); + assert.equal(first.ok, true); + if (!first.ok) return; + const originalContents = readFileSync(join(root, 'CLAUDE.md'), 'utf8'); + // Tamper to make sure overwrite=false really preserves user edits. + writeFileSync(join(root, 'CLAUDE.md'), '# my custom edits\n', 'utf8'); + + const second = runInstallAgentFiles({ root, targets: 'claude' }); + assert.equal(second.ok, true); + if (!second.ok) return; + const claudeMd = second.files.find((f) => f.path === 'CLAUDE.md'); + assert.equal(claudeMd?.result, 'skipped-exists'); + assert.equal( + readFileSync(join(root, 'CLAUDE.md'), 'utf8'), + '# my custom edits\n', + 'user edits must be preserved without --force', + ); + // Sanity: the rendered template differs from user edits — guards against + // the writer accidentally overwriting them with a byte-identical payload. + assert.notEqual(originalContents, '# my custom edits\n'); + }); + + it('overwrites existing files when overwrite=true', () => { + const root = makeTempDir(); + mkdirSync(root, { recursive: true }); + writeFileSync(join(root, 'CLAUDE.md'), '# my custom edits\n', 'utf8'); + + const result = runInstallAgentFiles({ root, targets: 'claude', overwrite: true }); + assert.equal(result.ok, true); + if (!result.ok) return; + const claudeMd = result.files.find((f) => f.path === 'CLAUDE.md'); + assert.equal(claudeMd?.result, 'overwritten'); + assert.notEqual(readFileSync(join(root, 'CLAUDE.md'), 'utf8'), '# my custom edits\n'); + }); + + it('dry-run never writes to disk', () => { + const root = makeTempDir(); + const result = runInstallAgentFiles({ root, targets: 'claude,codex', dryRun: true }); + assert.equal(result.ok, true); + if (!result.ok) return; + for (const f of result.files) { + assert.equal(f.result, 'dry-run'); + } + assert.ok(!existsSync(join(root, 'CLAUDE.md')), 'CLAUDE.md must not exist after dry-run'); + assert.ok(!existsSync(join(root, 'AGENTS.md')), 'AGENTS.md must not exist after dry-run'); + }); +}); diff --git a/packages/kit/test/report-cmd.test.ts b/packages/kit/test/report-cmd.test.ts new file mode 100644 index 0000000..1a4444b --- /dev/null +++ b/packages/kit/test/report-cmd.test.ts @@ -0,0 +1,587 @@ +/** + * v1.9 — `aqa report` CLI verb. + * + * Renders the on-disk run artifacts into report.md + report.json. These + * tests build a synthetic run directory (events.jsonl + findings.jsonl) + * because runRun is async, network-touched in some configurations, and + * over-coupled for a focused reporter test. The synthetic dir matches the + * canonical schema shapes from @aqa/schemas Run/Finding. + */ + +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { runReport } from '../dist/commands/report.js'; + +const RUN_ID = '20260520-000000-runabcdef'; +const STARTED_AT = '2026-05-20T00:00:00.000Z'; +const FINISHED_AT = '2026-05-20T00:00:30.000Z'; + +// Slight sleep used to put a real mtime delta between consecutive run dirs +// in the "latest run" test. Some filesystems otherwise group rapid mkdirs +// under the same mtimeMs and force the tie-breaker into action. +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +function makeTempRoot(): string { + return mkdtempSync(join(tmpdir(), 'aqa-report-')); +} + +function makeRunDir(root: string, runId: string): string { + const dir = join(root, '.aqa', 'runs', runId); + mkdirSync(dir, { recursive: true }); + return dir; +} + +function sha256Hex(s: string): string { + return createHash('sha256').update(s, 'utf8').digest('hex'); +} + +/** Build minimal valid hash-chained events.jsonl. */ +function writeEvents( + runDir: string, + opts: { runId: string; profile: string; project: string; findingsCount: number }, +): void { + const events: Array> = []; + let prev: string | null = null; + function append(partial: Omit, 'seq' | 'prev_hash' | 'hash'>): void { + const seq = events.length; + // Hash recomputation here is a stub — the writer's exact canonicalization + // is exercised in @aqa/runner / @aqa/compliance tests. `aqa report` + // doesn't validate the chain (it just parses fields), so any + // deterministic stub hash keeps schema.parse happy. + const body = JSON.stringify({ ...partial, seq }); + const hash = sha256Hex((prev ?? '') + body); + const evt = { schema_version: '1', seq, prev_hash: prev, hash, ...partial }; + events.push(evt); + prev = hash; + } + append({ + ts: STARTED_AT, + run_id: opts.runId, + kind: 'run_started', + actor: { type: 'orchestrator', id: 'aqa-cli' }, + payload: { profile: opts.profile, project: opts.project }, + }); + append({ + ts: FINISHED_AT, + run_id: opts.runId, + kind: 'run_finished', + actor: { type: 'orchestrator', id: 'aqa-cli' }, + payload: { + scenarios_run: 2, + findings: opts.findingsCount, + pack_errors: 0, + scenario_errors: 0, + missing_scenarios: 0, + unsafe_paths: 0, + runtime_errors: 0, + }, + }); + writeFileSync( + join(runDir, 'events.jsonl'), + `${events.map((e) => JSON.stringify(e)).join('\n')}\n`, + 'utf8', + ); +} + +function writeFindings(runDir: string, count: number, runId: string = RUN_ID): void { + const lines: string[] = []; + for (let i = 0; i < count; i++) { + const finding = { + schema_version: '1', + id: `AQA-2026-${String(i + 1).padStart(4, '0')}`, + run_id: runId, + risk_id: 'r-example', + scenario_id: 'scn-example-demo', + title: `Synthetic finding ${i + 1}`, + severity: i === 0 ? 'critical' : 'low', + status: 'draft', + summary: 'reporter smoke test', + evidence: [], + execution_mode: 'orchestrator', + verification_floor: 'scenario_level', + confidence: 0.5, + discovered_at: STARTED_AT, + }; + lines.push(JSON.stringify(finding)); + } + writeFileSync( + join(runDir, 'findings.jsonl'), + `${lines.join('\n')}${lines.length ? '\n' : ''}`, + 'utf8', + ); +} + +describe('aqa report — happy path', () => { + it('renders both report.md and report.json for the explicit run-id', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 2 }); + writeFindings(runDir, 2); + + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, true, `expected ok, got ${JSON.stringify(result)}`); + if (!result.ok) return; + + assert.equal(result.runId, RUN_ID); + assert.equal(result.findingsCount, 2); + assert.ok(existsSync(join(runDir, 'report.md')), 'report.md must exist'); + assert.ok(existsSync(join(runDir, 'report.json')), 'report.json must exist'); + + const md = readFileSync(join(runDir, 'report.md'), 'utf8'); + assert.match(md, /# AQA report/); + assert.match(md, /demo/); + assert.match(md, /AQA-2026-0001/); + assert.match(md, /Synthetic finding 1/); + + const json = JSON.parse(readFileSync(join(runDir, 'report.json'), 'utf8')) as { + schema_version: string; + run: { id: string; project: string; profile: string; state: string }; + findings: Array<{ id: string }>; + summary: { total: number; severities: Record }; + }; + assert.equal(json.schema_version, '1'); + assert.equal(json.run.id, RUN_ID); + assert.equal(json.run.project, 'demo'); + assert.equal(json.run.profile, 'smoke'); + assert.equal(json.run.state, 'succeeded'); + assert.equal(json.findings.length, 2); + assert.equal(json.summary.total, 2); + assert.equal(json.summary.severities.critical, 1); + assert.equal(json.summary.severities.low, 1); + }); + + it('defaults to the latest run by file mtime, not lexical name (Copilot iter 1 P2)', async () => { + // Critical correctness: `aqa run --seed` produces hash-based IDs + // (run-) that do NOT sort by recency. Picking by mtime keeps + // "latest" honest in mixed-naming directories. Here we intentionally + // create the lexically-EARLIER name LAST so a name-based sort would + // pick the wrong dir. + const root = makeTempRoot(); + const earlierName = 'run-aaaa-but-newer'; // lexically earlier + const olderName = 'run-zzzz-but-older'; // lexically later + const olderDir = makeRunDir(root, olderName); + writeEvents(olderDir, { + runId: olderName, + profile: 'smoke', + project: 'demo', + findingsCount: 1, + }); + writeFindings(olderDir, 1, olderName); + await sleep(20); + const newerDir = makeRunDir(root, earlierName); + writeEvents(newerDir, { + runId: earlierName, + profile: 'release-gate', + project: 'demo', + findingsCount: 3, + }); + writeFindings(newerDir, 3, earlierName); + + const result = runReport({ root }); + assert.equal(result.ok, true); + if (!result.ok) return; + // mtime-newer dir wins even though its name sorts EARLIER lexically. + assert.equal(result.runId, earlierName); + assert.equal(result.findingsCount, 3); + }); + + it('emits only report.md when format=md', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 0 }); + writeFindings(runDir, 0); + + const result = runReport({ root, runId: RUN_ID, format: 'md' }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.ok(existsSync(join(runDir, 'report.md'))); + assert.ok(!existsSync(join(runDir, 'report.json')), 'report.json must not exist for format=md'); + assert.equal(result.files.length, 1); + }); + + it('emits only report.json when format=json', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 0 }); + writeFindings(runDir, 0); + + const result = runReport({ root, runId: RUN_ID, format: 'json' }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.ok(!existsSync(join(runDir, 'report.md'))); + assert.ok(existsSync(join(runDir, 'report.json'))); + }); + + it('handles zero findings without crashing', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 0 }); + writeFindings(runDir, 0); + + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, true); + if (!result.ok) return; + assert.equal(result.findingsCount, 0); + const md = readFileSync(join(runDir, 'report.md'), 'utf8'); + assert.match(md, /No findings/); + }); +}); + +describe('aqa report — error cases', () => { + it('returns error when .aqa/runs does not exist', () => { + const root = makeTempRoot(); + const result = runReport({ root }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /no runs directory/); + }); + + it('returns error when --run-id points to a missing dir', () => { + const root = makeTempRoot(); + makeRunDir(root, RUN_ID); + const result = runReport({ root, runId: 'does-not-exist' }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /run directory not found/); + }); + + it('returns error when .aqa/runs exists but is empty', () => { + const root = makeTempRoot(); + mkdirSync(join(root, '.aqa', 'runs'), { recursive: true }); + const result = runReport({ root }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /no runs found/); + }); + + it('returns error on malformed JSONL line in events.jsonl', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeFileSync(join(runDir, 'events.jsonl'), '{not json\n', 'utf8'); + writeFileSync(join(runDir, 'findings.jsonl'), '', 'utf8'); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /cannot read events\.jsonl/); + }); + + it('returns error when events.jsonl is missing (Copilot iter 1 P1)', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + // findings.jsonl present, events.jsonl missing + writeFileSync(join(runDir, 'findings.jsonl'), '', 'utf8'); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /events\.jsonl is missing/); + }); + + it('returns error when findings.jsonl is missing (Copilot iter 1 P1)', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 0 }); + // findings.jsonl intentionally not created + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /findings\.jsonl is missing/); + }); + + it('rejects a --run-id that would escape .aqa/runs via traversal (Copilot iter 1)', () => { + const root = makeTempRoot(); + // .aqa/runs has to exist or we hit the prior guard first + mkdirSync(join(root, '.aqa', 'runs'), { recursive: true }); + const result = runReport({ root, runId: '../../etc/passwd' }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /invalid run id/); + }); + + it('rejects a --run-id containing characters outside [a-z0-9-]', () => { + const root = makeTempRoot(); + mkdirSync(join(root, '.aqa', 'runs'), { recursive: true }); + const result = runReport({ root, runId: 'NOT_A_SLUG' }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /invalid run id/); + }); + + it('rejects a --run-id with leading/trailing dashes (LongSlug parity — Copilot iter 2)', () => { + const root = makeTempRoot(); + mkdirSync(join(root, '.aqa', 'runs'), { recursive: true }); + for (const bad of ['-leading', 'trailing-', 'double--dash']) { + const result = runReport({ root, runId: bad }); + assert.equal(result.ok, false, `${bad} must be rejected`); + if (result.ok) continue; + assert.match(result.error, /invalid run id/); + } + }); + + it('rejects a --run-id longer than 256 chars (LongSlug cap — Copilot iter 2)', () => { + const root = makeTempRoot(); + mkdirSync(join(root, '.aqa', 'runs'), { recursive: true }); + const result = runReport({ root, runId: 'a'.repeat(257) }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /exceeds 256-char/); + }); + + it('refuses a run dir that is actually a symlink (Copilot iter 2)', () => { + const root = makeTempRoot(); + const runsRoot = join(root, '.aqa', 'runs'); + mkdirSync(runsRoot, { recursive: true }); + const realDir = makeRunDir(root, 'real-target'); + writeEvents(realDir, { + runId: 'real-target', + profile: 'smoke', + project: 'demo', + findingsCount: 0, + }); + writeFindings(realDir, 0, 'real-target'); + const linkPath = join(runsRoot, 'sneaky-link'); + try { + symlinkSync(realDir, linkPath, 'dir'); + } catch { + // Windows without dev-mode / non-admin can't create symlinks — + // skip this guard by returning instead of asserting; the + // production behaviour is exercised on Linux CI anyway. + return; + } + const result = runReport({ root, runId: 'sneaky-link' }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /symlinked run directory/); + }); + + it('rejects a JSONL line that is valid JSON but not a plain object (Copilot iter 2)', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeFileSync(join(runDir, 'events.jsonl'), 'null\n', 'utf8'); + writeFileSync(join(runDir, 'findings.jsonl'), '', 'utf8'); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /expected a JSON object, got null/); + }); + + it('rejects a JSONL line that parses as a JSON array (Copilot iter 2)', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeFileSync(join(runDir, 'events.jsonl'), '[1, 2, 3]\n', 'utf8'); + writeFileSync(join(runDir, 'findings.jsonl'), '', 'utf8'); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /expected a JSON object, got array/); + }); + + it('refuses to overwrite a symlinked report.md (Copilot iter 3)', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 0 }); + writeFindings(runDir, 0); + const outside = join(makeTempRoot(), 'evil-report.md'); + writeFileSync(outside, '# attacker controlled\n', 'utf8'); + try { + symlinkSync(outside, join(runDir, 'report.md'), 'file'); + } catch { + return; // Windows non-admin can't create symlinks; skip. + } + const result = runReport({ root, runId: RUN_ID, format: 'md' }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /refusing to overwrite symlinked report file/); + // Crucial: the attacker-controlled file MUST be untouched. + assert.equal(readFileSync(outside, 'utf8'), '# attacker controlled\n'); + }); + + it('latestRunId only considers dirs with events.jsonl (Copilot iter 3)', async () => { + const root = makeTempRoot(); + // Two real run dirs (older + newer) + a non-run subdir that's newer + // than both but lacks events.jsonl. The non-run dir must NOT win. + const olderDir = makeRunDir(root, 'run-aaaa'); + writeEvents(olderDir, { + runId: 'run-aaaa', + profile: 'smoke', + project: 'demo', + findingsCount: 0, + }); + writeFindings(olderDir, 0, 'run-aaaa'); + await sleep(20); + const newerDir = makeRunDir(root, 'run-bbbb'); + writeEvents(newerDir, { + runId: 'run-bbbb', + profile: 'smoke', + project: 'demo', + findingsCount: 0, + }); + writeFindings(newerDir, 0, 'run-bbbb'); + await sleep(20); + // Pollute .aqa/runs with a NEWER non-run directory. + mkdirSync(join(root, '.aqa', 'runs', 'readme-stash'), { recursive: true }); + + const result = runReport({ root }); + assert.equal(result.ok, true); + if (!result.ok) return; + // Must be the most-recent REAL run, not the polluting subdirectory. + assert.equal(result.runId, 'run-bbbb'); + }); +}); + +describe('aqa report — state reconstruction (Copilot iter 1 P1)', () => { + it('marks state=failed when run_finished payload has pack_errors > 0', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + // Custom events with non-zero pack_errors + const events = [ + { + schema_version: '1', + seq: 0, + prev_hash: null, + hash: '0'.repeat(64), + ts: STARTED_AT, + run_id: RUN_ID, + kind: 'run_started', + actor: { type: 'orchestrator', id: 'aqa-cli' }, + payload: { profile: 'smoke', project: 'demo' }, + }, + { + schema_version: '1', + seq: 1, + prev_hash: '0'.repeat(64), + hash: '1'.repeat(64), + ts: FINISHED_AT, + run_id: RUN_ID, + kind: 'run_finished', + actor: { type: 'orchestrator', id: 'aqa-cli' }, + payload: { + scenarios_run: 0, + findings: 0, + pack_errors: 1, + scenario_errors: 0, + missing_scenarios: 0, + unsafe_paths: 0, + runtime_errors: 0, + }, + }, + ]; + writeFileSync( + join(runDir, 'events.jsonl'), + `${events.map((e) => JSON.stringify(e)).join('\n')}\n`, + 'utf8', + ); + writeFindings(runDir, 0); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, true); + const json = JSON.parse(readFileSync(join(runDir, 'report.json'), 'utf8')) as { + run: { state: string }; + }; + assert.equal(json.run.state, 'failed'); + }); + + it('marks state=failed when scenarios_run is 0 even with no error counters', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 0 }); + // writeEvents above sets scenarios_run: 2 — override with a custom file + const events = [ + { + schema_version: '1', + seq: 0, + prev_hash: null, + hash: '0'.repeat(64), + ts: STARTED_AT, + run_id: RUN_ID, + kind: 'run_started', + actor: { type: 'orchestrator', id: 'aqa-cli' }, + payload: { profile: 'smoke', project: 'demo' }, + }, + { + schema_version: '1', + seq: 1, + prev_hash: '0'.repeat(64), + hash: '1'.repeat(64), + ts: FINISHED_AT, + run_id: RUN_ID, + kind: 'run_finished', + actor: { type: 'orchestrator', id: 'aqa-cli' }, + payload: { + scenarios_run: 0, + findings: 0, + pack_errors: 0, + scenario_errors: 0, + missing_scenarios: 0, + unsafe_paths: 0, + runtime_errors: 0, + }, + }, + ]; + writeFileSync( + join(runDir, 'events.jsonl'), + `${events.map((e) => JSON.stringify(e)).join('\n')}\n`, + 'utf8', + ); + writeFindings(runDir, 0); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, true); + const json = JSON.parse(readFileSync(join(runDir, 'report.json'), 'utf8')) as { + run: { state: string }; + }; + assert.equal(json.run.state, 'failed'); + }); + + it('marks state=running when no run_finished event is present', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + const events = [ + { + schema_version: '1', + seq: 0, + prev_hash: null, + hash: '0'.repeat(64), + ts: STARTED_AT, + run_id: RUN_ID, + kind: 'run_started', + actor: { type: 'orchestrator', id: 'aqa-cli' }, + payload: { profile: 'smoke', project: 'demo' }, + }, + ]; + writeFileSync( + join(runDir, 'events.jsonl'), + `${events.map((e) => JSON.stringify(e)).join('\n')}\n`, + 'utf8', + ); + writeFindings(runDir, 0); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, true); + const json = JSON.parse(readFileSync(join(runDir, 'report.json'), 'utf8')) as { + run: { state: string; finished_at?: string }; + }; + assert.equal(json.run.state, 'running'); + assert.equal(json.run.finished_at, undefined); + }); + + it('returns error on schema-invalid finding', () => { + const root = makeTempRoot(); + const runDir = makeRunDir(root, RUN_ID); + writeEvents(runDir, { runId: RUN_ID, profile: 'smoke', project: 'demo', findingsCount: 1 }); + writeFileSync(join(runDir, 'findings.jsonl'), `${JSON.stringify({ id: 'broken' })}\n`, 'utf8'); + const result = runReport({ root, runId: RUN_ID }); + assert.equal(result.ok, false); + if (result.ok) return; + assert.match(result.error, /cannot read findings\.jsonl/); + }); +}); diff --git a/packages/pack-author/README.md b/packages/pack-author/README.md new file mode 100644 index 0000000..c20c4e2 --- /dev/null +++ b/packages/pack-author/README.md @@ -0,0 +1,23 @@ +# `@aqa/pack-author` + +Pack scaffolding primitives — shared between `@aqa/kit` (CLI) and `@aqa/server` (API). + +## Why this package exists + +Both the CLI (`aqa pack new `) and the server (`POST /api/packs/scaffold`) need to scaffold runnable AQA packs on disk. Originally the logic lived inside `@aqa/kit` and `@aqa/server` imported it from there. When the v1.9 `aqa admin` command needed `@aqa/kit` to depend on `@aqa/server` (for `makeApi()`), the cycle made the topological build non-deterministic and the kit's TypeScript compile started failing in CI. + +This package breaks the cycle by owning the scaffolding logic. Both kit and server depend on `@aqa/pack-author`; neither depends on the other. + +## API surface + +```ts +import { runPackNew } from '@aqa/pack-author'; +import type { PackNewOptions, PackNewResult, PackNewErrorCode } from '@aqa/pack-author'; +``` + +- `runPackNew(opts: PackNewOptions): PackNewResult` — synchronous; creates `/packs//` with a schema-valid `pack.yaml`, one starter scenario, one starter risk, README, and package.json. Atomic-ish `--force`: a failed scaffold restores the original pack from a backup directory. +- `PackNewErrorCode = 'EEXIST' | 'EINVAL' | 'EIO'` — stable error codes that callers can map to HTTP status / CLI exit codes without regex-matching the human-readable message. + +## Tests + +Heavy behaviour coverage lives in `packages/kit/test/pack-new.test.ts` (which calls through the kit's 5-line re-export shim). This package's own `test/pack-author.test.ts` is a package-boundary smoke — verifies the named export shape and the structured-error contract. diff --git a/packages/pack-author/package.json b/packages/pack-author/package.json new file mode 100644 index 0000000..2df52a5 --- /dev/null +++ b/packages/pack-author/package.json @@ -0,0 +1,25 @@ +{ + "name": "@aqa/pack-author", + "version": "0.0.1", + "description": "Pack scaffolding primitives — shared between @aqa/kit (CLI) and @aqa/server (API).", + "type": "module", + "license": "Apache-2.0", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" } + }, + "files": ["dist", "README.md"], + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "pretest": "tsc -p tsconfig.json", + "test": "node --experimental-strip-types --no-warnings=ExperimentalWarning --test test/pack-author.test.ts", + "clean": "node --input-type=module -e \"import { rmSync } from 'node:fs'; for (const p of ['dist','.tsbuildinfo']) { try { rmSync(p, { recursive: true, force: true }); } catch {} }\"" + }, + "dependencies": { + "@aqa/schemas": "workspace:*", + "yaml": "^2.6.0" + }, + "publishConfig": { "access": "public" } +} diff --git a/packages/pack-author/src/index.ts b/packages/pack-author/src/index.ts new file mode 100644 index 0000000..d9a2a4d --- /dev/null +++ b/packages/pack-author/src/index.ts @@ -0,0 +1,483 @@ +/** + * `aqa pack new ` — scaffold a runnable pack on disk. + * + * The output is the smallest schema-valid pack that `aqa run` will execute + * cleanly against the no-network probe stub: one risk, one scenario whose + * `http_status` oracle expects 200 (which the stub returns by default). + * That gives community authors a green starting point — they then replace + * the placeholder probe URL + add real scenarios. + * + * The CLI accepts `--sut-type api|web|cli|lib|agent|pipeline`. The scaffold + * adapts `applies_when.sut_type` and the example scenario's URL accordingly. + */ + +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { resolve } from 'node:path'; +import { PackManifest, RiskMap, Scenario } from '@aqa/schemas'; +import { parse as yamlParse, stringify as yamlStringify } from 'yaml'; + +export interface PackNewOptions { + root: string; + slug: string; + sutType: string; + /** Overwrite an existing target directory. Defaults to false. */ + force?: boolean; + description?: string; + author?: string; + license?: string; +} + +/** + * Structured error code returned alongside `error` on failure. Lets + * callers distinguish causes programmatically without regex-matching + * the human-readable error string: + * + * - `EEXIST` — the target pack directory already exists and `force` + * was not set. The right HTTP mapping is 409 Conflict. + * - `EINVAL` — the input was invalid (bad slug, unsupported + * sut-type, slug too long, malformed manifest). 400. + * - `EIO` — filesystem operation failed (permission, disk full, + * cross-device rename). 500. + * + * Always present on failure; callers can safely switch on it. The + * `error` field carries the human-readable detail. + */ +export type PackNewErrorCode = 'EEXIST' | 'EINVAL' | 'EIO'; + +export interface PackNewResult { + ok: boolean; + /** Absolute path to the scaffolded pack directory. Present only on success. */ + packDir?: string; + /** Files created, relative to `packDir`. Present only on success. */ + files?: string[]; + error?: string; + /** Structured cause; see {@link PackNewErrorCode}. Present only on failure. */ + code?: PackNewErrorCode; +} + +const VALID_SUT_TYPES = new Set(['api', 'web', 'cli', 'lib', 'agent', 'pipeline']); +const SLUG_PATTERN = /^[a-z0-9](?:-?[a-z0-9])*$/; +// Derived IDs prepend at most `inv-` (4 chars) and append `-starter` (8 chars). +// `Slug` schema allows up to 64 chars, so the user-supplied slug fragment must +// stay within (64 - 12) = 52 chars or `Scenario.parse` / `RiskMap.parse` will +// later reject the scaffolded files. +const MAX_SLUG_LEN = 52; + +/** + * Build a failure result. `code` defaults to `EINVAL` because the vast + * majority of failures are user-input or schema-validation issues; the + * two non-default sites (`EEXIST` for an existing packDir without + * --force, and `EIO` for actual filesystem operation failures) pass + * the right code explicitly. + */ +function makeError(error: string, code: PackNewErrorCode = 'EINVAL'): PackNewResult { + return { ok: false, error, code }; +} + +/** Starter URL each SUT-type's example probe points at. Stub returns 200 either way. */ +function exampleUrlFor(sutType: string): string { + switch (sutType) { + case 'api': + return '/healthz'; + case 'web': + return '/'; + case 'cli': + return '/usage'; + case 'agent': + return '/agent/ping'; + default: + return '/'; + } +} + +export function runPackNew(opts: PackNewOptions): PackNewResult { + if (!opts.slug || opts.slug.trim() === '') { + return makeError('slug is required'); + } + if (!SLUG_PATTERN.test(opts.slug)) { + return makeError( + `slug "${opts.slug}" must be lowercase alphanumeric with single dashes (matches /^[a-z0-9](?:-?[a-z0-9])*$/)`, + ); + } + if (opts.slug.length > MAX_SLUG_LEN) { + return makeError( + `slug "${opts.slug}" is ${opts.slug.length} chars; max ${MAX_SLUG_LEN} (the scaffold generates derived IDs "scn--starter", "r--starter", and "inv--starter" — the worst-case overhead is 4+8=12 chars, and the underlying Slug schema caps every id at 64)`, + ); + } + if (!VALID_SUT_TYPES.has(opts.sutType)) { + return makeError( + `unsupported sut-type "${opts.sutType}" — must be one of: ${[...VALID_SUT_TYPES].join(', ')}`, + ); + } + + // Scaffold into `/packs//` (not `//`) so the + // pack ends up in a location `aqa run`'s `defaultPacksRoot()` actually + // discovers. Otherwise the user would scaffold a pack, hit `aqa run`, + // and see "0 scenarios" with no clue why. `resolve` always returns an + // absolute path even if `opts.root` was relative; the SLUG_PATTERN + // check above already rejects any slug containing `/` or `\`, so the + // prior `isAbsolute(opts.slug)` branch was unreachable. + const packDir = resolve(opts.root, 'packs', opts.slug); + // Also check the `packs/` parent — a symlinked parent would let + // `mkdirSync(packDir, { recursive: true })` follow the link and write + // outside the project root. Both the parent and the leaf target are + // refused if they're symlinks. + const packsParent = resolve(opts.root, 'packs'); + if (existsSync(packsParent)) { + let parentStat: ReturnType; + try { + parentStat = lstatSync(packsParent); + } catch (e) { + return makeError( + `cannot stat ${packsParent}: ${e instanceof Error ? e.message : String(e)}`, + 'EIO', + ); + } + if (parentStat.isSymbolicLink()) { + return makeError( + `parent directory ${packsParent} is a symlink — refusing to scaffold (would follow the link and write outside the project root)`, + ); + } + // Reject anything that isn't a directory (regular file, socket, + // device) up-front with a clear message — otherwise we'd fail later + // in `mkdirSync` with a generic ENOTDIR that doesn't pinpoint the + // wrong path. + if (!parentStat.isDirectory()) { + return makeError( + `${packsParent} exists but is not a directory — refusing to scaffold (move/remove the file first, then re-run)`, + ); + } + } + // Whether packDir already exists (as anything that's *not* a symlink — + // we explicitly reject symlinks above, but the path could still be a + // regular file rather than a directory; either way `rmSync({recursive, + // force})` will clear it) and therefore needs to be removed before we + // recreate it. We compute this up-front so we can refuse fast (no + // --force + path exists, or symlink at any time) but defer the actual + // destructive rmSync until *after* all schema validation passes. + // Otherwise a scaffold that fails schema validation would still have + // nuked the user's existing pack, with nothing recreated to take its + // place. + let existingPackDirNeedsRm = false; + if (existsSync(packDir)) { + // Use lstat (not stat) so symlinks don't transparently pass the + // directory check — following them with `mkdirSync` later would let a + // malicious or accidental symlink overwrite files outside packDir. + // Failure to stat is treated as a hard refusal: we can't confirm the + // target is safe to overwrite, so we don't. + let isSymlink: boolean; + try { + isSymlink = lstatSync(packDir).isSymbolicLink(); + } catch (e) { + return makeError( + `cannot stat pack directory ${packDir}: ${e instanceof Error ? e.message : String(e)} — refusing to scaffold (cannot confirm path is not a symlink)`, + 'EIO', + ); + } + if (isSymlink) { + return makeError( + `pack directory ${packDir} is a symlink — refusing to scaffold into it (would follow the link and write outside the pack root)`, + ); + } + if (!opts.force) { + return makeError( + `pack directory ${packDir} already exists; pass --force to overwrite`, + 'EEXIST', + ); + } + existingPackDirNeedsRm = true; + } + + const description = opts.description ?? 'Pack scaffolded by aqa pack new'; + const author = opts.author ?? 'You'; + const license = opts.license ?? 'Apache-2.0'; + const exampleUrl = exampleUrlFor(opts.sutType); + + // Build the manifest object and validate against the canonical schema + // before we touch the filesystem. Catches typos in the starter content + // before they end up on disk. + const manifest = { + schema_version: '1' as const, + name: opts.slug, + version: '0.1.0', + description, + author, + license, + applies_when: { + sut_type: [opts.sutType], + }, + templates: [], + scenarios: ['scenarios/starter.yaml'], + risks: ['risks/starter.yaml'], + oracles: [], + probes: [], + }; + const validated = PackManifest.PackManifest.safeParse(manifest); + if (!validated.success) { + return makeError( + `generated manifest failed schema validation (this is a bug in aqa pack new — please report): ${validated.error.message}`, + ); + } + + const scenarioObj = { + schema_version: '1' as const, + id: `scn-${opts.slug}-starter`, + title: `Starter scenario for ${opts.slug} — replace with a real test`, + risk_refs: [`r-${opts.slug}-starter`], + invariant_refs: [`inv-${opts.slug}-starter`], + preconditions: [], + steps: [ + { + id: 'probe-starter', + kind: 'http' as const, + with: { method: 'GET', url: exampleUrl }, + }, + ], + oracles: [ + { + id: 'o-starter-ok', + kind: 'http_status' as const, + // The no-network probe stub returns status=200, so this passes + // out of the box. When you wire a real probe runner the oracle + // will be checked against the actual server response. + with: { expected: 200 }, + }, + ], + tags: [opts.sutType, 'starter'], + }; + const scenarioValid = Scenario.Scenario.safeParse(scenarioObj); + if (!scenarioValid.success) { + return makeError( + `generated scenario failed schema validation (likely a too-long slug): ${scenarioValid.error.message}`, + ); + } + const scenarioYaml = yamlStringify(scenarioObj); + + const riskObj = { + schema_version: '1' as const, + project: opts.slug, + risks: [ + { + id: `r-${opts.slug}-starter`, + category: 'integrity' as const, + title: 'Starter risk — replace with a real one', + severity: 'medium' as const, + likelihood: 'possible' as const, + invariants: [ + { + id: `inv-${opts.slug}-starter`, + statement: + 'Replace this with the real invariant your scenarios prove. The current statement is a placeholder so the pack passes schema validation.', + }, + ], + }, + ], + }; + const riskValid = RiskMap.RiskMap.safeParse(riskObj); + if (!riskValid.success) { + return makeError( + `generated risk map failed schema validation (likely a too-long slug): ${riskValid.error.message}`, + ); + } + const riskYaml = yamlStringify(riskObj); + + const readmeMd = `# ${opts.slug} + +Scaffolded by \`aqa pack new\`. Replace this with a real description. + +## Files + +- \`pack.yaml\` — the manifest. Update \`name\`, \`description\`, and \`applies_when\` to match your project. +- \`scenarios/starter.yaml\` — example scenario. Edit the probe URL + oracle to match real behavior. +- \`risks/starter.yaml\` — risk declaration. Replace the placeholder \`r-${opts.slug}-starter\` with the real risk you're proving. +- \`package.json\` — only used if you publish to npm. The scaffold sets \`name: "${opts.slug}"\` (unscoped) so vendor/copy distribution works as-is. **Before \`npm publish\` you must change \`name\` to a scope you own** (e.g. \`"@your-scope/${opts.slug}"\`), or the publish will fail with "name already taken" / "you do not have permission". \`pack.yaml.name\` (the discovery key) stays as the unscoped slug — those two names are independent. + +## Run it + +Drop this pack under \`/packs/${opts.slug}/\` and reference it from \`.aqa/profiles.yaml\`. To distribute it across projects: publish under your own npm scope (\`@your-scope/${opts.slug}\`) and have consumers either (a) vendor/copy/extract the published tarball into their \`/packs/\` directory, or (b) install it normally via \`npm install\` and add an alias into the \`@aqa\` scope for auto-discovery (\`"@aqa/${opts.slug}": "npm:@your-scope/${opts.slug}"\` in their \`package.json\` — packs under \`/node_modules/@aqa/*\` are auto-discovered). The snippet below is the smallest schema-valid form — both top-level \`schema_version\` and per-profile fields (\`schema_version\`, \`execution_mode\`) are required by \`@aqa/schemas/ProfilesFile\`: + +\`\`\`yaml +schema_version: "1" +profiles: + smoke: + schema_version: "1" + name: smoke + execution_mode: orchestrator + packs: ["${opts.slug}"] + tags: ["${opts.sutType}", "starter"] +\`\`\` + +Then \`aqa run --profile smoke\` will pick it up. + +See the [pack authoring guide](https://github.com/padosoft/agentic-qa-kit/blob/main/docs/PACK-AUTHORING.md) for the full reference. +`; + + // All the content is built and validated. Wrap writes in try/catch so + // any FS failure (permission, file-at-path, partial-write) returns a + // structured error instead of throwing past the CLI's top handler. + // + // Atomic-ish `--force`: when overwriting, we rename the existing pack + // out of the way to a sibling backup path BEFORE writing the new one. + // If the write phase fails (permissions, disk full, partial write), we + // remove the half-written new pack and rename the backup back into + // place — so the user is left with their original pack intact rather + // than neither pack. On success, the backup is removed. The rename + // stays inside the same parent directory, so it's atomic on any sane + // filesystem (no cross-device move). + // + // Symlink safety inside `packDir`: we already rejected `packDir` itself + // as a symlink up top. The rename here moves the *entire* old tree out + // of the way as one atomic operation, so any symlinks living inside + // the old packDir (e.g. `/scenarios` as a symlink) follow the + // rename and never get touched. The fresh `mkdirSync(packDir)` below + // creates a brand-new empty directory, so `mkdirSync(packDir/scenarios)` + // and `writeFileSync(...)` cannot follow any leftover symlink — there + // are none to follow. Stale files from the previous pack are likewise + // impossible: the new packDir starts empty. + let backupDir: string | null = null; + if (existingPackDirNeedsRm) { + backupDir = `${packDir}.aqa-backup-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; + try { + renameSync(packDir, backupDir); + } catch (e) { + return makeError( + `cannot rename existing pack directory ${packDir} → ${backupDir} (needed to make --force non-destructive): ${e instanceof Error ? e.message : String(e)}`, + 'EIO', + ); + } + } + try { + mkdirSync(packDir, { recursive: true }); + mkdirSync(resolve(packDir, 'scenarios'), { recursive: true }); + mkdirSync(resolve(packDir, 'risks'), { recursive: true }); + writeFileSync(resolve(packDir, 'pack.yaml'), yamlStringify(manifest), 'utf8'); + writeFileSync(resolve(packDir, 'scenarios', 'starter.yaml'), scenarioYaml, 'utf8'); + writeFileSync(resolve(packDir, 'risks', 'starter.yaml'), riskYaml, 'utf8'); + writeFileSync(resolve(packDir, 'README.md'), readmeMd, 'utf8'); + writeFileSync( + resolve(packDir, 'package.json'), + JSON.stringify( + { + name: opts.slug, + version: '0.1.0', + description, + license, + author, + // `files` lists what to include in the published tarball. + // We only list directories the scaffold actually creates; if + // the author later adds custom `oracles/` or `probes/`, they + // should extend this array themselves. Listing non-existent + // directories here makes some tooling warn on `npm pack`. + files: ['pack.yaml', 'scenarios', 'risks', 'README.md'], + // No `private: true`. The pack must remain publishable from + // this scaffold: setting `private: true` would make npm refuse + // the publish outright. Note that an author will still need to + // change `name` to a scope they own (e.g. `@your-scope/`) + // before `npm publish` — the README explains this — but + // leaving `private: true` would block them at a less-obvious + // step. For vendor/copy distribution into `/packs/`, + // the package.json is irrelevant either way. + }, + null, + 2, + ), + 'utf8', + ); + } catch (e) { + return rollbackAndError( + packDir, + backupDir, + `cannot write pack files to ${packDir}: ${e instanceof Error ? e.message : String(e)}`, + ); + } + + // Re-validate by parsing what we wrote — catches any serializer divergence. + try { + const roundTrip = PackManifest.PackManifest.safeParse( + yamlParse(readFileSync(resolve(packDir, 'pack.yaml'), 'utf8')), + ); + if (!roundTrip.success) { + return rollbackAndError( + packDir, + backupDir, + `scaffolded pack.yaml failed round-trip validation: ${roundTrip.error.message}`, + ); + } + } catch (e) { + return rollbackAndError( + packDir, + backupDir, + `cannot re-read scaffolded pack.yaml at ${packDir}: ${e instanceof Error ? e.message : String(e)}`, + ); + } + + // Scaffold succeeded — drop the backup. A failure to clean up the + // backup is intentionally silent: the new pack is in place and fully + // usable, and surfacing a partial success would force every caller + // (CLI, tests, future programmatic users) to handle a `warnings` + // shape on the happy path. The worst-case outcome is a stale + // `.aqa-backup-*` sibling that a user can rm manually; the + // randomly-suffixed name makes it obviously not part of the pack. + if (backupDir !== null) { + try { + rmSync(backupDir, { recursive: true, force: true }); + } catch { + // best-effort; new pack is valid either way + } + } + + return { + ok: true, + packDir, + files: [ + 'pack.yaml', + 'scenarios/starter.yaml', + 'risks/starter.yaml', + 'README.md', + 'package.json', + ], + }; +} + +/** + * Restore the user's original pack (renamed to `backupDir`) and return a + * structured error. Called on every failure path after the rename has + * happened, so an interrupted scaffold leaves the working tree in the + * same state it was in before `aqa pack new --force` was invoked. + * + * If `backupDir` is null (no existing pack to begin with), we just clear + * any half-written content at `packDir` and surface the error. + */ +function rollbackAndError( + packDir: string, + backupDir: string | null, + message: string, +): PackNewResult { + try { + rmSync(packDir, { recursive: true, force: true }); + } catch { + // best-effort — surface the original error regardless + } + if (backupDir !== null) { + try { + renameSync(backupDir, packDir); + } catch (restoreErr) { + // We failed to restore. Don't lose the data silently — point the + // user at the backup path so they can recover manually. + return makeError( + `${message} (rollback FAILED — your original pack is at ${backupDir}, please restore it manually: ${restoreErr instanceof Error ? restoreErr.message : String(restoreErr)})`, + 'EIO', + ); + } + } + // Write/round-trip phase failures are all EIO (FS or serializer issues). + return makeError(message, 'EIO'); +} diff --git a/packages/pack-author/test/pack-author.test.ts b/packages/pack-author/test/pack-author.test.ts new file mode 100644 index 0000000..b789ad0 --- /dev/null +++ b/packages/pack-author/test/pack-author.test.ts @@ -0,0 +1,38 @@ +/** + * @aqa/pack-author smoke — the heavy behaviour coverage stays in + * `packages/kit/test/pack-new.test.ts`, which exercises `runPackNew` + * end-to-end through the CLI re-export. This file just guards the + * package contract: `runPackNew` is exposed as a NAMED export, accepts + * the `PackNewOptions` interface as documented, and returns the + * `PackNewResult` shape on both happy + error paths. + */ + +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { runPackNew } from '../dist/index.js'; + +describe('@aqa/pack-author — package boundary', () => { + it('runPackNew is callable and returns the documented shape', () => { + const root = mkdtempSync(join(tmpdir(), 'aqa-pack-author-')); + const result = runPackNew({ root, slug: 'pack-demo', sutType: 'api' }); + assert.equal(result.ok, true); + assert.ok(result.packDir, 'packDir must be set on success'); + assert.ok( + result.files && result.files.length >= 4, + 'files must list at least the 4 scaffold artifacts', + ); + const manifest = readFileSync(join(result.packDir as string, 'pack.yaml'), 'utf8'); + assert.match(manifest, /name: pack-demo/); + }); + + it('returns a structured error with `code` on slug validation failure', () => { + const root = mkdtempSync(join(tmpdir(), 'aqa-pack-author-')); + const result = runPackNew({ root, slug: 'NOT_A_SLUG', sutType: 'api' }); + assert.equal(result.ok, false); + assert.equal(result.code, 'EINVAL'); + assert.match(result.error ?? '', /must be lowercase alphanumeric/); + }); +}); diff --git a/packages/pack-author/tsconfig.json b/packages/pack-author/tsconfig.json new file mode 100644 index 0000000..20726d4 --- /dev/null +++ b/packages/pack-author/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "incremental": true, + "tsBuildInfoFile": ".tsbuildinfo" + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules", "**/*.test.ts"] +} diff --git a/packages/server/package.json b/packages/server/package.json index 90af60a..a19e2d8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -17,7 +17,7 @@ }, "dependencies": { "@aqa/auth": "workspace:*", - "@aqa/kit": "workspace:*", + "@aqa/pack-author": "workspace:*", "@aqa/schemas": "workspace:*", "@aqa/store": "workspace:*", "yaml": "^2.9.0" diff --git a/packages/server/src/api.ts b/packages/server/src/api.ts index f96c18e..97996d8 100644 --- a/packages/server/src/api.ts +++ b/packages/server/src/api.ts @@ -1,7 +1,7 @@ import { Permission, rolePermissions } from '@aqa/auth'; import type { Permission as PermissionType, Role, User, allows } from '@aqa/auth'; -import { runPackNew } from '@aqa/kit'; -import type { PackNewErrorCode } from '@aqa/kit'; +import { runPackNew } from '@aqa/pack-author'; +import type { PackNewErrorCode } from '@aqa/pack-author'; import { PackManifest as PackManifestSchema, Profile as ProfileSchema, @@ -330,9 +330,10 @@ export function makeApi(): ApiHandler[] { }, { // v1.7 slice 3 — scaffold a new pack on disk for the Admin - // "Create pack" wizard. Delegates to `runPackNew` from `@aqa/kit` - // (the same code path as `aqa pack new` on the CLI) so the two - // UIs stay in lockstep on validation, atomic --force, etc. + // "Create pack" wizard. Delegates to `runPackNew` from + // `@aqa/pack-author` (the same code path the kit's `aqa pack new` + // CLI calls through its re-export shim) so the two UIs stay in + // lockstep on validation, atomic --force, etc. // // Synchronous FS work in an async handler: `runPackNew` is sync // (mkdir / writeFile / rename of ~5 small files plus a few schema diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 426e662..f46ab06 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1,2 +1,9 @@ -export { makeApi, type ApiContext } from './api.js'; +export { + makeApi, + type ApiContext, + type ApiHandler, + type ApiMethod, + type ApiRequest, + type ApiResponse, +} from './api.js'; export { RunnerQueue, type EnqueuedJob, type RunnerJob } from './runner-queue.js';