diff --git a/.gitattributes b/.gitattributes new file mode 100755 index 000000000..c0b85c760 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# 统一行尾符为 LF，避免 Windows/WSL 跨平台换行符问题 +* text=auto eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 9a9f924ca..18d808c28 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,7 @@ pnpm-debug.log* # Brainstorm assets (not tracked in source control) .superpowers/ .claude/ +.playwright-mcp/ + +# Worktrees +.worktrees/ diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/README_CN.md b/README_CN.md old mode 100644 new mode 100755 diff --git a/README_JA.md b/README_JA.md old mode 100644 new mode 100755 diff --git a/README_KO.md b/README_KO.md old mode 100644 new mode 100755 diff --git a/assets/1-deepresearch.jpg b/assets/1-deepresearch.jpg old mode 100644 new mode 100755 diff --git a/assets/2-ai_chat.jpg b/assets/2-ai_chat.jpg old mode 100644 new mode 100755 diff --git a/assets/3-knowledge_graph.jpg b/assets/3-knowledge_graph.jpg old mode 100644 new mode 100755 diff --git a/assets/4-chrome_extension_webclipper.jpg b/assets/4-chrome_extension_webclipper.jpg old mode 100644 new mode 100755 diff --git a/assets/5-obsidian_compatibility.jpg b/assets/5-obsidian_compatibility.jpg old mode 100644 new mode 100755 diff --git a/assets/kg_community.jpg b/assets/kg_community.jpg old mode 100644 new mode 100755 diff --git a/assets/kg_insights.jpg b/assets/kg_insights.jpg old mode 100644 new mode 100755 diff --git a/assets/llm_wiki_arch.jpg b/assets/llm_wiki_arch.jpg old mode 100644 new mode 100755 diff --git a/assets/overview.jpg b/assets/overview.jpg old mode 100644 new mode 100755 diff --git a/components.json b/components.json old mode 100644 new mode 100755 diff --git a/extension/Readability.js b/extension/Readability.js old mode 100644 new mode 100755 diff --git a/extension/Turndown.js b/extension/Turndown.js old mode 100644 new mode 100755 diff --git a/extension/icon128.png b/extension/icon128.png old mode 100644 new mode 100755 diff --git a/extension/icon16.png b/extension/icon16.png old mode 100644 new mode 100755 diff --git a/extension/icon48.png b/extension/icon48.png old mode 100644 new mode 100755 diff --git a/extension/manifest.json b/extension/manifest.json old mode 100644 new mode 100755 diff --git a/extension/popup.html b/extension/popup.html old mode 100644 new mode 100755 diff --git a/extension/popup.js b/extension/popup.js old mode 100644 new mode 100755 diff --git a/index.html b/index.html old mode 100644 new mode 100755 diff --git a/llm-wiki.md b/llm-wiki.md old mode 100644 new mode 100755 diff --git a/logo.jpg b/logo.jpg old mode 100644 new mode 100755 diff --git a/mcp-server/README.md b/mcp-server/README.md old mode 100644 new mode 100755 diff --git a/mcp-server/package-lock.json b/mcp-server/package-lock.json old mode 100644 new mode 100755 diff --git a/mcp-server/package.json b/mcp-server/package.json old mode 100644 new mode 100755 diff --git a/mcp-server/src/api-client.ts b/mcp-server/src/api-client.ts old mode 100644 new mode 100755 diff --git a/mcp-server/src/index.ts b/mcp-server/src/index.ts old mode 100644 new mode 100755 diff --git a/mcp-server/src/version.ts b/mcp-server/src/version.ts old mode 100644 new mode 100755 diff --git a/mcp-server/test/api-client.test.ts b/mcp-server/test/api-client.test.ts old mode 100644 new mode 100755 diff --git a/mcp-server/test/version.test.ts b/mcp-server/test/version.test.ts old mode 100644 new mode 100755 diff --git a/mcp-server/tsconfig.json b/mcp-server/tsconfig.json old mode 100644 new mode 100755 diff --git a/package-lock.json b/package-lock.json old mode 100644 new mode 100755 diff --git a/package.json b/package.json old mode 100644 new mode 100755 diff --git a/plans/multimodal-images.md b/plans/multimodal-images.md old mode 100644 new mode 100755 diff --git a/scripts/debug_ollama_tokens.py b/scripts/debug_ollama_tokens.py old mode 100644 new mode 100755 diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock old mode 100644 new mode 100755 diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml old mode 100644 new mode 100755 diff --git a/src-tauri/build.rs b/src-tauri/build.rs old mode 100644 new mode 100755 diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json old mode 100644 new mode 100755 diff --git a/src-tauri/icons/128x128.png b/src-tauri/icons/128x128.png old mode 100644 new mode 100755 diff --git a/src-tauri/icons/128x128@2x.png b/src-tauri/icons/128x128@2x.png old mode 100644 new mode 100755 diff --git a/src-tauri/icons/32x32.png b/src-tauri/icons/32x32.png old mode 100644 new mode 100755 diff --git a/src-tauri/icons/icon.icns b/src-tauri/icons/icon.icns old mode 100644 new mode 100755 diff --git a/src-tauri/icons/icon.ico b/src-tauri/icons/icon.ico old mode 100644 new mode 100755 diff --git a/src-tauri/icons/icon.png b/src-tauri/icons/icon.png old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/SHA256SUMS b/src-tauri/pdfium/SHA256SUMS old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium-arm64.so b/src-tauri/pdfium/libpdfium-arm64.so old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium-x86_64.dylib b/src-tauri/pdfium/libpdfium-x86_64.dylib old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium.dylib b/src-tauri/pdfium/libpdfium.dylib old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium.so b/src-tauri/pdfium/libpdfium.so old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/pdfium.dll b/src-tauri/pdfium/pdfium.dll old mode 100644 new mode 100755 diff --git a/src-tauri/src/api_server.rs b/src-tauri/src/api_server.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/clip_server.rs b/src-tauri/src/clip_server.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/claude_cli.rs b/src-tauri/src/commands/claude_cli.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/cli_resolver.rs b/src-tauri/src/commands/cli_resolver.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/codex_cli.rs b/src-tauri/src/commands/codex_cli.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/extract_images.rs b/src-tauri/src/commands/extract_images.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/file_sync.rs b/src-tauri/src/commands/file_sync.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/fs.rs b/src-tauri/src/commands/fs.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/project.rs b/src-tauri/src/commands/project.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/search.rs b/src-tauri/src/commands/search.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/vectorstore.rs b/src-tauri/src/commands/vectorstore.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/panic_guard.rs b/src-tauri/src/panic_guard.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/proxy.rs b/src-tauri/src/proxy.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/tray.rs b/src-tauri/src/tray.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/types/mod.rs b/src-tauri/src/types/mod.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/types/wiki.rs b/src-tauri/src/types/wiki.rs old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.linux.conf.json b/src-tauri/tauri.linux.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.macos.conf.json b/src-tauri/tauri.macos.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.windows.conf.json b/src-tauri/tauri.windows.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/windows-app-manifest.xml b/src-tauri/windows-app-manifest.xml old mode 100644 new mode 100755 diff --git a/src/App.tsx b/src/App.tsx old mode 100644 new mode 100755 diff --git a/src/assets/logo.jpg b/src/assets/logo.jpg old mode 100644 new mode 100755 diff --git a/src/commands/file-sync.ts b/src/commands/file-sync.ts old mode 100644 new mode 100755 diff --git a/src/commands/fs.test.ts b/src/commands/fs.test.ts old mode 100644 new mode 100755 diff --git a/src/commands/fs.ts b/src/commands/fs.ts old mode 100644 new mode 100755 diff --git a/src/components/chat/chat-input.tsx b/src/components/chat/chat-input.tsx old mode 100644 new mode 100755 diff --git a/src/components/chat/chat-message.tsx b/src/components/chat/chat-message.tsx old mode 100644 new mode 100755 diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/file-preview.tsx b/src/components/editor/file-preview.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/frontmatter-panel.tsx b/src/components/editor/frontmatter-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/wiki-editor.tsx b/src/components/editor/wiki-editor.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/wiki-reader.tsx b/src/components/editor/wiki-reader.tsx old mode 100644 new mode 100755 diff --git a/src/components/error-boundary.tsx b/src/components/error-boundary.tsx old mode 100644 new mode 100755 diff --git a/src/components/graph/graph-layout-worker.ts b/src/components/graph/graph-layout-worker.ts old mode 100644 new mode 100755 diff --git a/src/components/graph/graph-view.tsx b/src/components/graph/graph-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/activity-panel.tsx b/src/components/layout/activity-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/app-layout-visibility.test.ts b/src/components/layout/app-layout-visibility.test.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/app-layout-visibility.ts b/src/components/layout/app-layout-visibility.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/app-layout.tsx b/src/components/layout/app-layout.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/content-area.tsx b/src/components/layout/content-area.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/file-tree.tsx b/src/components/layout/file-tree.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/icon-sidebar.tsx b/src/components/layout/icon-sidebar.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/knowledge-tree.tsx b/src/components/layout/knowledge-tree.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/preview-panel.tsx b/src/components/layout/preview-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/research-panel-nav.test.ts b/src/components/layout/research-panel-nav.test.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/research-panel-nav.ts b/src/components/layout/research-panel-nav.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/research-panel.tsx b/src/components/layout/research-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/sidebar-panel.tsx b/src/components/layout/sidebar-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/update-banner.tsx b/src/components/layout/update-banner.tsx old mode 100644 new mode 100755 diff --git a/src/components/lint/lint-view.test.ts b/src/components/lint/lint-view.test.ts old mode 100644 new mode 100755 diff --git a/src/components/lint/lint-view.tsx b/src/components/lint/lint-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/mermaid-diagram.test.tsx b/src/components/mermaid-diagram.test.tsx old mode 100644 new mode 100755 diff --git a/src/components/mermaid-diagram.tsx b/src/components/mermaid-diagram.tsx old mode 100644 new mode 100755 diff --git a/src/components/project/create-project-dialog.test.ts b/src/components/project/create-project-dialog.test.ts old mode 100644 new mode 100755 diff --git a/src/components/project/create-project-dialog.tsx b/src/components/project/create-project-dialog.tsx old mode 100644 new mode 100755 diff --git a/src/components/project/template-picker.tsx b/src/components/project/template-picker.tsx old mode 100644 new mode 100755 diff --git a/src/components/project/welcome-screen.tsx b/src/components/project/welcome-screen.tsx old mode 100644 new mode 100755 diff --git a/src/components/review/review-view.tsx b/src/components/review/review-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/search/search-view.tsx b/src/components/search/search-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/context-size-selector.tsx b/src/components/settings/context-size-selector.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/llm-presets.ts b/src/components/settings/llm-presets.ts old mode 100644 new mode 100755 index 879eaa116..34c5e5871 --- a/src/components/settings/llm-presets.ts +++ b/src/components/settings/llm-presets.ts @@ -182,7 +182,7 @@ export const LLM_PRESETS: LlmPreset[] = [ "deepseek-chat", "deepseek-reasoner", ], - suggestedContextSize: 64000, + suggestedContextSize: 1000000, }, { id: "atlascloud", diff --git a/src/components/settings/preset-resolver.test.ts b/src/components/settings/preset-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/preset-resolver.ts b/src/components/settings/preset-resolver.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/about-section.tsx b/src/components/settings/sections/about-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/api-server-section.test.ts b/src/components/settings/sections/api-server-section.test.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/api-server-section.tsx b/src/components/settings/sections/api-server-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/changelog-section.tsx b/src/components/settings/sections/changelog-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/embedding-section.tsx b/src/components/settings/sections/embedding-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/general-section.tsx b/src/components/settings/sections/general-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/interface-section.tsx b/src/components/settings/sections/interface-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/llm-provider-section.tsx b/src/components/settings/sections/llm-provider-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/maintenance-section.tsx b/src/components/settings/sections/maintenance-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/mineru-section.tsx b/src/components/settings/sections/mineru-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/multimodal-section.tsx b/src/components/settings/sections/multimodal-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/network-section.tsx b/src/components/settings/sections/network-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/output-section.tsx b/src/components/settings/sections/output-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/scheduled-import-section.tsx b/src/components/settings/sections/scheduled-import-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/source-watch-section.tsx b/src/components/settings/sections/source-watch-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/web-search-section.tsx b/src/components/settings/sections/web-search-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/settings-types.ts b/src/components/settings/settings-types.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/sources/sources-view.tsx b/src/components/sources/sources-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/button.tsx b/src/components/ui/button.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/dialog.tsx b/src/components/ui/dialog.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/input.tsx b/src/components/ui/input.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/label.tsx b/src/components/ui/label.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/resizable.tsx b/src/components/ui/resizable.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/scroll-area.tsx b/src/components/ui/scroll-area.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/separator.tsx b/src/components/ui/separator.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/tooltip.tsx b/src/components/ui/tooltip.tsx old mode 100644 new mode 100755 diff --git a/src/i18n/en.json b/src/i18n/en.json old mode 100644 new mode 100755 diff --git a/src/i18n/i18n-parity.test.ts b/src/i18n/i18n-parity.test.ts old mode 100644 new mode 100755 diff --git a/src/i18n/index.ts b/src/i18n/index.ts old mode 100644 new mode 100755 diff --git a/src/i18n/zh.json b/src/i18n/zh.json old mode 100644 new mode 100755 diff --git a/src/index.css b/src/index.css old mode 100644 new mode 100755 diff --git a/src/lib/__tests__/claude-cli-transport.test.ts b/src/lib/__tests__/claude-cli-transport.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/__tests__/dedup_embedding.test.ts b/src/lib/__tests__/dedup_embedding.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/__tests__/llm-providers.test.ts b/src/lib/__tests__/llm-providers.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/anytxt-search.test.ts b/src/lib/anytxt-search.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/anytxt-search.ts b/src/lib/anytxt-search.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-server-constants.ts b/src/lib/api-server-constants.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-server.real-llm.test.ts b/src/lib/api-server.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-token.test.ts b/src/lib/api-token.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-token.ts b/src/lib/api-token.ts old mode 100644 new mode 100755 diff --git a/src/lib/auto-save.test.ts b/src/lib/auto-save.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/auto-save.ts b/src/lib/auto-save.ts old mode 100644 new mode 100755 diff --git a/src/lib/azure-openai.test.ts b/src/lib/azure-openai.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/azure-openai.ts b/src/lib/azure-openai.ts old mode 100644 new mode 100755 diff --git a/src/lib/changelog.ts b/src/lib/changelog.ts old mode 100644 new mode 100755 diff --git a/src/lib/chat-image-utils.ts b/src/lib/chat-image-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/chat-save-to-wiki.test.ts b/src/lib/chat-save-to-wiki.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/chat-save-to-wiki.ts b/src/lib/chat-save-to-wiki.ts old mode 100644 new mode 100755 diff --git a/src/lib/claude-cli-transport.ts b/src/lib/claude-cli-transport.ts old mode 100644 new mode 100755 diff --git a/src/lib/clip-watcher.ts b/src/lib/clip-watcher.ts old mode 100644 new mode 100755 diff --git a/src/lib/codex-cli-transport.test.ts b/src/lib/codex-cli-transport.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/codex-cli-transport.ts b/src/lib/codex-cli-transport.ts old mode 100644 new mode 100755 diff --git a/src/lib/connection-tests.test.ts b/src/lib/connection-tests.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/connection-tests.ts b/src/lib/connection-tests.ts old mode 100644 new mode 100755 diff --git a/src/lib/context-budget.test.ts b/src/lib/context-budget.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/context-budget.ts b/src/lib/context-budget.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-queue.test.ts b/src/lib/dedup-queue.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-queue.ts b/src/lib/dedup-queue.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-runner.test.ts b/src/lib/dedup-runner.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-runner.ts b/src/lib/dedup-runner.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-storage.ts b/src/lib/dedup-storage.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup.test.ts b/src/lib/dedup.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup.ts b/src/lib/dedup.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup_embedding.ts b/src/lib/dedup_embedding.ts old mode 100644 new mode 100755 diff --git a/src/lib/deep-research.test.ts b/src/lib/deep-research.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/deep-research.ts b/src/lib/deep-research.ts old mode 100644 new mode 100755 diff --git a/src/lib/detect-language.property.test.ts b/src/lib/detect-language.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/detect-language.test.ts b/src/lib/detect-language.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/detect-language.ts b/src/lib/detect-language.ts old mode 100644 new mode 100755 diff --git a/src/lib/embedding.real-llm.test.ts b/src/lib/embedding.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/embedding.ts b/src/lib/embedding.ts old mode 100644 new mode 100755 diff --git a/src/lib/endpoint-normalizer.test.ts b/src/lib/endpoint-normalizer.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/endpoint-normalizer.ts b/src/lib/endpoint-normalizer.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.real-llm.test.ts b/src/lib/enrich-wikilinks.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.scenarios.test.ts b/src/lib/enrich-wikilinks.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.test.ts b/src/lib/enrich-wikilinks.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.ts b/src/lib/enrich-wikilinks.ts old mode 100644 new mode 100755 diff --git a/src/lib/extract-source-images.test.ts b/src/lib/extract-source-images.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/extract-source-images.ts b/src/lib/extract-source-images.ts old mode 100644 new mode 100755 diff --git a/src/lib/file-types.test.ts b/src/lib/file-types.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/file-types.ts b/src/lib/file-types.ts old mode 100644 new mode 100755 diff --git a/src/lib/frontmatter.test.ts b/src/lib/frontmatter.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/frontmatter.ts b/src/lib/frontmatter.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-filters.test.ts b/src/lib/graph-filters.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-filters.ts b/src/lib/graph-filters.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-insights.ts b/src/lib/graph-insights.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-relevance.ts b/src/lib/graph-relevance.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-search.test.ts b/src/lib/graph-search.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-search.ts b/src/lib/graph-search.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-visibility.test.ts b/src/lib/graph-visibility.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-visibility.ts b/src/lib/graph-visibility.ts old mode 100644 new mode 100755 diff --git a/src/lib/greeting-detector.test.ts b/src/lib/greeting-detector.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/greeting-detector.ts b/src/lib/greeting-detector.ts old mode 100644 new mode 100755 diff --git a/src/lib/has-usable-llm.test.ts b/src/lib/has-usable-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/has-usable-llm.ts b/src/lib/has-usable-llm.ts old mode 100644 new mode 100755 diff --git a/src/lib/image-caption-pipeline.test.ts b/src/lib/image-caption-pipeline.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/image-caption-pipeline.ts b/src/lib/image-caption-pipeline.ts old mode 100644 new mode 100755 diff --git a/src/lib/index-chunker.test.ts b/src/lib/index-chunker.test.ts new file mode 100755 index 000000000..2160b625f --- /dev/null +++ b/src/lib/index-chunker.test.ts @@ -0,0 +1,430 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { + chunkIndexByEntries, + parsePrematchOutput, + assembleReducedIndex, + buildPrematchPrompt, + parseIndexBlocks, + appendIndexEntries, + runPrematchParallel, + type ParsedIndexBlock, +} from "./index-chunker" +import { streamChat } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" + +vi.mock("@/lib/llm-client", () => ({ + streamChat: vi.fn(), +})) + +const mockStreamChat = vi.mocked(streamChat) + +describe("chunkIndexByEntries", () => { + it("returns empty array for empty index", () => { + expect(chunkIndexByEntries("", 50)).toEqual([]) + }) + + it("returns single chunk when entries <= chunkSize", () => { + const index = [ + "# Index", + "", + "## Concepts", + "- [[attention]] — Core mechanism", + "- [[transformer]] — Architecture", + ].join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(1) + expect(chunks[0]).toContain("[[attention]]") + expect(chunks[0]).toContain("[[transformer]]") + expect(chunks[0]).toContain("## Concepts") + }) + + it("splits into multiple chunks at exact boundary", () => { + const lines: string[] = ["# Index", "", "## Concepts"] + for (let i = 0; i < 100; i++) { + lines.push(`- [[page-${i}]] — desc ${i}`) + } + const index = lines.join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(2) + expect(chunks[0].match(/\[\d+\]/g)).toHaveLength(50) + expect(chunks[1].match(/\[\d+\]/g)).toHaveLength(50) + }) + + it("preserves category headers within chunks that span categories", () => { + const lines: string[] = ["# Index", "", "## Concepts"] + for (let i = 0; i < 40; i++) { + lines.push(`- [[concept-${i}]] — desc ${i}`) + } + lines.push("## Entities") + for (let i = 0; i < 20; i++) { + lines.push(`- [[entity-${i}]] — desc ${i}`) + } + const index = lines.join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(2) + expect(chunks[0]).toContain("## Concepts") + expect(chunks[0]).toContain("## Entities") + expect(chunks[1]).toContain("## Entities") + expect(chunks[1]).not.toContain("## Concepts") + }) + + it("numbers entries sequentially across chunks starting from 1", () => { + const lines: string[] = ["# Index", "", "## Concepts"] + for (let i = 0; i < 120; i++) { + lines.push(`- [[page-${i}]] — desc ${i}`) + } + const index = lines.join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(3) + expect(chunks[0]).toContain("[1]") + expect(chunks[0]).toContain("[50]") + expect(chunks[1]).toContain("[51]") + expect(chunks[2]).toContain("[101]") + expect(chunks[2]).toContain("[120]") + }) + + it("skips non-entry lines (empty lines, H1) when counting", () => { + const index = [ + "# Index", + "", + "## Concepts", + "- [[a]] — desc a", + "", + "- [[b]] — desc b", + "", + "## Papers", + "(none yet)", + "- [[c]] — desc c", + ].join("\n") + const chunks = chunkIndexByEntries(index, 2) + expect(chunks).toHaveLength(2) + }) +}) + +describe("parsePrematchOutput", () => { + it("parses bracket format with spaces", () => { + expect(parsePrematchOutput("[3, 12, 47]")).toEqual([3, 12, 47]) + }) + + it("parses bracket format without spaces", () => { + expect(parsePrematchOutput("[3,12,47]")).toEqual([3, 12, 47]) + }) + + it("parses comma-separated without brackets", () => { + expect(parsePrematchOutput("3, 12, 47")).toEqual([3, 12, 47]) + }) + + it("parses single number", () => { + expect(parsePrematchOutput("[5]")).toEqual([5]) + expect(parsePrematchOutput("5")).toEqual([5]) + }) + + it("returns empty for none variations", () => { + expect(parsePrematchOutput("none")).toEqual([]) + expect(parsePrematchOutput("None")).toEqual([]) + expect(parsePrematchOutput("NONE")).toEqual([]) + expect(parsePrematchOutput("无")).toEqual([]) + }) + + it("returns empty for empty or whitespace", () => { + expect(parsePrematchOutput("")).toEqual([]) + expect(parsePrematchOutput(" ")).toEqual([]) + }) + + it("tolerates surrounding explanation text", () => { + expect(parsePrematchOutput("Matching items: [3, 12, 47]")).toEqual([3, 12, 47]) + }) + + it("ignores non-numeric tokens", () => { + expect(parsePrematchOutput("[3, abc, 47]")).toEqual([3, 47]) + }) + + it("returns empty on complete parse failure", () => { + expect(parsePrematchOutput("I think none match")).toEqual([]) + }) + + it("deduplicates numbers", () => { + expect(parsePrematchOutput("[3, 3, 12]")).toEqual([3, 12]) + }) +}) + +describe("assembleReducedIndex", () => { + it("returns empty string for empty matches", () => { + const index = "## Concepts\n- [[a]] — desc a\n- [[b]] — desc b" + expect(assembleReducedIndex(index, [])).toBe("") + }) + + it("returns single matched entry with its category header", () => { + const index = [ + "# Index", "", + "## Concepts", + "- [[attention]] — Core mechanism", + "- [[transformer]] — Architecture", + ].join("\n") + const result = assembleReducedIndex(index, [1]) + expect(result).toContain("## Concepts") + expect(result).toContain("[[attention]]") + expect(result).not.toContain("[[transformer]]") + }) + + it("returns multiple matches preserving original order", () => { + const index = [ + "## Concepts", + "- [[a]] — desc a", + "- [[b]] — desc b", + "- [[c]] — desc c", + ].join("\n") + const result = assembleReducedIndex(index, [3, 1]) + expect(result.indexOf("[[a]]")).toBeLessThan(result.indexOf("[[c]]")) + }) + + it("deduplicates category headers", () => { + const index = [ + "## Concepts", + "- [[a]] — desc a", + "- [[b]] — desc b", + "## Entities", + "- [[c]] — desc c", + ].join("\n") + const result = assembleReducedIndex(index, [1, 2]) + const headerCount = (result.match(/## Concepts/g) ?? []).length + expect(headerCount).toBe(1) + }) + + it("handles matches from different categories", () => { + const index = [ + "## Concepts", + "- [[a]] — desc a", + "## Entities", + "- [[b]] — desc b", + ].join("\n") + const result = assembleReducedIndex(index, [1, 2]) + expect(result).toContain("## Concepts") + expect(result).toContain("## Entities") + expect(result).toContain("[[a]]") + expect(result).toContain("[[b]]") + }) + + it("ignores out-of-range numbers gracefully", () => { + const index = "## Concepts\n- [[a]] — desc a" + const result = assembleReducedIndex(index, [1, 999]) + expect(result).toContain("[[a]]") + }) +}) + +describe("buildPrematchPrompt", () => { + it("includes source document content", () => { + const prompt = buildPrematchPrompt("This is a source about transformers.", "## Concepts\n[1] attention — desc") + expect(prompt).toContain("This is a source about transformers.") + }) + + it("includes the index chunk", () => { + const chunk = "## Concepts\n[1] attention — desc\n[2] transformer — desc" + const prompt = buildPrematchPrompt("source content", chunk) + expect(prompt).toContain("[1] attention") + expect(prompt).toContain("[2] transformer") + }) + + it("includes strict output format instructions", () => { + const prompt = buildPrematchPrompt("src", "chunk") + expect(prompt).toContain("[3, 12, 47]") + expect(prompt).toContain("none") + expect(prompt).toContain("STRICT") + }) +}) + +describe("parseIndexBlocks", () => { + it("parses a single INDEX block", () => { + const text = [ + "---INDEX: Concepts---", + "rope — Rotary Position Embedding", + "flash-attention — IO-aware attention", + "---END INDEX---", + ].join("\n") + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].category).toBe("Concepts") + expect(blocks[0].entries).toEqual([ + "rope — Rotary Position Embedding", + "flash-attention — IO-aware attention", + ]) + }) + + it("parses multiple INDEX blocks", () => { + const text = [ + "---INDEX: Concepts---", + "rope — desc", + "---END INDEX---", + "", + "---INDEX: Entities---", + "openai — desc", + "---END INDEX---", + ].join("\n") + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(2) + expect(blocks[0].category).toBe("Concepts") + expect(blocks[1].category).toBe("Entities") + }) + + it("returns empty array when no INDEX blocks", () => { + expect(parseIndexBlocks("no blocks here")).toEqual([]) + expect(parseIndexBlocks("")).toEqual([]) + }) + + it("tolerates extra whitespace in markers", () => { + const text = "---INDEX: Concepts ---\nrope — desc\n---END INDEX---" + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].category).toBe("Concepts") + }) + + it("handles empty entries", () => { + const text = "---INDEX: Concepts---\n---END INDEX---" + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].entries).toEqual([]) + }) + + it("normalizes CRLF line endings", () => { + const text = "---INDEX: Concepts---\r\nrope — desc\r\n---END INDEX---" + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].entries).toEqual(["rope — desc"]) + }) +}) + +describe("appendIndexEntries", () => { + it("appends entries to existing category", () => { + const index = [ + "# Index", "", + "## Concepts", + "- [[attention]] — Core mechanism", + "- [[transformer]] — Architecture", + ].join("\n") + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["rope — Rotary Position Embedding"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[attention]]") + expect(result).toContain("- [[transformer]]") + expect(result).toContain("- [[rope]] — Rotary Position Embedding") + expect(result.indexOf("[[transformer]]")).toBeLessThan(result.indexOf("rope")) + }) + + it("creates new category if it does not exist", () => { + const index = "## Concepts\n- [[a]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Entities", entries: ["openai — AI company"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("## Entities") + expect(result).toContain("- [[openai]] — AI company") + expect(result).toContain("## Concepts") + }) + + it("appends to multiple categories", () => { + const index = "## Concepts\n- [[a]] — desc\n## Entities\n- [[b]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["c — desc c"] }, + { category: "Entities", entries: ["d — desc d"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[c]] — desc c") + expect(result).toContain("- [[d]] — desc d") + }) + + it("handles entries that need slug format conversion", () => { + const index = "## Concepts\n- [[a]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["rope — desc"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[rope]] — desc") + }) + + it("preserves entries with no description", () => { + const index = "## Concepts\n- [[a]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["rope"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[rope]]") + }) + + it("handles empty blocks array", () => { + const index = "## Concepts\n- [[a]] — desc" + expect(appendIndexEntries(index, [])).toBe(index) + }) +}) + +describe("runPrematchParallel", () => { + beforeEach(() => { + mockStreamChat.mockReset() + }) + + it("returns matched numbers from all chunks", async () => { + const chunks = ["## Concepts\n[1] attention — desc", "## Entities\n[51] openai — desc"] + // The prompt includes the chunk content, so we can differentiate by content + mockStreamChat.mockImplementation(async (_cfg, messages, callbacks) => { + const sys = String(messages[0]?.content ?? "") + if (sys.includes("openai")) { + callbacks?.onToken?.("[51]") + } else { + callbacks?.onToken?.("[1, 3]") + } + callbacks?.onDone?.() + }) + + const result = await runPrematchParallel( + chunks, + "source content", + {} as LlmConfig, + undefined, + ) + expect(result).toContain(1) + expect(result).toContain(3) + expect(result).toContain(51) + }) + + it("skips failed chunks and continues", async () => { + const chunks = ["chunk0-content", "chunk1-content"] + let callCount = 0 + mockStreamChat.mockImplementation(async (_cfg, _messages, callbacks) => { + callCount++ + if (callCount === 1) { + callbacks?.onError?.(new Error("network error")) + return + } + callbacks?.onToken?.("[42]") + callbacks?.onDone?.() + }) + + const result = await runPrematchParallel( + chunks, + "source content", + {} as LlmConfig, + undefined, + ) + expect(result).toEqual([42]) + }) + + it("returns empty when all chunks fail", async () => { + mockStreamChat.mockImplementation(async (_cfg, _messages, callbacks) => { + callbacks?.onError?.(new Error("total failure")) + }) + + const result = await runPrematchParallel( + ["c0", "c1"], + "source", + {} as LlmConfig, + undefined, + ) + expect(result).toEqual([]) + }) + + it("returns empty for empty chunks array", async () => { + const result = await runPrematchParallel([], "source", {} as LlmConfig, undefined) + expect(result).toEqual([]) + expect(mockStreamChat).not.toHaveBeenCalled() + }) +}) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts new file mode 100755 index 000000000..90a301306 --- /dev/null +++ b/src/lib/index-chunker.ts @@ -0,0 +1,323 @@ +/** + * Chunk an index.md string into fixed-size chunks by entry count. + * Each chunk preserves the ## Category headers for context. + * Entries are numbered sequentially with [N] prefix across all chunks. + */ + +import { streamChat, isReasoningOnlyResponseError } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" + +interface IndexEntry { + category: string + text: string +} + +/** Parse index.md into structured entries with their category context. */ +function parseIndexEntries(index: string): { + entries: IndexEntry[] +} { + const lines = index.split("\n") + const entries: IndexEntry[] = [] + let currentCategory = "" + + for (const line of lines) { + const headerMatch = line.match(/^##\s+(.+)/) + if (headerMatch) { + currentCategory = headerMatch[1].trim() + continue + } + // Entry lines start with - [[ or * [[ + if (/^[-*]\s+\[\[/.test(line)) { + entries.push({ + category: currentCategory, + text: line, + }) + } + } + + return { entries } +} + +export function chunkIndexByEntries(index: string, chunkSize: number): string[] { + if (!index.trim()) return [] + + const { entries } = parseIndexEntries(index) + if (entries.length === 0) return [] + + const chunks: string[] = [] + let entryIdx = 0 + let globalNum = 1 + + while (entryIdx < entries.length) { + const chunkLines: string[] = [] + let currentCat = "" + const end = Math.min(entryIdx + chunkSize, entries.length) + + for (let i = entryIdx; i < end; i++) { + const entry = entries[i] + if (entry.category !== currentCat) { + currentCat = entry.category + chunkLines.push(`## ${currentCat}`) + } + chunkLines.push(`[${globalNum}] ${entry.text.replace(/^[-*]\s+/, "")}`) + globalNum++ + } + + chunks.push(chunkLines.join("\n")) + entryIdx = end + } + + return chunks +} + +/** + * Parse the pre-match LLM output into an array of matching entry numbers. + * Tolerant: handles [3, 12, 47], 3, 12, 47, none, 无, surrounding text. + * Returns deduplicated numbers, or empty array if no valid numbers found. + */ +export function parsePrematchOutput(output: string): number[] { + const trimmed = output.trim() + if (!trimmed) return [] + + const lower = trimmed.toLowerCase() + if (lower === "none" || trimmed === "无") return [] + + // Try to extract bracketed numbers first: [3, 12, 47] + const bracketMatch = trimmed.match(/\[([\d,\s]+)\]/) + const source = bracketMatch ? bracketMatch[1] : trimmed + + // Extract all integer tokens + const tokens = source.match(/\d+/g) + if (!tokens) return [] + + const numbers = tokens + .map((t) => parseInt(t, 10)) + .filter((n) => Number.isFinite(n) && n > 0) + + return [...new Set(numbers)] +} + +/** + * Assemble a reduced index from the original index.md and matched entry numbers. + * Entry numbers are 1-based as assigned by chunkIndexByEntries. + * Preserves original index order and deduplicates category headers. + */ +export function assembleReducedIndex(index: string, matchedNumbers: number[]): string { + if (matchedNumbers.length === 0) return "" + + const { entries } = parseIndexEntries(index) + const matchSet = new Set(matchedNumbers) + const lines: string[] = [] + let currentCat = "" + + for (let i = 0; i < entries.length; i++) { + const entryNum = i + 1 + if (!matchSet.has(entryNum)) continue + + if (entries[i].category !== currentCat) { + currentCat = entries[i].category + lines.push(`## ${currentCat}`) + } + lines.push(entries[i].text) + } + + return lines.join("\n") +} + +/** + * Build the system prompt for a pre-match LLM call. + * The LLM reads the source document and a chunk of index entries, + * then outputs matching entry numbers. + */ +export function buildPrematchPrompt(sourceContent: string, chunk: string): string { + return [ + "You are a relevance matcher. Read the source document and determine", + "which wiki index entries are related to it.", + "", + "## Source Document", + sourceContent, + "", + "## Index Chunk", + "Below is a chunk of the wiki index. Each numbered item is a page entry.", + "For each item, determine whether it covers the same subject as any", + "entity or concept in the source document. A match means the page", + "is about the same entity, concept, method, or topic.", + "", + chunk, + "", + "## Output Format (STRICT)", + "", + "Output ONLY matching item numbers in bracket format: [3, 12, 47]", + "If no items match, output exactly: none", + "", + "Do not output explanations, reasoning, or any other text.", + ].join("\n") +} + +export interface ParsedIndexBlock { + category: string + entries: string[] +} + +const INDEX_BLOCK_REGEX = /---INDEX:\s*(.+?)\s*---\n([\s\S]*?)---END INDEX---/g + +/** + * Parse ---INDEX: Category--- blocks from LLM generation output. + * Pattern follows existing parseFileBlocks / parseReviewBlocks conventions. + */ +export function parseIndexBlocks(text: string): ParsedIndexBlock[] { + const normalized = text.replace(/\r\n/g, "\n") + const blocks: ParsedIndexBlock[] = [] + + for (const match of normalized.matchAll(INDEX_BLOCK_REGEX)) { + const category = match[1].trim() + const body = match[2].trim() + const entries = body + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + blocks.push({ category, entries }) + } + + return blocks +} + +/** + * Convert a raw entry line from an INDEX block into index.md format. + * Input: "rope — Rotary Position Embedding" or "rope" + * Output: "- [[rope]] — Rotary Position Embedding" or "- [[rope]]" + */ +function formatIndexEntry(rawEntry: string): string { + const trimmed = rawEntry.trim() + // Already has [[...]] format? Keep as-is (just ensure leading - [[) + if (/^\[\[/.test(trimmed)) { + return `- ${trimmed}` + } + // Split on em-dash or hyphen to separate slug from description + const dashMatch = trimmed.match(/^(.+?)\s+[—–-]\s+(.+)$/) + if (dashMatch) { + const slug = dashMatch[1].trim() + const desc = dashMatch[2].trim() + return `- [[${slug}]] — ${desc}` + } + // No description — just the slug + return `- [[${trimmed}]]` +} + +/** + * Programmatically append parsed INDEX block entries to existing index.md content. + * - Appends to existing category sections + * - Creates new category sections at the end if needed + */ +export function appendIndexEntries(indexContent: string, blocks: ParsedIndexBlock[]): string { + if (blocks.length === 0) return indexContent + + const lines = indexContent.split("\n") + const result = [...lines] + + for (const block of blocks) { + if (block.entries.length === 0) continue + + const categoryHeader = `## ${block.category}` + const formattedEntries = block.entries.map(formatIndexEntry) + + // Find the category section + let catStartIdx = -1 + for (let i = 0; i < result.length; i++) { + if (result[i].trim() === categoryHeader) { + catStartIdx = i + break + } + } + + if (catStartIdx >= 0) { + // Category exists — find the last entry line in this section + let insertIdx = catStartIdx + 1 + for (let i = catStartIdx + 1; i < result.length; i++) { + const line = result[i].trim() + if (/^##\s/.test(line)) break // Hit next category + if (/^[-*]\s+\[\[/.test(line) || /^\(none yet\)/.test(line)) { + insertIdx = i + 1 + } + } + // Remove "(none yet)" placeholder if present + const placeholderIdx = result.indexOf("(none yet)", catStartIdx) + if (placeholderIdx >= 0 && placeholderIdx < insertIdx) { + result.splice(placeholderIdx, 1) + insertIdx-- + } + // Insert entries + result.splice(insertIdx, 0, ...formattedEntries) + } else { + // Category doesn't exist — append at end + result.push("") + result.push(categoryHeader) + result.push(...formattedEntries) + } + } + + return result.join("\n") +} + +const PREMATCH_CONCURRENCY = 8 + +/** + * Run pre-match LLM calls in parallel across all index chunks. + * Returns the union of all matched entry numbers. + * Failed chunks are logged and skipped (treated as 0 matches). + */ +export async function runPrematchParallel( + chunks: string[], + sourceContent: string, + llmConfig: LlmConfig, + signal: AbortSignal | undefined, +): Promise { + if (chunks.length === 0) return [] + + const results: number[][] = [] + + // Process in batches of PREMATCH_CONCURRENCY + for (let i = 0; i < chunks.length; i += PREMATCH_CONCURRENCY) { + if (signal?.aborted) break + const batch = chunks.slice(i, i + PREMATCH_CONCURRENCY) + + const batchResults = await Promise.all( + batch.map(async (chunk) => { + let output = "" + let hadError = false + + try { + await streamChat( + llmConfig, + [ + { role: "system", content: buildPrematchPrompt(sourceContent, chunk) }, + { role: "user", content: "Output matching item numbers for the chunk above." }, + ], + { + onToken: (token: string) => { output += token }, + onDone: () => {}, + onError: (err: Error) => { + hadError = true + console.warn(`[prematch] chunk failed: ${err.message}`) + }, + }, + signal, + { temperature: 0.1, reasoning: { mode: "high" } }, + ) + } catch (err) { + if (isReasoningOnlyResponseError(err)) return [] + hadError = true + console.warn(`[prematch] chunk threw: ${err instanceof Error ? err.message : String(err)}`) + } + + if (hadError) return [] + return parsePrematchOutput(output) + }), + ) + + results.push(...batchResults) + } + + // Flatten and deduplicate + return [...new Set(results.flat())] +} diff --git a/src/lib/ingest-cache.test.ts b/src/lib/ingest-cache.test.ts old mode 100644 new mode 100755 index ed949cd63..69c578402 --- a/src/lib/ingest-cache.test.ts +++ b/src/lib/ingest-cache.test.ts @@ -7,7 +7,7 @@ vi.mock("@/commands/fs", () => ({ fileExists: vi.fn(), })) -import { checkIngestCache, saveIngestCache } from "./ingest-cache" +import { checkIngestCache, saveIngestCache, checkPrematchCache, savePrematchCache } from "./ingest-cache" import { readFile, writeFile, fileExists } from "@/commands/fs" const mockReadFile = vi.mocked(readFile) @@ -94,3 +94,45 @@ describe("ingest-cache — checkIngestCache", () => { expect(result).toBeNull() }) }) + +describe("prematch cache", () => { + beforeEach(() => { + mockReadFile.mockReset() + mockWriteFile.mockReset() + mockWriteFile.mockResolvedValue(undefined as unknown as void) + }) + + it("returns null when no cache entry exists", async () => { + mockReadFile.mockResolvedValue(JSON.stringify({ entries: {}, prematch: {} })) + const result = await checkPrematchCache("/project", "source-hash-abc", "chunk-hash-xyz") + expect(result).toBeNull() + }) + + it("returns cached result on hit", async () => { + let persisted = "" + mockReadFile.mockImplementation(async () => persisted || JSON.stringify({ entries: {}, prematch: {} })) + mockWriteFile.mockImplementation(async (_p, c) => { persisted = c }) + + await savePrematchCache("/project", "src-hash", "chunk-hash", [1, 3, 5]) + const result = await checkPrematchCache("/project", "src-hash", "chunk-hash") + expect(result).toEqual([1, 3, 5]) + }) + + it("returns null when prematch key not in cache", async () => { + let persisted = "" + mockReadFile.mockImplementation(async () => persisted || JSON.stringify({ entries: {}, prematch: {} })) + mockWriteFile.mockImplementation(async (_p, c) => { persisted = c }) + + await savePrematchCache("/project", "src-hash", "chunk-hash", [1, 3]) + // Different key should miss + const result = await checkPrematchCache("/project", "different-src", "chunk-hash") + expect(result).toBeNull() + }) + + it("handles cache file with no prematch field (backwards compat)", async () => { + // Old cache format without prematch field + mockReadFile.mockResolvedValue(JSON.stringify({ entries: {} })) + const result = await checkPrematchCache("/project", "src", "chunk") + expect(result).toBeNull() + }) +}) diff --git a/src/lib/ingest-cache.ts b/src/lib/ingest-cache.ts old mode 100644 new mode 100755 index eaa82d7d8..a8f5b97e8 --- a/src/lib/ingest-cache.ts +++ b/src/lib/ingest-cache.ts @@ -15,6 +15,7 @@ interface CacheEntry { interface CacheData { entries: Record // keyed by source filename + prematch?: Record // keyed by "sourceHash:chunkHash" } async function sha256(content: string): Promise { @@ -103,13 +104,42 @@ export async function saveIngestCache( ): Promise { const cache = await loadCache(projectPath) const hash = await sha256(sourceContent) - const newEntries = { ...cache.entries } - newEntries[sourceFileName] = { + cache.entries[sourceFileName] = { hash, timestamp: Date.now(), filesWritten, } - await saveCache(projectPath, { entries: newEntries }) + await saveCache(projectPath, cache) +} + +/** + * Check if a pre-match result is cached for a source+chunk combination. + * Returns the cached matched numbers, or null if not cached. + */ +export async function checkPrematchCache( + projectPath: string, + sourceHash: string, + chunkHash: string, +): Promise { + const cache = await loadCache(projectPath) + const key = `${sourceHash}:${chunkHash}` + return cache.prematch?.[key] ?? null +} + +/** + * Save a pre-match result to the cache. + */ +export async function savePrematchCache( + projectPath: string, + sourceHash: string, + chunkHash: string, + matchedNumbers: number[], +): Promise { + const cache = await loadCache(projectPath) + if (!cache.prematch) cache.prematch = {} + const key = `${sourceHash}:${chunkHash}` + cache.prematch[key] = matchedNumbers + await saveCache(projectPath, cache) } /** @@ -120,9 +150,8 @@ export async function removeFromIngestCache( sourceFileName: string, ): Promise { const cache = await loadCache(projectPath) - const newEntries = { ...cache.entries } - delete newEntries[sourceFileName] - await saveCache(projectPath, { entries: newEntries }) + delete cache.entries[sourceFileName] + await saveCache(projectPath, cache) } /** Move a cache entry when an unchanged source is renamed inside raw/sources. */ diff --git a/src/lib/ingest-parse.test.ts b/src/lib/ingest-parse.test.ts old mode 100644 new mode 100755 index 525b4d728..fa72b5bc9 --- a/src/lib/ingest-parse.test.ts +++ b/src/lib/ingest-parse.test.ts @@ -30,7 +30,6 @@ import { rewriteIngestPathFromTitleForTargetLanguage, canonicalizeSourcesField, isAppManagedAggregatePath, - updateBoundedRecentIndexSection, } from "./ingest" // ── Happy paths ───────────────────────────────────────────────────── @@ -92,6 +91,22 @@ describe("parseFileBlocks — canonical shapes", () => { ].join("\n") expect(parseFileBlocks(text).blocks).toHaveLength(1) }) + + it("silently skips FILE block for wiki/overview.md", () => { + const text = [ + "---FILE: wiki/overview.md---", + "# Overview", + "This should be skipped.", + "---END FILE---", + "", + "---FILE: wiki/entities/foo.md---", + "This should be kept.", + "---END FILE---", + ].join("\n") + const { blocks } = parseFileBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].path).toBe("wiki/entities/foo.md") + }) }) describe("source summary media refs", () => { @@ -113,24 +128,31 @@ describe("source summary media refs", () => { }) describe("aggregate repair targeting", () => { - it("repairs only the append-only log and leaves deterministic aggregates to the app", () => { + it("requests missing aggregate pages and aggregate pages dropped by truncation warnings", () => { + // wiki/log.md is the only aggregate repair target — overview.md is + // now maintained incrementally via OVERVIEW blocks, so a truncation + // warning for it must NOT trigger a full-overwrite repair. expect(aggregatePathsNeedingRepair( - ["wiki/index.md", "wiki/log.md"], - ['FILE block "wiki/overview.md" was not closed before end of stream — likely truncation.'], - )).toEqual([]) - - expect(aggregatePathsNeedingRepair(["wiki/index.md"], [])).toEqual(["wiki/log.md"]) + ["wiki/entities/foo.md"], + ['FILE block "wiki/log.md" was not closed before end of stream — likely truncation.'], + )).toEqual(["wiki/log.md"]) expect(aggregatePathsNeedingRepair( - ["wiki/index.md", "wiki/overview.md", "wiki/log.md"], + ["wiki/log.md"], [], )).toEqual([]) + + // overview.md truncation is no longer an aggregate-repair trigger. + expect(aggregatePathsNeedingRepair( + ["wiki/log.md"], + ['FILE block "wiki/overview.md" was not closed before end of stream — likely truncation.'], + )).toEqual([]) }) it("filters aggregate repair output to the requested aggregate paths only", () => { const raw = [ - "---FILE: wiki/overview.md---", - "# Overview", + "---FILE: wiki/log.md---", + "# Log", "---END FILE---", "", "---FILE: wiki/sources/should-not-touch.md---", @@ -142,9 +164,9 @@ describe("aggregate repair targeting", () => { "---END FILE---", ].join("\n") - const filtered = filterAggregateRepairOutput(raw, ["wiki/overview.md"]) + const filtered = filterAggregateRepairOutput(raw, ["wiki/log.md"]) - expect(filtered.text).toContain("---FILE: wiki/overview.md---") + expect(filtered.text).toContain("---FILE: wiki/log.md---") expect(filtered.text).not.toContain("should-not-touch") expect(filtered.text).not.toContain("wiki/entities/stray.md") expect(filtered.warnings.join("\n")).toContain("Dropped 2 non-aggregate") @@ -664,21 +686,4 @@ describe("application-managed aggregate boundaries", () => { expect(isAppManagedAggregatePath("wiki\\overview.MD")).toBe(true) expect(isAppManagedAggregatePath("wiki/entities/index.md")).toBe(false) }) - - it("bounds recent entries and preserves following sections", () => { - const existing = [ - "# Wiki Index", - "", - "## Recently Updated", - ...Array.from({ length: 205 }, (_, index) => `- [[old-${index}]] — Old ${index}`), - "", - "## Other", - "Keep me", - ].join("\n") - const result = updateBoundedRecentIndexSection(existing, ["- [[new]] — New"]) - const recent = result.split("## Recently Updated")[1].split("## Other")[0] - expect(recent.match(/^- \[\[/gm)).toHaveLength(200) - expect(recent).toContain("[[new]]") - expect(result).toContain("## Other\nKeep me") - }) }) diff --git a/src/lib/ingest-queue.integration.test.ts b/src/lib/ingest-queue.integration.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-queue.test.ts b/src/lib/ingest-queue.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-queue.ts b/src/lib/ingest-queue.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-sanitize.test.ts b/src/lib/ingest-sanitize.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-sanitize.ts b/src/lib/ingest-sanitize.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-source-path-collision.test.ts b/src/lib/ingest-source-path-collision.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest.prompt.test.ts b/src/lib/ingest.prompt.test.ts old mode 100644 new mode 100755 index 05ad17922..4db0f0daf --- a/src/lib/ingest.prompt.test.ts +++ b/src/lib/ingest.prompt.test.ts @@ -202,8 +202,8 @@ describe("long-source ingest planning", () => { expect(computeIngestGenerationMaxTokens(64_000)).toBe(8_192) expect(computeIngestGenerationMaxTokens(128_000)).toBe(16_384) expect(computeIngestGenerationMaxTokens(256_000)).toBe(24_576) - expect(computeIngestGenerationMaxTokens(1_000_000)).toBe(32_768) - expect(computeIngestReviewMaxTokens(1_000_000)).toBe(8_192) + expect(computeIngestGenerationMaxTokens(1_000_000)).toBe(384_000) + expect(computeIngestReviewMaxTokens(1_000_000)).toBe(384_000) }) it("scales source budget from the configured context window instead of a fixed 50k cap", () => { diff --git a/src/lib/ingest.real-llm.test.ts b/src/lib/ingest.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest.scenarios.test.ts b/src/lib/ingest.scenarios.test.ts old mode 100644 new mode 100755 index b9d1e9031..eb17b9c91 --- a/src/lib/ingest.scenarios.test.ts +++ b/src/lib/ingest.scenarios.test.ts @@ -3,7 +3,7 @@ * * Each scenario materializes an initial project, a source document, and two * canned LLM responses (stage 1 analysis, stage 2 generation with FILE + - * REVIEW blocks). The runner mocks streamChat to emit them sequentially. + * INDEX + REVIEW blocks). The runner mocks streamChat to emit them sequentially. * * After ingest runs, the runner asserts: * - expected files exist on disk with expected substrings @@ -104,7 +104,12 @@ async function setup(scenario: IngestScenario): Promise { maxContextSize: 128000, }) - // Queue up the two sequenced LLM responses + // Queue up the sequenced LLM responses: + // 1. Index prematch: BASIC_INDEX has 1 entry ([[attention]]), so [1] matches. + // 2. Overview prematch: skipped if no overview.md exists. + // 3. Analysis (Stage 1) + // 4. Generation (Stage 2) + // 5. Review suggestion (if triggered) const analysis = await fs.readFile( path.join(FIXTURES_ROOT, scenario.name, "llm-analysis.txt"), "utf-8", @@ -113,7 +118,7 @@ async function setup(scenario: IngestScenario): Promise { path.join(FIXTURES_ROOT, scenario.name, "llm-generation.txt"), "utf-8", ) - pendingResponses = [analysis, generation] + pendingResponses = ["[1]", analysis, generation] return { tmp } } @@ -244,6 +249,7 @@ describe("ingest scenarios (fixture-driven)", () => { }) pendingResponses = [ + "[1]", "analysis", [ "---FILE: wiki/sources/schema-routing.md---", diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 1a5c50a0e..3894f1f73 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -39,9 +39,24 @@ import { } from "@/lib/extract-source-images" import { captionMarkdownImages, loadCaptionCache } from "@/lib/image-caption-pipeline" import type { MultimodalConfig } from "@/stores/wiki-store" -import { GENERATION_WIKI_TYPES } from "@/lib/wiki-page-types" +import { GENERATION_WIKI_TYPES, inferWikiTypeFromPath, wikiTypeLabel } from "@/lib/wiki-page-types" import { computeContextBudget } from "@/lib/context-budget" import { refreshProjectFileTree } from "@/lib/project-file-tree-refresh" +import { + chunkIndexByEntries, + assembleReducedIndex, + runPrematchParallel, + parseIndexBlocks, + appendIndexEntries, +} from "./index-chunker" +import { + chunkOverviewBySections, + runOverviewPrematchParallel, + assembleReducedOverview, + parseOverviewBlocks, + appendOverviewContent, + createInitialOverview, +} from "@/lib/overview-blocks" const LONG_SOURCE_MIN_BUDGET = 8_000 const LONG_SOURCE_MAX_SINGLE_PASS_BUDGET = 300_000 @@ -53,9 +68,10 @@ const INGEST_GENERATION_TOKENS_DEFAULT = 8_192 const INGEST_GENERATION_TOKENS_128K = 16_384 const INGEST_GENERATION_TOKENS_256K = 24_576 const INGEST_GENERATION_TOKENS_512K = 32_768 +const INGEST_GENERATION_TOKENS_1M = 384_000 const REVIEW_STAGE_MIN_SIGNAL_CHARS = 10_000 const REVIEW_STAGE_MIN_FILE_BLOCKS = 4 -const AGGREGATE_WIKI_PATHS = ["wiki/index.md", "wiki/overview.md", "wiki/log.md"] as const +const AGGREGATE_WIKI_PATHS = ["wiki/log.md"] as const function appendSavedImageRefsForCaption(content: string, images: SavedImage[]): string { if (images.length === 0) return content @@ -513,6 +529,14 @@ export function parseFileBlocks(text: string): ParseFileBlocksResult { i++ } + // index.md and overview.md are updated exclusively via INDEX/OVERVIEW + // blocks (append mode). + // If the model produces a FILE block for either (despite explicit prompt + // instructions not to), silently skip — no warning, no truncation error. + if (path === "wiki/index.md" || path === "wiki/overview.md") { + continue + } + if (!closed) { // H2 fix (partial): we can't fabricate content the LLM never // sent, but we surface the drop instead of silently hiding it. @@ -923,7 +947,70 @@ async function autoIngestImpl( } } - const stableContextLength = schema.length + purpose.length + index.length + overview.length + // ── Step 0.7: Pre-match index chunks ───────────────────── + // Split index into chunks, run parallel LLM calls to find + // entries relevant to this source, assemble a reduced index. + // Runs BEFORE budget calculation so stableContextLength reflects + // the actual (reduced) context size, not the full index length. + const CHUNK_SIZE = 500 + let reducedIndex = index + if (index.trim()) { + const chunks = chunkIndexByEntries(index, CHUNK_SIZE) + if (chunks.length > 0) { + activity.updateItem(activityId, { + detail: `Step 0.7: Pre-matching index (${chunks.length} chunks)...`, + }) + const matchedNumbers = await runPrematchParallel( + chunks, + enrichedSourceContent, + llmConfig, + signal, + ) + reducedIndex = assembleReducedIndex(index, matchedNumbers) + if (!reducedIndex) { + reducedIndex = "(no matching wiki entries found)" + } + // If prematch returned nothing meaningful, keep the full index + // as fallback — an empty reduced index would make the model + // skip all page generation. + if (reducedIndex.length < 10) { + reducedIndex = index + } + console.log( + `[ingest:prematch] index reduced from ${index.length} to ${reducedIndex.length} chars ` + + `(${matchedNumbers.length} matches from ${chunks.length} chunks)`, + ) + } + } + + // ── Step 0.8: Pre-match overview sections ───────────────────── + let reducedOverview = overview + if (overview.trim()) { + const overviewChunks = chunkOverviewBySections(overview, 16000) + if (overviewChunks.length > 0) { + activity.updateItem(activityId, { + detail: `Step 0.8: Pre-matching overview (${overviewChunks.length} chunks)...`, + }) + const matchedParagraphs = await runOverviewPrematchParallel( + overviewChunks, + enrichedSourceContent, + llmConfig, + signal, + ) + reducedOverview = assembleReducedOverview(overview, matchedParagraphs) + if (!reducedOverview) { + reducedOverview = "(no matching overview paragraphs found)" + } + console.log( + `[ingest:overview-prematch] overview reduced from ${overview.length} to ${reducedOverview.length} chars ` + + `(${matchedParagraphs.length} paragraphs matched from ${overviewChunks.length} chunks)`, + ) + } + } + + // Compute budget with REDUCED sizes (post-prematch) so sourceBudget + // is not artificially constrained by the full index/overview length. + const stableContextLength = schema.length + purpose.length + reducedIndex.length + reducedOverview.length const sourceBudget = computeIngestSourceBudget(llmConfig.maxContextSize, stableContextLength) let sourceContext = enrichedSourceContent let precomputedAnalysis = "" @@ -935,7 +1022,7 @@ async function autoIngestImpl( llmConfig, purpose, schema, - index, + reducedIndex, sourceIdentity, sourceSummarySlug, folderContext, @@ -966,7 +1053,7 @@ async function autoIngestImpl( await streamChat( llmConfig, [ - { role: "system", content: buildAnalysisPrompt(purpose, index, sourceContext, schema) }, + { role: "system", content: buildAnalysisPrompt(purpose, reducedIndex, sourceContext, schema) }, { role: "user", content: `Analyze this source document:\n\n**File:** ${sourceIdentity}${folderContext ? `\n**Folder context:** ${folderContext}` : ""}\n\n---\n\n${sourceContext}` }, ], { @@ -977,7 +1064,7 @@ async function autoIngestImpl( }, }, signal, - { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: 4096 }, + { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: computeIngestReviewMaxTokens(llmConfig.maxContextSize) }, ) } @@ -998,7 +1085,7 @@ async function autoIngestImpl( await streamChat( llmConfig, [ - { role: "system", content: buildGenerationPrompt(schema, purpose, index, sourceIdentity, overview, sourceContext, sourceSummaryPath) }, + { role: "system", content: buildGenerationPrompt(schema, purpose, reducedIndex, sourceIdentity, reducedOverview, sourceContext, sourceSummaryPath) }, { role: "user", content: [ @@ -1056,7 +1143,7 @@ async function autoIngestImpl( role: "system", content: buildReviewSuggestionPrompt( purpose, - index, + reducedIndex, sourceIdentity, analysis, sourceContext, @@ -1066,7 +1153,7 @@ async function autoIngestImpl( }, { role: "user", - content: "Emit only high-value REVIEW blocks for follow-up research or unresolved knowledge gaps. Output nothing if there are none.", + content: "Emit only high-value REVIEW blocks for follow-up research or unresolved knowledge gaps. If there are none, output exactly: none", }, ], { @@ -1111,15 +1198,77 @@ async function autoIngestImpl( const writeWarnings = writeResult.warnings const hardFailures = writeResult.hardFailures - try { - if (await updateWikiIndexDeterministically(pp, writtenPaths)) { - writtenPaths.push("wiki/index.md") - onFileWritten?.("wiki/index.md") + // ── Step 3.5: Process INDEX blocks ────────────────────── + // Parse ---INDEX: blocks from generation output and append + // entries to the existing index.md programmatically. + const indexBlocks = parseIndexBlocks(generation) + if (indexBlocks.length > 0) { + try { + const indexAbs = `${pp}/wiki/index.md` + const existingIndex = await tryReadFile(indexAbs) + const updatedIndex = appendIndexEntries(existingIndex, indexBlocks) + await writeFile(indexAbs, updatedIndex) + console.log( + `[ingest:index] appended ${indexBlocks.reduce((s, b) => s + b.entries.length, 0)} entries ` + + `across ${indexBlocks.length} categories to index.md`, + ) + if (!writtenPaths.includes("wiki/index.md")) { + writtenPaths.push("wiki/index.md") + } + } catch (err) { + writeWarnings.push( + `Failed to append INDEX blocks: ${err instanceof Error ? err.message : String(err)}`, + ) + } + } + + // ── Step 3.6: Process OVERVIEW blocks ────────────────────── + const overviewBlocks = parseOverviewBlocks(generation) + if (overviewBlocks.length > 0) { + try { + const overviewAbs = `${pp}/wiki/overview.md` + const existingOverview = await tryReadFile(overviewAbs) + const updatedOverview = existingOverview + ? appendOverviewContent(existingOverview, overviewBlocks) + : createInitialOverview(overviewBlocks) + await writeFile(overviewAbs, updatedOverview) + console.log( + `[ingest:overview] appended ${overviewBlocks.length} section(s) to overview.md`, + ) + if (!writtenPaths.includes("wiki/overview.md")) { + writtenPaths.push("wiki/overview.md") + } + } catch (err) { + writeWarnings.push( + `Failed to append OVERVIEW blocks: ${err instanceof Error ? err.message : String(err)}`, + ) + } + } + + // ── Step 3.6: Fallback INDEX construction ─────────────── + // If no INDEX blocks were emitted, construct them from the + // written FILE blocks as a fallback. This replaces the old + // aggregate-repair-for-index.md path. + if (indexBlocks.length === 0 && writtenPaths.length > 1) { + try { + const fallbackBlocks = buildFallbackIndexBlocks(generation) + if (fallbackBlocks.length > 0) { + const indexAbs = `${pp}/wiki/index.md` + const existingIndex = await tryReadFile(indexAbs) + const updatedIndex = appendIndexEntries(existingIndex, fallbackBlocks) + await writeFile(indexAbs, updatedIndex) + writeWarnings.push("No INDEX blocks in generation — constructed fallback entries from FILE blocks") + if (!writtenPaths.includes("wiki/index.md")) { + writtenPaths.push("wiki/index.md") + } + console.log( + `[ingest:index-fallback] constructed ${fallbackBlocks.reduce((s, b) => s + b.entries.length, 0)} entries ` + + `across ${fallbackBlocks.length} categories from FILE blocks`, + ) + } + } catch (err) { + writeWarnings.push(`Fallback index construction failed: ${err instanceof Error ? err.message : String(err)}`) } - } catch (err) { - writeWarnings.push( - `Deterministic index update failed: ${err instanceof Error ? err.message : String(err)}`, - ) } const aggregateRepairPaths = aggregatePathsNeedingRepair(writtenPaths, writeWarnings) @@ -1148,8 +1297,8 @@ async function autoIngestImpl( content: buildAggregateRepairPrompt( repairableAggregatePaths, purpose, - index, - overview, + "", + "", sourceIdentity, analysis, sourceContext, @@ -1257,11 +1406,12 @@ async function autoIngestImpl( } } - // ── Step 4: Parse review items ──────────────────────────────── + // ── Step 4: Parse review items ──────────────────────────────── throwIfIngestAborted(signal, activityId) const reviewItems = [ ...parseReviewBlocks(generation, sp), ...parseReviewBlocks(reviewSuggestionOutput, sp), + ...detectCodeReviewItems(generation, index, sp), ] if (reviewItems.length > 0) { useReviewStore.getState().addItems(reviewItems) @@ -1377,6 +1527,35 @@ export function isAppManagedAggregatePath(relativePath: string): boolean { return normalized === "wiki/index.md" || normalized === "wiki/overview.md" } +/** + * When generation output has no ---INDEX: blocks, construct fallback + * entries from the FILE blocks that were written. Extracts slug from + * path and title from frontmatter. + */ +function buildFallbackIndexBlocks( + generation: string, +): import("./index-chunker").ParsedIndexBlock[] { + const { blocks } = parseFileBlocks(generation) + const byCategory = new Map() + + for (const block of blocks) { + // Skip aggregate files and source summaries + if (block.path === "wiki/index.md" || block.path === "wiki/log.md" || block.path === "wiki/overview.md") continue + if (block.path.startsWith("wiki/sources/")) continue + + const type = inferWikiTypeFromPath(block.path) ?? "entity" + const category = wikiTypeLabel(type) + "s" + const slug = block.path.replace(/\.md$/, "").split("/").pop() ?? "" + const title = extractGeneratedPageTitle(block.content) ?? slug + + const entry = title === slug ? slug : `${slug} — ${title}` + if (!byCategory.has(category)) byCategory.set(category, []) + byCategory.get(category)!.push(entry) + } + + return [...byCategory.entries()].map(([category, entries]) => ({ category, entries })) +} + const CJK_OUTPUT_LANGUAGES = new Set(["Chinese", "Traditional Chinese", "Japanese", "Korean"]) function containsCjk(text: string): boolean { @@ -1429,63 +1608,6 @@ export function aggregatePathsNeedingRepair(writtenPaths: string[], warnings: st ) } -async function updateWikiIndexDeterministically( - projectPath: string, - writtenPaths: string[], -): Promise { - const candidates = Array.from(new Set(writtenPaths.map(normalizePath))).filter((path) => - path.startsWith("wiki/") - && path.endsWith(".md") - && !AGGREGATE_WIKI_PATHS.includes(path as (typeof AGGREGATE_WIKI_PATHS)[number]), - ) - if (candidates.length === 0) return false - - const indexPath = `${projectPath}/wiki/index.md` - const index = await readFile(indexPath).catch(() => "# Wiki Index\n") - const knownTargets = new Set( - Array.from(index.matchAll(/\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|[^\]]+)?\]\]/g)) - .map((match) => normalizeIndexTarget(match[1])), - ) - const additions: string[] = [] - for (const path of candidates) { - const target = path.replace(/^wiki\//, "").replace(/\.md$/i, "") - if (knownTargets.has(normalizeIndexTarget(target))) continue - const content = await readFile(`${projectPath}/${path}`).catch(() => "") - const parsed = parseFrontmatter(content) - const title = typeof parsed.frontmatter?.title === "string" - ? parsed.frontmatter.title.trim() - : getFileName(path).replace(/\.md$/i, "") - additions.push(`- [[${target}]] — ${title}`) - } - if (additions.length === 0) return false - - await writeFile(indexPath, updateBoundedRecentIndexSection(index, additions)) - return true -} - -function normalizeIndexTarget(target: string): string { - return normalizePath(target) - .replace(/^wiki\//i, "") - .replace(/\.md$/i, "") - .toLowerCase() -} - -export function updateBoundedRecentIndexSection(index: string, additions: string[]): string { - const section = "## Recently Updated" - const lines = index.trimEnd().split("\n") - const start = lines.findIndex((line) => line.trim() === section) - const prefix = start >= 0 ? lines.slice(0, start) : lines - const sectionEnd = start >= 0 - ? lines.findIndex((line, position) => position > start && /^##\s+/.test(line)) - : -1 - const existing = start >= 0 - ? lines.slice(start + 1, sectionEnd >= 0 ? sectionEnd : undefined).filter((line) => /^-\s+/.test(line)) - : [] - const suffix = sectionEnd >= 0 ? lines.slice(sectionEnd) : [] - const recent = Array.from(new Set([...additions, ...existing])).slice(0, 200) - return [...prefix, "", section, ...recent, ...(suffix.length ? ["", ...suffix] : []), ""].join("\n") -} - export function filterAggregateRepairOutput(text: string, allowedPaths: string[]): { text: string warnings: string[] @@ -1966,6 +2088,66 @@ function isOwnedOnlyBySource(content: string, sourceIdentity: string): boolean { const REVIEW_BLOCK_REGEX = /---REVIEW:\s*(\w[\w-]*)\s*\|\s*(.+?)\s*---\n([\s\S]*?)---END REVIEW---/g +/** + * Scan generation output for wikilinks and detect missing pages / duplicates + * purely from the wiki index — no LLM call needed. + */ +function detectCodeReviewItems( + generation: string, + index: string, + sourcePath: string, +): Omit[] { + if (!generation.trim() || !index.trim()) return [] + + // Collect existing slugs from the wiki index + const existingSlugs = new Set() + for (const m of index.matchAll(/\[\[([^\]|#]+)(?:\|[^\]]*)?\]\]/g)) { + existingSlugs.add(m[1].trim().toLowerCase()) + } + + // Also collect slugs from FILE blocks in the generation output — + // pages being created by this ingest should not be reported as missing. + // Match: ---FILE: wiki/.../slug.md AND title: "Title Name" inside frontmatter. + const FILE_BLOCK_REGEX = /---FILE:\s*(wiki\/[^\n]+?\.md)[^\n]*\n([\s\S]*?)---END FILE---/g + for (const fb of generation.matchAll(FILE_BLOCK_REGEX)) { + const filePath = fb[1].trim() + const frontmatter = fb[2] + // Add file-stem slug (wiki/concepts/rope → rope) + const stem = filePath.replace(/^wiki\/[^/]+\/(.+)\.md$/, "$1") + existingSlugs.add(stem.toLowerCase()) + // Add frontmatter title + const titleMatch = frontmatter.match(/^title:\s*"?(.+?)"?\s*$/m) + if (titleMatch) existingSlugs.add(titleMatch[1].trim().toLowerCase()) + } + + // Extract referenced slugs from the generation output + const wikilinkRegex = /\[\[([^\]|#]+)(?:\|[^\]]*)?\]\]/g + const referencedSlugs = new Map() + for (const m of generation.matchAll(wikilinkRegex)) { + const slug = m[1].trim() + const key = slug.toLowerCase() + if (!referencedSlugs.has(key)) referencedSlugs.set(key, slug) + } + + const items: Omit[] = [] + for (const [key, original] of referencedSlugs) { + if (!existingSlugs.has(key)) { + items.push({ + type: "missing-page", + title: original, + description: `Page "[[${original}]]" is referenced but does not exist in the wiki. Created automatically from source ingestion.`, + sourcePath, + options: [ + { label: "Create Page", action: "Create Page" }, + { label: "Skip", action: "Skip" }, + ], + }) + } + } + return items +} + + function parseReviewBlocks( text: string, sourcePath: string, @@ -2109,7 +2291,7 @@ export function buildGenerationPrompt( purpose: string, index: string, sourceFileName: string, - overview?: string, + reducedOverview?: string, sourceContent: string = "", sourceSummaryPath?: string, ): string { @@ -2146,8 +2328,10 @@ export function buildGenerationPrompt( `1. A source summary page at **${summaryPath}** (MUST use this exact path)`, "2. Entity or schema-defined typed pages for key named things identified in the analysis. Prefer schema-defined directories when present; otherwise use wiki/entities/.", "3. Concept or schema-defined typed pages for key ideas, methods, techniques, and abstractions. Prefer schema-defined directories when present; otherwise use wiki/concepts/.", - "4. A log entry for wiki/log.md (just the new entry to append, format: ## [YYYY-MM-DD] ingest | Title)", - "Do not generate wiki/index.md or wiki/overview.md. The application maintains aggregate navigation separately so large wikis are never rewritten through model output.", + "4. INDEX blocks for new entries (see INDEX block format below) — do NOT output a complete wiki/index.md file", + "5. A log entry for wiki/log.md (just the new entry to append, format: ## [YYYY-MM-DD] ingest | Title)", + "6. OVERVIEW blocks for new sections (see OVERVIEW block format below) — do NOT output a complete wiki/overview.md file", + "Do not generate wiki/index.md or wiki/overview.md as complete files. The application maintains aggregate navigation separately so large wikis are never rewritten through model output.", "", "## Frontmatter Rules (CRITICAL — parser is strict)", "", @@ -2227,8 +2411,8 @@ export function buildGenerationPrompt( " SEARCH: automated technical debt detection AI generated code | software quality metrics LLM code generation | static analysis tools agentic software development", "", purpose ? `## Wiki Purpose\n${purpose}` : "", - index ? `## Current Wiki Index (preserve all existing entries, add new ones)\n${index}` : "", - overview ? `## Current Overview (update this to reflect the new source)\n${overview}` : "", + index ? `## Current Wiki Index (for reference only — do NOT reproduce it)\n${index}` : "", + reducedOverview ? `## Current Overview (relevant sections only — for reference)\nIf the source only restates information already covered in this context,\noutput NO OVERVIEW block — skip it entirely.\n\n${reducedOverview}` : "", "", // ── OUTPUT FORMAT MUST BE THE LAST SECTION — models weight recent instructions highest ── "## Output Format (MUST FOLLOW EXACTLY — this is how the parser reads your response)", @@ -2252,6 +2436,28 @@ export function buildGenerationPrompt( "---END REVIEW---", "```", "", + "INDEX block template (for new index entries only):", + "```", + "---INDEX: CategoryName---", + "slug — one-line description", + "---END INDEX---", + "```", + "", + "Output one INDEX block per category that has new entries.", + "CategoryName must match an existing ## heading in the index.", + "Do NOT output a FILE block for the index page. The index is updated ONLY via INDEX blocks.", + "", + "OVERVIEW block template (for overview paragraphs):", + "```", + "---OVERVIEW: SectionName---", + "1-2 paragraphs about this source's key topics.", + "---END OVERVIEW---", + "```", + "", + "Output one OVERVIEW block. SectionName should match an existing ## heading", + "in the overview context below, or create a new section name if the topic is new.", + "Do NOT output a FILE block for the overview page. Use OVERVIEW blocks instead.", + "", "## Output Requirements (STRICT — deviations will cause parse failure)", "", "1. The FIRST character of your response MUST be `-` (the opening of `---FILE:`).", @@ -2262,6 +2468,11 @@ export function buildGenerationPrompt( "6. Between blocks, use only blank lines — no prose.", "7. FILE block prose (body, explanations, descriptions, section text) must use the mandatory output language specified below. Preserve proper nouns, acronyms, model names, dataset names, tool/library names, code identifiers, URLs, file names, citation strings, paper titles, and technical terms with no widely-used localized equivalent in their standard original form, including in page names and section headings.", "", + "8. INDEX blocks appear AFTER all FILE blocks and BEFORE any REVIEW blocks.", + "9. Each INDEX block must specify a category name after ---INDEX:.", + "10. Each entry line in an INDEX block must start with a slug followed by \" — \" and a description.", + "11. Do NOT output a FILE block for the index page. The index is updated ONLY via INDEX blocks.", + "", "If you start with anything other than `---FILE:`, the entire response will be discarded.", "", // Repeat the language directive at the very end so it wins the "most @@ -2275,7 +2486,7 @@ export function buildGenerationPrompt( function buildReviewSuggestionPrompt( purpose: string, - index: string, + reducedIndex: string, sourceIdentity: string, analysis: string, sourceContext: string, @@ -2284,7 +2495,6 @@ function buildReviewSuggestionPrompt( ): string { const { maxCtx } = computeContextBudget(maxContextSize) const sectionCap = Math.max(4_000, Math.floor(maxCtx * 0.15)) - const indexCap = Math.max(3_000, Math.floor(sectionCap * 0.8)) return [ "You are identifying high-value follow-up research items for a personal wiki.", "Do not output chain-of-thought, hidden reasoning, or explanatory preamble.", @@ -2292,16 +2502,13 @@ function buildReviewSuggestionPrompt( languageRule(sourceContext), "", "Your job is NOT to generate wiki pages. The wiki page generation already happened.", - "Output only REVIEW blocks for unresolved knowledge gaps that deserve human attention or Deep Research.", + "Missing-page and duplicate checks are handled automatically. Your focus:", "", - "Create REVIEW blocks only for genuinely useful follow-up work:", - "- missing-page: an important entity/concept is referenced but still lacks a dedicated page", "- suggestion: a research question, source type, or comparison that would materially improve the wiki", "- contradiction: a conflict or tension that requires user judgment", - "- duplicate: likely duplicate pages/names that need user review", "", - "Prefer 1-5 high-signal reviews. If there is nothing worth reviewing, output nothing.", - "For suggestion and missing-page reviews, include a SEARCH line with 2-3 keyword-rich web search queries separated by ` | `.", + "Prefer 1-5 high-signal reviews. If there is nothing worth reviewing, output exactly: none.", + "For suggestion reviews, include a SEARCH line with 2-3 keyword-rich web search queries separated by ` | `.", "Use only these options: OPTIONS: Create Page | Skip", "", "REVIEW block template:", @@ -2317,7 +2524,7 @@ function buildReviewSuggestionPrompt( "Return REVIEW blocks only. Do not output FILE blocks. Do not wrap the response in markdown fences.", "", purpose ? `## Wiki Purpose\n${purpose}` : "", - index ? `## Current Wiki Index\n${trimLongText(index, indexCap)}` : "", + reducedIndex ? `## Relevant Wiki Index\n${reducedIndex}` : "", "", `## Source\n${sourceIdentity}`, "", @@ -2359,8 +2566,6 @@ function buildAggregateRepairPrompt( "", "Rules:", `- Use today's date ${today} for log entries and frontmatter dates.`, - "- For wiki/index.md: output the complete updated index, preserving existing entries and adding the new source-derived entries.", - "- For wiki/overview.md: output the complete updated overview, reflecting the full wiki plus this new source.", "- For wiki/log.md: output only the new log entry to append, format `## [YYYY-MM-DD] ingest | Title`.", "- Output only FILE blocks. Nothing else.", "", @@ -2426,6 +2631,7 @@ export function computeIngestSourceBudget( export function computeIngestGenerationMaxTokens(maxContextSize: number | undefined): number { const { maxCtx } = computeContextBudget(maxContextSize) + if (maxCtx >= 1_000_000) return INGEST_GENERATION_TOKENS_1M if (maxCtx >= 512_000) return INGEST_GENERATION_TOKENS_512K if (maxCtx >= 256_000) return INGEST_GENERATION_TOKENS_256K if (maxCtx >= 128_000) return INGEST_GENERATION_TOKENS_128K @@ -2433,7 +2639,7 @@ export function computeIngestGenerationMaxTokens(maxContextSize: number | undefi } export function computeIngestReviewMaxTokens(maxContextSize: number | undefined): number { - return Math.min(8_192, Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize) / 2))) + return Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize))) } function splitOversizedBlock(block: string, targetChars: number): string[] { @@ -2793,7 +2999,7 @@ async function analyzeLongSourceInChunks( }, }, signal, - { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: 4096 }, + { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: computeIngestReviewMaxTokens(llmConfig.maxContextSize) }, ) throwIfIngestAborted(signal, activityId) diff --git a/src/lib/keyboard-utils.test.ts b/src/lib/keyboard-utils.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/keyboard-utils.ts b/src/lib/keyboard-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/language-metadata.test.ts b/src/lib/language-metadata.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/language-metadata.ts b/src/lib/language-metadata.ts old mode 100644 new mode 100755 diff --git a/src/lib/latex-to-unicode.ts b/src/lib/latex-to-unicode.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint-fixes.test.ts b/src/lib/lint-fixes.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint-fixes.ts b/src/lib/lint-fixes.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.real-llm.test.ts b/src/lib/lint.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.scenarios.test.ts b/src/lib/lint.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.test.ts b/src/lib/lint.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.ts b/src/lib/lint.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-client.real-llm.test.ts b/src/lib/llm-client.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-client.test.ts b/src/lib/llm-client.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-client.ts b/src/lib/llm-client.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-providers.test.ts b/src/lib/llm-providers.test.ts old mode 100644 new mode 100755 index 885fd9719..f065b6bff --- a/src/lib/llm-providers.test.ts +++ b/src/lib/llm-providers.test.ts @@ -540,10 +540,10 @@ describe("reasoning controls", () => { expect(body.reasoning).toBeUndefined() }) - it("maps DeepSeek V4 reasoning off to thinking disabled for structured tasks", () => { + it("maps DeepSeek V4 pro reasoning off to thinking disabled for structured tasks", () => { const cfg = mkConfig({ provider: "custom", - model: "deepseek-v4-flash", + model: "deepseek-v4-pro", customEndpoint: "https://api.deepseek.com/v1", apiMode: "chat_completions", }) @@ -556,6 +556,37 @@ describe("reasoning controls", () => { expect(body.reasoning).toBeUndefined() }) + it("maps DeepSeek V4 flash reasoning off to thinking disabled", () => { + const cfg = mkConfig({ + provider: "custom", + model: "deepseek-v4-flash", + customEndpoint: "https://api.deepseek.com/v1", + apiMode: "chat_completions", + }) + const body = getProviderConfig(cfg).buildBody( + [{ role: "user", content: "hi" }], + { reasoning: { mode: "off" } }, + ) as Record + + expect(body.thinking).toEqual({ type: "disabled" }) + }) + + it("sends thinking=enabled for flash when reasoning mode is max", () => { + const cfg = mkConfig({ + provider: "custom", + model: "deepseek-v4-flash", + customEndpoint: "https://api.deepseek.com/v1", + apiMode: "chat_completions", + }) + const body = getProviderConfig(cfg).buildBody( + [{ role: "user", content: "hi" }], + { reasoning: { mode: "max" } }, + ) as Record + + expect(body.thinking).toEqual({ type: "enabled" }) + expect(body.reasoning_effort).toBe("max") + }) + it("maps DeepSeek V4 high reasoning to thinking enabled plus reasoning_effort", () => { const cfg = mkConfig({ provider: "custom", diff --git a/src/lib/llm-providers.ts b/src/lib/llm-providers.ts old mode 100644 new mode 100755 diff --git a/src/lib/markdown-image-resolver.test.ts b/src/lib/markdown-image-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/markdown-image-resolver.ts b/src/lib/markdown-image-resolver.ts old mode 100644 new mode 100755 diff --git a/src/lib/mineru.test.ts b/src/lib/mineru.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/mineru.ts b/src/lib/mineru.ts old mode 100644 new mode 100755 diff --git a/src/lib/optimize-research-topic.test.ts b/src/lib/optimize-research-topic.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/optimize-research-topic.ts b/src/lib/optimize-research-topic.ts old mode 100644 new mode 100755 diff --git a/src/lib/output-language-options.ts b/src/lib/output-language-options.ts old mode 100644 new mode 100755 diff --git a/src/lib/output-language.test.ts b/src/lib/output-language.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/output-language.ts b/src/lib/output-language.ts old mode 100644 new mode 100755 diff --git a/src/lib/overview-blocks.test.ts b/src/lib/overview-blocks.test.ts new file mode 100644 index 000000000..d7e724338 --- /dev/null +++ b/src/lib/overview-blocks.test.ts @@ -0,0 +1,270 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { + chunkOverviewBySections, + parseOverviewPrematchOutput, + buildOverviewPrematchPrompt, + assembleReducedOverview, + runOverviewPrematchParallel, + parseOverviewBlocks, + appendOverviewContent, + createInitialOverview, + type ParsedOverviewBlock, +} from "./overview-blocks" +import { streamChat } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" + +vi.mock("@/lib/llm-client", () => ({ + streamChat: vi.fn(), +})) + +const mockStreamChat = vi.mocked(streamChat) +const mockLlmConfig = { model: "test", provider: "openai" } as unknown as LlmConfig + +function mockStreamResponse(text: string) { + return async (_config: unknown, _messages: unknown, callbacks: { + onToken: (t: string) => void; onDone: () => void; onError: (e: Error) => void + }) => { + for (const char of text) callbacks.onToken(char) + callbacks.onDone() + } +} + +beforeEach(() => { + mockStreamChat.mockReset() +}) + +describe("chunkOverviewBySections", () => { + it("returns empty array for empty overview", () => { + expect(chunkOverviewBySections("", 2000)).toEqual([]) + }) + + it("returns single chunk when overview is small", () => { + const overview = [ + "# Overview", + "", + "## 操作系统", + "操作系统是管理硬件资源的软件。", + "", + "## 网络", + "计算机网络是互联互通的系统。", + ].join("\n") + const chunks = chunkOverviewBySections(overview, 2000) + expect(chunks).toHaveLength(1) + expect(chunks[0]).toContain("[1]") + expect(chunks[0]).toContain("操作系统是管理硬件资源的软件。") + expect(chunks[0]).toContain("[2]") + expect(chunks[0]).toContain("计算机网络是互联互通的系统。") + }) + + it("splits into multiple chunks when overview exceeds maxChunkChars", () => { + const sections: string[] = ["# Overview", ""] + for (let i = 0; i < 5; i++) { + sections.push(`## Section ${i}`) + sections.push("x".repeat(600)) + sections.push("") + } + const overview = sections.join("\n") + const chunks = chunkOverviewBySections(overview, 1000) + expect(chunks.length).toBeGreaterThan(1) + }) + + it("handles overview with no ## headings (single chunk)", () => { + const overview = "Just some prose without headings." + const chunks = chunkOverviewBySections(overview, 2000) + expect(chunks).toHaveLength(1) + expect(chunks[0]).toContain("[1]") + expect(chunks[0]).toContain("Just some prose without headings.") + }) +}) + +describe("parseOverviewPrematchOutput", () => { + it("parses bracket paragraph numbers", () => { + expect(parseOverviewPrematchOutput("[2, 5, 12]")).toEqual([2, 5, 12]) + }) + it("returns empty for none", () => { + expect(parseOverviewPrematchOutput("none")).toEqual([]) + }) + it("returns empty for empty string", () => { + expect(parseOverviewPrematchOutput("")).toEqual([]) + }) + it("handles extra whitespace", () => { + expect(parseOverviewPrematchOutput("[ 2 , 5 ]")).toEqual([2, 5]) + }) +}) + +describe("buildOverviewPrematchPrompt", () => { + it("includes source content and chunk", () => { + const prompt = buildOverviewPrematchPrompt("source text", "chunk text") + expect(prompt).toContain("source text") + expect(prompt).toContain("chunk text") + expect(prompt).toContain("paragraph numbers") + }) +}) + +describe("assembleReducedOverview", () => { + it("returns empty string for no matches", () => { + expect(assembleReducedOverview("any", [])).toBe("") + }) + it("assembles only matched paragraphs", () => { + const overview = [ + "# Overview", "", + "## 操作系统", "OS content", "", + "## 网络", "Network content", "", + "## 数据库", "DB content", + ].join("\n") + // Paragraphs: [1] preamble, [2] 操作系统, [3] 网络, [4] 数据库 + const result = assembleReducedOverview(overview, [2, 4]) + expect(result).toContain("## 操作系统") + expect(result).toContain("OS content") + expect(result).toContain("## 数据库") + expect(result).toContain("DB content") + expect(result).not.toContain("## 网络") + expect(result).not.toContain("Network content") + }) +}) + +describe("runOverviewPrematchParallel", () => { + it("returns empty array for empty chunks", async () => { + const result = await runOverviewPrematchParallel([], "source", mockLlmConfig, undefined) + expect(result).toEqual([]) + }) + it("collects matched paragraph numbers from all chunks", async () => { + mockStreamChat + .mockImplementationOnce(mockStreamResponse("[1, 3]")) + .mockImplementationOnce(mockStreamResponse("[5]")) + const result = await runOverviewPrematchParallel(["chunk1", "chunk2"], "source", mockLlmConfig, undefined) + expect(result.sort()).toEqual([1, 3, 5]) + }) + it("handles none response", async () => { + mockStreamChat.mockImplementationOnce(mockStreamResponse("none")) + const result = await runOverviewPrematchParallel(["chunk1"], "source", mockLlmConfig, undefined) + expect(result).toEqual([]) + }) + it("handles LLM error gracefully", async () => { + mockStreamChat.mockImplementationOnce(async (_c, _m, callbacks: { + onToken: (t: string) => void; onDone: () => void; onError: (e: Error) => void + }) => { + callbacks.onError(new Error("LLM failed")) + callbacks.onDone() + }) + const result = await runOverviewPrematchParallel(["chunk1"], "source", mockLlmConfig, undefined) + expect(result).toEqual([]) + }) +}) + +describe("parseOverviewBlocks", () => { + it("parses a single OVERVIEW block", () => { + const text = [ + "---OVERVIEW: 操作系统---", + "操作系统是管理硬件资源的软件。", + "", + "它负责进程调度、内存管理和文件系统。", + "---END OVERVIEW---", + ].join("\n") + const blocks = parseOverviewBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].section).toBe("操作系统") + expect(blocks[0].content).toContain("操作系统是管理硬件资源的软件。") + }) + + it("parses multiple OVERVIEW blocks", () => { + const text = [ + "---OVERVIEW: 操作系统---", + "OS content", + "---END OVERVIEW---", + "", + "---OVERVIEW: 网络---", + "Network content", + "---END OVERVIEW---", + ].join("\n") + const blocks = parseOverviewBlocks(text) + expect(blocks).toHaveLength(2) + expect(blocks[0].section).toBe("操作系统") + expect(blocks[1].section).toBe("网络") + }) + + it("returns empty array when no OVERVIEW blocks", () => { + expect(parseOverviewBlocks("just some text")).toEqual([]) + expect(parseOverviewBlocks("")).toEqual([]) + }) + + it("handles extra whitespace in section name", () => { + const text = "---OVERVIEW: 操作系统 ---\ncontent\n---END OVERVIEW---" + const blocks = parseOverviewBlocks(text) + expect(blocks[0].section).toBe("操作系统") + }) +}) + +describe("appendOverviewContent", () => { + it("appends to existing section", () => { + const existing = [ + "# Overview", "", + "## 操作系统", "OS original content", "", + "## 网络", "Network content", + ].join("\n") + const blocks: ParsedOverviewBlock[] = [ + { section: "操作系统", content: "New OS paragraph." }, + ] + const result = appendOverviewContent(existing, blocks) + expect(result).toContain("## 操作系统") + expect(result).toContain("OS original content") + expect(result).toContain("New OS paragraph.") + expect(result.indexOf("OS original content")).toBeLessThan(result.indexOf("New OS paragraph.")) + expect(result).toContain("## 网络") + expect(result).toContain("Network content") + }) + + it("creates new section at end when section does not exist", () => { + const existing = ["# Overview", "", "## 操作系统", "OS content"].join("\n") + const blocks: ParsedOverviewBlock[] = [ + { section: "数据库", content: "DB content." }, + ] + const result = appendOverviewContent(existing, blocks) + expect(result).toContain("## 数据库") + expect(result).toContain("DB content.") + expect(result.indexOf("## 操作系统")).toBeLessThan(result.indexOf("## 数据库")) + }) + + it("handles multiple blocks", () => { + const existing = "# Overview\n\n## 操作系统\nOS content" + const blocks: ParsedOverviewBlock[] = [ + { section: "操作系统", content: "OS new." }, + { section: "网络", content: "Network new." }, + ] + const result = appendOverviewContent(existing, blocks) + expect(result).toContain("OS new.") + expect(result).toContain("## 网络") + expect(result).toContain("Network new.") + }) + + it("returns original when no blocks", () => { + const existing = "# Overview\n\n## A\ncontent" + expect(appendOverviewContent(existing, [])).toBe(existing) + }) +}) + +describe("createInitialOverview", () => { + it("creates overview with frontmatter and sections", () => { + const blocks: ParsedOverviewBlock[] = [ + { section: "操作系统", content: "OS content." }, + ] + const result = createInitialOverview(blocks, "2026-06-26") + expect(result).toContain("type: overview") + expect(result).toContain("---") + expect(result).toContain("# Overview") + expect(result).toContain("## 操作系统") + expect(result).toContain("OS content.") + expect(result).toContain("created: 2026-06-26") + expect(result).toContain("updated: 2026-06-26") + }) + + it("handles multiple sections", () => { + const blocks: ParsedOverviewBlock[] = [ + { section: "A", content: "Content A." }, + { section: "B", content: "Content B." }, + ] + const result = createInitialOverview(blocks, "2026-06-26") + expect(result).toContain("## A") + expect(result).toContain("## B") + }) +}) diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts new file mode 100644 index 000000000..89824f206 --- /dev/null +++ b/src/lib/overview-blocks.ts @@ -0,0 +1,358 @@ +/** + * Overview section chunking — symmetric to index-chunker.ts but for prose overview.md. + * + * Splits an overview into `##` sections, numbers each with an [N] prefix for + * prematch reference, and groups sections into chunks that stay under + * maxChunkChars. Later tasks add prematch, OVERVIEW block parsing and + * incremental append on top of this file. + */ + +import { streamChat, isReasoningOnlyResponseError } from "@/lib/llm-client" +import { currentWikiDate } from "@/lib/ingest" +import type { LlmConfig } from "@/stores/wiki-store" + +interface OverviewSection { + heading: string | null + content: string +} + +interface OverviewParagraph { + sectionHeading: string | null + number: number + text: string +} + +function parseOverviewSections(overview: string): OverviewSection[] { + const lines = overview.split("\n") + const sections: OverviewSection[] = [] + let currentLines: string[] = [] + + for (const line of lines) { + if (/^##\s/.test(line)) { + if (currentLines.length > 0) { + sections.push({ heading: extractHeading(currentLines[0]), content: currentLines.join("\n") }) + } + currentLines = [line] + } else { + currentLines.push(line) + } + } + if (currentLines.length > 0) { + sections.push({ heading: extractHeading(currentLines[0]), content: currentLines.join("\n") }) + } + + return sections +} + +function extractHeading(line: string): string | null { + const match = line.match(/^##\s+(.+)$/) + return match ? match[1].trim() : null +} + +/** Parse overview into globally-numbered paragraphs (split by \n\n within ## sections). */ +function parseOverviewParagraphs(overview: string): OverviewParagraph[] { + const sections = parseOverviewSections(overview) + const paragraphs: OverviewParagraph[] = [] + let num = 1 + + for (const section of sections) { + // Split section body by one or more blank lines + let body = section.heading + ? section.content.replace(/^##\s.*$/m, "").trim() + : section.content.trim() + if (!body) continue + + const parts = body.split(/\n\n+/).map((p) => p.trim()).filter((p) => p.length > 0) + for (const part of parts) { + paragraphs.push({ sectionHeading: section.heading, number: num++, text: part }) + } + } + + return paragraphs +} + +/** + * Chunk overview paragraphs into groups, numbered globally for prematch. + * Each chunk is prefixed with paragraph numbers for LLM reference. + */ +export function chunkOverviewBySections(overview: string, maxChunkChars: number): string[] { + if (!overview.trim()) return [] + + const paragraphs = parseOverviewParagraphs(overview) + if (paragraphs.length === 0) return [] + if (paragraphs.length === 1) return [`[${paragraphs[0].number}] ${paragraphs[0].text}`] + + const chunks: string[] = [] + let current = "" + + for (const p of paragraphs) { + const line = `[${p.number}] ${p.text}` + if (current && current.length + line.length + 2 > maxChunkChars) { + chunks.push(current) + current = line + } else { + current = current ? `${current}\n\n${line}` : line + } + } + if (current) chunks.push(current) + return chunks +} + +/** + * Parse prematch LLM output into paragraph numbers. + * Tolerant: handles [2, 5, 12], none, surrounding text. + * Returns deduplicated paragraph numbers, or empty array if none. + */ +export function parseOverviewPrematchOutput(output: string): number[] { + const trimmed = output.trim() + if (!trimmed) return [] + + const lower = trimmed.toLowerCase() + if (lower === "none" || trimmed === "无") return [] + + // Try to extract bracketed numbers first: [2, 5, 12] + const bracketMatch = trimmed.match(/\[([^\]]+)\]/) + const source = bracketMatch ? bracketMatch[1] : trimmed + + const numbers = source + .split(",") + .map((s) => parseInt(s.trim(), 10)) + .filter((n) => Number.isFinite(n) && n > 0) + + return [...new Set(numbers)] +} + +/** + * Build the system prompt for an overview pre-match LLM call. + * The LLM reads the source document and a chunk of overview paragraphs, + * then outputs matching paragraph numbers. + */ +export function buildOverviewPrematchPrompt(sourceContent: string, chunk: string): string { + return [ + "You are a relevance matcher. Read the source document and determine", + "which overview paragraphs would need to be UPDATED based on it.", + "", + "## Source Document", + sourceContent, + "", + "## Overview Paragraphs", + "Below is a chunk of the wiki overview. Each numbered item is a paragraph.", + "For each paragraph, determine whether the source adds NEW information that", + "would meaningfully change or extend it. Do NOT match a paragraph if it", + "already covers the topic adequately and the source only restates the same", + "facts — re-ingesting the same topic does not require an update.", + "", + chunk, + "", + "## Output Format (STRICT)", + "", + "Output ONLY matching paragraph numbers in bracket format: [2, 5, 12]", + "If no paragraphs match, output exactly: none", + "", + "Do not output explanations, reasoning, or any other text.", + ].join("\n") +} + +/** + * Assemble a reduced overview from the original overview.md and matched paragraph numbers. + * Only includes matched paragraphs, grouped under their ## section headings. + */ +export function assembleReducedOverview(overview: string, matchedParagraphs: number[]): string { + if (matchedParagraphs.length === 0) return "" + + const matchSet = new Set(matchedParagraphs) + const paragraphs = parseOverviewParagraphs(overview) + const grouped = new Map() + + for (const p of paragraphs) { + if (matchSet.has(p.number)) { + const key = p.sectionHeading + if (!grouped.has(key)) grouped.set(key, []) + grouped.get(key)!.push(p.text) + } + } + + if (grouped.size === 0) return "" + + const lines: string[] = [] + for (const [heading, texts] of grouped) { + if (heading) lines.push(`## ${heading}`) + for (const t of texts) lines.push(t) + lines.push("") + } + + return lines.join("\n").trim() +} + +const OVERVIEW_PREMATCH_CONCURRENCY = 8 + +/** + * Run pre-match LLM calls in parallel across all overview chunks. + * Returns the union of all matched section names. + * Failed chunks are logged and skipped (treated as 0 matches). + */ +export async function runOverviewPrematchParallel( + chunks: string[], + sourceContent: string, + llmConfig: LlmConfig, + signal: AbortSignal | undefined, +): Promise { + if (chunks.length === 0) return [] + + const results: number[][] = [] + + // Process in batches of OVERVIEW_PREMATCH_CONCURRENCY + for (let i = 0; i < chunks.length; i += OVERVIEW_PREMATCH_CONCURRENCY) { + if (signal?.aborted) break + const batch = chunks.slice(i, i + OVERVIEW_PREMATCH_CONCURRENCY) + + const batchResults = await Promise.all( + batch.map(async (chunk) => { + let output = "" + let hadError = false + + try { + await streamChat( + llmConfig, + [ + { role: "system", content: buildOverviewPrematchPrompt(sourceContent, chunk) }, + { role: "user", content: "Output matching paragraph numbers for the chunk above." }, + ], + { + onToken: (token: string) => { output += token }, + onDone: () => {}, + onError: (err: Error) => { + hadError = true + console.warn(`[overview-prematch] chunk failed: ${err.message}`) + }, + }, + signal, + { temperature: 0.1, reasoning: { mode: "high" } }, + ) + } catch (err) { + if (isReasoningOnlyResponseError(err)) return [] + hadError = true + console.warn(`[overview-prematch] chunk threw: ${err instanceof Error ? err.message : String(err)}`) + } + + if (hadError) return [] + return parseOverviewPrematchOutput(output) + }), + ) + + results.push(...batchResults) + } + + // Flatten and deduplicate + return [...new Set(results.flat())] +} + +export interface ParsedOverviewBlock { + section: string + content: string +} + +const OVERVIEW_BLOCK_REGEX = /---OVERVIEW:\s*(.+?)\s*---\n([\s\S]*?)---END OVERVIEW---/g + +/** + * Parse ---OVERVIEW: SectionName--- blocks from LLM generation output. + * Symmetric to parseIndexBlocks in index-chunker.ts. + */ +export function parseOverviewBlocks(text: string): ParsedOverviewBlock[] { + const normalized = text.replace(/\r\n/g, "\n") + const blocks: ParsedOverviewBlock[] = [] + for (const match of normalized.matchAll(OVERVIEW_BLOCK_REGEX)) { + const section = match[1].trim() + const content = match[2].trim() + blocks.push({ section, content }) + } + return blocks +} + +/** + * Programmatically append parsed OVERVIEW block content to existing overview.md. + * - Appends to existing `## Section` (before the next ## or EOF) + * - Creates a new `## Section` at the end if it does not exist + * - Updates the `updated:` field in frontmatter to today's date (when present) + * - Returns the original content unchanged when no blocks are provided + * Symmetric to appendIndexEntries in index-chunker.ts. + */ +export function appendOverviewContent( + overviewContent: string, + blocks: ParsedOverviewBlock[], +): string { + if (blocks.length === 0) return overviewContent + + const lines = overviewContent.split("\n") + const result = [...lines] + + for (const block of blocks) { + if (!block.content.trim()) continue + const sectionHeader = `## ${block.section}` + + let sectionStartIdx = -1 + for (let i = 0; i < result.length; i++) { + if (result[i].trim() === sectionHeader) { + sectionStartIdx = i + break + } + } + + if (sectionStartIdx >= 0) { + let insertIdx = result.length + for (let i = sectionStartIdx + 1; i < result.length; i++) { + if (/^##\s/.test(result[i].trim())) { + insertIdx = i + break + } + } + result.splice(insertIdx, 0, "", block.content) + } else { + result.push("", sectionHeader, block.content) + } + } + + // Update frontmatter `updated:` date if present + if (result[0]?.trim() === "---") { + for (let i = 1; i < result.length; i++) { + if (result[i].trim() === "---") break + if (/^updated:/.test(result[i])) { + result[i] = `updated: ${currentWikiDate()}` + break + } + } + } + + return result.join("\n") +} + +/** + * Create the initial overview.md with frontmatter for the first ingest. + * Symmetric to the index.md bootstrap, but for prose overview sections. + */ +export function createInitialOverview( + blocks: ParsedOverviewBlock[], + date?: string, +): string { + const d = date ?? currentWikiDate() + const lines: string[] = [ + "---", + "type: overview", + 'title: "Overview"', + `created: ${d}`, + `updated: ${d}`, + "tags: []", + "related: []", + "---", + "", + "# Overview", + ] + + for (const block of blocks) { + lines.push("") + lines.push(`## ${block.section}`) + lines.push(block.content) + } + + return lines.join("\n") +} diff --git a/src/lib/page-merge.test.ts b/src/lib/page-merge.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/page-merge.ts b/src/lib/page-merge.ts old mode 100644 new mode 100755 diff --git a/src/lib/path-utils.property.test.ts b/src/lib/path-utils.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/path-utils.test.ts b/src/lib/path-utils.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/path-utils.ts b/src/lib/path-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/persist.integration.test.ts b/src/lib/persist.integration.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/persist.ts b/src/lib/persist.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-file-sync.test.ts b/src/lib/project-file-sync.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-file-sync.ts b/src/lib/project-file-sync.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-identity.ts b/src/lib/project-identity.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-mutex.test.ts b/src/lib/project-mutex.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-mutex.ts b/src/lib/project-mutex.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-store.test.ts b/src/lib/project-store.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-store.ts b/src/lib/project-store.ts old mode 100644 new mode 100755 diff --git a/src/lib/proxy-config.test.ts b/src/lib/proxy-config.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/proxy-config.ts b/src/lib/proxy-config.ts old mode 100644 new mode 100755 diff --git a/src/lib/raw-source-resolver.test.ts b/src/lib/raw-source-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/raw-source-resolver.ts b/src/lib/raw-source-resolver.ts old mode 100644 new mode 100755 diff --git a/src/lib/reasoning-detector.test.ts b/src/lib/reasoning-detector.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/reasoning-detector.ts b/src/lib/reasoning-detector.ts old mode 100644 new mode 100755 diff --git a/src/lib/reset-project-state.test.ts b/src/lib/reset-project-state.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/reset-project-state.ts b/src/lib/reset-project-state.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-create-page.test.ts b/src/lib/review-create-page.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-create-page.ts b/src/lib/review-create-page.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-utils.property.test.ts b/src/lib/review-utils.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-utils.test.ts b/src/lib/review-utils.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-utils.ts b/src/lib/review-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/scheduled-import.test.ts b/src/lib/scheduled-import.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/scheduled-import.ts b/src/lib/scheduled-import.ts old mode 100644 new mode 100755 diff --git a/src/lib/search-rrf.test.ts b/src/lib/search-rrf.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/search.scenarios.test.ts b/src/lib/search.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/search.ts b/src/lib/search.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-delete-decision.test.ts b/src/lib/source-delete-decision.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-delete-decision.ts b/src/lib/source-delete-decision.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-identity.test.ts b/src/lib/source-identity.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-identity.ts b/src/lib/source-identity.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-lifecycle-delete.test.ts b/src/lib/source-lifecycle-delete.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-lifecycle.test.ts b/src/lib/source-lifecycle.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-watch-config.test.ts b/src/lib/source-watch-config.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-watch-config.ts b/src/lib/source-watch-config.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-watch-defaults.json b/src/lib/source-watch-defaults.json old mode 100644 new mode 100755 diff --git a/src/lib/sources-merge.test.ts b/src/lib/sources-merge.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sources-merge.ts b/src/lib/sources-merge.ts old mode 100644 new mode 100755 diff --git a/src/lib/sources-tree-delete.test.ts b/src/lib/sources-tree-delete.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sources-tree-delete.ts b/src/lib/sources-tree-delete.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-chained.real-llm.test.ts b/src/lib/sweep-chained.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.property.test.ts b/src/lib/sweep-reviews.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.race.test.ts b/src/lib/sweep-reviews.race.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.real-llm.test.ts b/src/lib/sweep-reviews.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.scenarios.test.ts b/src/lib/sweep-reviews.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.test.ts b/src/lib/sweep-reviews.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.ts b/src/lib/sweep-reviews.ts old mode 100644 new mode 100755 diff --git a/src/lib/tauri-fetch.test.ts b/src/lib/tauri-fetch.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/tauri-fetch.ts b/src/lib/tauri-fetch.ts old mode 100644 new mode 100755 diff --git a/src/lib/templates.ts b/src/lib/templates.ts old mode 100644 new mode 100755 diff --git a/src/lib/text-chunker.test.ts b/src/lib/text-chunker.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/text-chunker.ts b/src/lib/text-chunker.ts old mode 100644 new mode 100755 diff --git a/src/lib/theme.ts b/src/lib/theme.ts old mode 100644 new mode 100755 diff --git a/src/lib/update-check.test.ts b/src/lib/update-check.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/update-check.ts b/src/lib/update-check.ts old mode 100644 new mode 100755 diff --git a/src/lib/utils.ts b/src/lib/utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision-caption.real-llm.test.ts b/src/lib/vision-caption.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision-caption.test.ts b/src/lib/vision-caption.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision-caption.ts b/src/lib/vision-caption.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision.real-llm.test.ts b/src/lib/vision.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/web-search.test.ts b/src/lib/web-search.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/web-search.ts b/src/lib/web-search.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-cleanup.test.ts b/src/lib/wiki-cleanup.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-cleanup.ts b/src/lib/wiki-cleanup.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-filename.test.ts b/src/lib/wiki-filename.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-filename.ts b/src/lib/wiki-filename.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-graph.ts b/src/lib/wiki-graph.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-delete.test.ts b/src/lib/wiki-page-delete.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-delete.ts b/src/lib/wiki-page-delete.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-resolver.test.ts b/src/lib/wiki-page-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-resolver.ts b/src/lib/wiki-page-resolver.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-types.test.ts b/src/lib/wiki-page-types.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-types.ts b/src/lib/wiki-page-types.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-schema.test.ts b/src/lib/wiki-schema.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-schema.ts b/src/lib/wiki-schema.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-type-style.test.ts b/src/lib/wiki-type-style.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-type-style.ts b/src/lib/wiki-type-style.ts old mode 100644 new mode 100755 diff --git a/src/lib/wikilink-transform.test.ts b/src/lib/wikilink-transform.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wikilink-transform.ts b/src/lib/wikilink-transform.ts old mode 100644 new mode 100755 diff --git a/src/main.tsx b/src/main.tsx old mode 100644 new mode 100755 diff --git a/src/stores/activity-store.ts b/src/stores/activity-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/chat-messages-to-llm.test.ts b/src/stores/chat-messages-to-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/chat-store.ts b/src/stores/chat-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/file-sync-store.ts b/src/stores/file-sync-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/lint-store.test.ts b/src/stores/lint-store.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/lint-store.ts b/src/stores/lint-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/research-store.ts b/src/stores/research-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/review-store.property.test.ts b/src/stores/review-store.property.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/review-store.test.ts b/src/stores/review-store.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/review-store.ts b/src/stores/review-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/update-store.ts b/src/stores/update-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/wiki-store.test.ts b/src/stores/wiki-store.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/zoom-store.ts b/src/stores/zoom-store.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/deferred.ts b/src/test-helpers/deferred.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/fs-temp.ts b/src/test-helpers/fs-temp.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/load-test-env.ts b/src/test-helpers/load-test-env.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/mock-stream-chat.ts b/src/test-helpers/mock-stream-chat.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/real-content.ts b/src/test-helpers/real-content.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/enrich-scenarios.ts b/src/test-helpers/scenarios/enrich-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/ingest-scenarios.ts b/src/test-helpers/scenarios/ingest-scenarios.ts old mode 100644 new mode 100755 index 2966d26bf..90df89c16 --- a/src/test-helpers/scenarios/ingest-scenarios.ts +++ b/src/test-helpers/scenarios/ingest-scenarios.ts @@ -2,13 +2,23 @@ import type { IngestScenario } from "./types" /** * Ingest scenarios drive autoIngest end-to-end. Two LLM responses per - * scenario (stage 1 analysis, stage 2 generation with FILE + REVIEW blocks). + * scenario (stage 1 analysis, stage 2 generation with FILE + INDEX + + * REVIEW blocks). * * FILE block format (what stage 2 must emit to write a wiki file): * ---FILE: wiki/path/to/page.md--- * (file content, usually with YAML frontmatter) * ---END FILE--- * + * INDEX block format (what stage 2 emits to append new entries to + * wiki/index.md instead of regenerating the whole file): + * ---INDEX: CategoryName--- + * slug — Short description + * ---END INDEX--- + * INDEX blocks appear AFTER all FILE blocks and BEFORE any REVIEW blocks. + * Only NEW entries are listed (pre-existing index entries are preserved + * by the append path, never re-emitted). + * * REVIEW block format (what stage 2 emits to inject a review item): * ---REVIEW: missing-page | Short title--- * Description. @@ -100,11 +110,16 @@ export const ingestScenarios: IngestScenario[] = [ "", "Paper introducing [[Rotary Position Embedding]].", "---END FILE---", + "", + "---INDEX: Concepts---", + "rope — Rotary Position Embedding for variable-length contexts", + "---END INDEX---", ].join("\n"), expected: { writtenPaths: [ "wiki/concepts/rope.md", "wiki/sources/rope-paper.md", + "wiki/index.md", ], fileContains: { "wiki/concepts/rope.md": [ @@ -112,6 +127,7 @@ export const ingestScenarios: IngestScenario[] = [ "[[attention]]", ], "wiki/sources/rope-paper.md": ["rope-paper.md"], + "wiki/index.md": ["[[attention]]", "[[rope]]"], }, reviewsCreated: [], }, @@ -146,6 +162,10 @@ export const ingestScenarios: IngestScenario[] = [ "FlashAttention is mentioned here.", "---END FILE---", "", + "---INDEX: Sources---", + "flash-attention — FlashAttention source summary", + "---END INDEX---", + "", "---REVIEW: missing-page | FlashAttention---", "The source introduces FlashAttention but no dedicated page exists.", "OPTIONS: Create page | Skip", @@ -157,7 +177,10 @@ export const ingestScenarios: IngestScenario[] = [ "---END REVIEW---", ].join("\n"), expected: { - writtenPaths: ["wiki/sources/flash-attention.md"], + writtenPaths: ["wiki/sources/flash-attention.md", "wiki/index.md"], + fileContains: { + "wiki/index.md": ["[[flash-attention]]"], + }, reviewsCreated: [ { type: "missing-page", titleContains: "FlashAttention" }, { type: "suggestion", titleContains: "IO-aware" }, @@ -205,14 +228,20 @@ export const ingestScenarios: IngestScenario[] = [ "", "Source for multi-head [[attention]].", "---END FILE---", + "", + "---INDEX: Concepts---", + "multi-head-attention — Parallel multi-head attention variant", + "---END INDEX---", ].join("\n"), expected: { writtenPaths: [ "wiki/concepts/multi-head-attention.md", "wiki/sources/multi-head.md", + "wiki/index.md", ], fileContains: { "wiki/concepts/multi-head-attention.md": ["[[attention]]"], + "wiki/index.md": ["[[attention]]", "[[multi-head-attention]]"], }, }, }, @@ -253,17 +282,23 @@ export const ingestScenarios: IngestScenario[] = [ "", "关于 [[Transformer]] 的综述。", "---END FILE---", + "", + "---INDEX: Concepts---", + "transformer — 基于注意力机制的神经网络架构", + "---END INDEX---", ].join("\n"), expected: { writtenPaths: [ "wiki/concepts/transformer.md", "wiki/sources/transformer-survey.md", + "wiki/index.md", ], fileContains: { "wiki/concepts/transformer.md": [ "title: Transformer", "[[注意力机制]]", ], + "wiki/index.md": ["[[注意力机制]]", "[[transformer]]"], }, }, }, diff --git a/src/test-helpers/scenarios/lint-scenarios.ts b/src/test-helpers/scenarios/lint-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/materialize.ts b/src/test-helpers/scenarios/materialize.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/search-scenarios.ts b/src/test-helpers/scenarios/search-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/sweep-scenarios.ts b/src/test-helpers/scenarios/sweep-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/types.ts b/src/test-helpers/scenarios/types.ts old mode 100644 new mode 100755 diff --git a/src/types/wiki.ts b/src/types/wiki.ts old mode 100644 new mode 100755 diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts old mode 100644 new mode 100755 diff --git a/tsconfig.app.json b/tsconfig.app.json old mode 100644 new mode 100755 diff --git a/tsconfig.json b/tsconfig.json old mode 100644 new mode 100755 diff --git a/tsconfig.node.json b/tsconfig.node.json old mode 100644 new mode 100755 diff --git a/vite.config.ts b/vite.config.ts old mode 100644 new mode 100755