From 6da46c9488667287fba3b2d5ad55fde2141d4773 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 10:35:37 +0800 Subject: [PATCH 01/29] feat: add chunkIndexByEntries for splitting index into fixed-size chunks --- src/lib/index-chunker.test.ts | 85 +++++++++++++++++++++++++++++++++++ src/lib/index-chunker.ts | 68 ++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100755 src/lib/index-chunker.test.ts create mode 100755 src/lib/index-chunker.ts diff --git a/src/lib/index-chunker.test.ts b/src/lib/index-chunker.test.ts new file mode 100755 index 000000000..9dc697eaf --- /dev/null +++ b/src/lib/index-chunker.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect } from "vitest" +import { chunkIndexByEntries } from "./index-chunker" + +describe("chunkIndexByEntries", () => { + it("returns empty array for empty index", () => { + expect(chunkIndexByEntries("", 50)).toEqual([]) + }) + + it("returns single chunk when entries <= chunkSize", () => { + const index = [ + "# Index", + "", + "## Concepts", + "- [[attention]] — Core mechanism", + "- [[transformer]] — Architecture", + ].join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(1) + expect(chunks[0]).toContain("[[attention]]") + expect(chunks[0]).toContain("[[transformer]]") + expect(chunks[0]).toContain("## Concepts") + }) + + it("splits into multiple chunks at exact boundary", () => { + const lines: string[] = ["# Index", "", "## Concepts"] + for (let i = 0; i < 100; i++) { + lines.push(`- [[page-${i}]] — desc ${i}`) + } + const index = lines.join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(2) + expect(chunks[0].match(/\[\d+\]/g)).toHaveLength(50) + expect(chunks[1].match(/\[\d+\]/g)).toHaveLength(50) + }) + + it("preserves category headers within chunks that span categories", () => { + const lines: string[] = ["# Index", "", "## Concepts"] + for (let i = 0; i < 40; i++) { + lines.push(`- [[concept-${i}]] — desc ${i}`) + } + lines.push("## Entities") + for (let i = 0; i < 20; i++) { + lines.push(`- [[entity-${i}]] — desc ${i}`) + } + const index = lines.join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(2) + expect(chunks[0]).toContain("## Concepts") + expect(chunks[0]).toContain("## Entities") + expect(chunks[1]).toContain("## Entities") + expect(chunks[1]).not.toContain("## Concepts") + }) + + it("numbers entries sequentially across chunks starting from 1", () => { + const lines: string[] = ["# Index", "", "## Concepts"] + for (let i = 0; i < 120; i++) { + lines.push(`- [[page-${i}]] — desc ${i}`) + } + const index = lines.join("\n") + const chunks = chunkIndexByEntries(index, 50) + expect(chunks).toHaveLength(3) + expect(chunks[0]).toContain("[1]") + expect(chunks[0]).toContain("[50]") + expect(chunks[1]).toContain("[51]") + expect(chunks[2]).toContain("[101]") + expect(chunks[2]).toContain("[120]") + }) + + it("skips non-entry lines (empty lines, H1) when counting", () => { + const index = [ + "# Index", + "", + "## Concepts", + "- [[a]] — desc a", + "", + "- [[b]] — desc b", + "", + "## Papers", + "(none yet)", + "- [[c]] — desc c", + ].join("\n") + const chunks = chunkIndexByEntries(index, 2) + expect(chunks).toHaveLength(2) + }) +}) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts new file mode 100755 index 000000000..9b3430fa8 --- /dev/null +++ b/src/lib/index-chunker.ts @@ -0,0 +1,68 @@ +/** + * Chunk an index.md string into fixed-size chunks by entry count. + * Each chunk preserves the ## Category headers for context. + * Entries are numbered sequentially with [N] prefix across all chunks. + */ + +interface IndexEntry { + category: string + text: string +} + +/** Parse index.md into structured entries with their category context. */ +function parseIndexEntries(index: string): { + entries: IndexEntry[] +} { + const lines = index.split("\n") + const entries: IndexEntry[] = [] + let currentCategory = "" + + for (const line of lines) { + const headerMatch = line.match(/^##\s+(.+)/) + if (headerMatch) { + currentCategory = headerMatch[1].trim() + continue + } + // Entry lines start with - [[ or * [[ + if (/^[-*]\s+\[\[/.test(line)) { + entries.push({ + category: currentCategory, + text: line, + }) + } + } + + return { entries } +} + +export function chunkIndexByEntries(index: string, chunkSize: number): string[] { + if (!index.trim()) return [] + + const { entries } = parseIndexEntries(index) + if (entries.length === 0) return [] + + const chunks: string[] = [] + let entryIdx = 0 + let globalNum = 1 + + while (entryIdx < entries.length) { + const chunkLines: string[] = [] + let currentCat = "" + const end = Math.min(entryIdx + chunkSize, entries.length) + + for (let i = entryIdx; i < end; i++) { + const entry = entries[i] + if (entry.category !== currentCat) { + currentCat = entry.category + chunkLines.push(`## ${currentCat}`) + } + chunkLines.push(`[${globalNum}] ${entry.text.replace(/^[-*]\s+/, "")}`) + globalNum++ + } + + chunks.push(chunkLines.join("\n")) + entryIdx = end + } + + return chunks +} From 26ab5d433d6c4a9b1e7634e38d6beccf1db6a25e Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 10:37:34 +0800 Subject: [PATCH 02/29] feat: add parsePrematchOutput for tolerant LLM output parsing --- src/lib/index-chunker.test.ts | 49 ++++++++++++++++++++++++++++++++++- src/lib/index-chunker.ts | 27 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/lib/index-chunker.test.ts b/src/lib/index-chunker.test.ts index 9dc697eaf..a3fc7593b 100755 --- a/src/lib/index-chunker.test.ts +++ b/src/lib/index-chunker.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest" -import { chunkIndexByEntries } from "./index-chunker" +import { chunkIndexByEntries, parsePrematchOutput } from "./index-chunker" describe("chunkIndexByEntries", () => { it("returns empty array for empty index", () => { @@ -83,3 +83,50 @@ describe("chunkIndexByEntries", () => { expect(chunks).toHaveLength(2) }) }) + +describe("parsePrematchOutput", () => { + it("parses bracket format with spaces", () => { + expect(parsePrematchOutput("[3, 12, 47]")).toEqual([3, 12, 47]) + }) + + it("parses bracket format without spaces", () => { + expect(parsePrematchOutput("[3,12,47]")).toEqual([3, 12, 47]) + }) + + it("parses comma-separated without brackets", () => { + expect(parsePrematchOutput("3, 12, 47")).toEqual([3, 12, 47]) + }) + + it("parses single number", () => { + expect(parsePrematchOutput("[5]")).toEqual([5]) + expect(parsePrematchOutput("5")).toEqual([5]) + }) + + it("returns empty for none variations", () => { + expect(parsePrematchOutput("none")).toEqual([]) + expect(parsePrematchOutput("None")).toEqual([]) + expect(parsePrematchOutput("NONE")).toEqual([]) + expect(parsePrematchOutput("无")).toEqual([]) + }) + + it("returns empty for empty or whitespace", () => { + expect(parsePrematchOutput("")).toEqual([]) + expect(parsePrematchOutput(" ")).toEqual([]) + }) + + it("tolerates surrounding explanation text", () => { + expect(parsePrematchOutput("Matching items: [3, 12, 47]")).toEqual([3, 12, 47]) + }) + + it("ignores non-numeric tokens", () => { + expect(parsePrematchOutput("[3, abc, 47]")).toEqual([3, 47]) + }) + + it("returns empty on complete parse failure", () => { + expect(parsePrematchOutput("I think none match")).toEqual([]) + }) + + it("deduplicates numbers", () => { + expect(parsePrematchOutput("[3, 3, 12]")).toEqual([3, 12]) + }) +}) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 9b3430fa8..854dba672 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -66,3 +66,30 @@ export function chunkIndexByEntries(index: string, chunkSize: number): string[] return chunks } + +/** + * Parse the pre-match LLM output into an array of matching entry numbers. + * Tolerant: handles [3, 12, 47], 3, 12, 47, none, 无, surrounding text. + * Returns deduplicated numbers, or empty array if no valid numbers found. + */ +export function parsePrematchOutput(output: string): number[] { + const trimmed = output.trim() + if (!trimmed) return [] + + const lower = trimmed.toLowerCase() + if (lower === "none" || trimmed === "无") return [] + + // Try to extract bracketed numbers first: [3, 12, 47] + const bracketMatch = trimmed.match(/\[([\d,\s]+)\]/) + const source = bracketMatch ? bracketMatch[1] : trimmed + + // Extract all integer tokens + const tokens = source.match(/\d+/g) + if (!tokens) return [] + + const numbers = tokens + .map((t) => parseInt(t, 10)) + .filter((n) => Number.isFinite(n) && n > 0) + + return [...new Set(numbers)] +} From ac12269976a762c4ceed0d8298ca9171a81886a9 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 10:40:55 +0800 Subject: [PATCH 03/29] feat: add assembleReducedIndex for building reduced index from matches --- src/lib/index-chunker.test.ts | 66 ++++++++++++++++++++++++++++++++++- src/lib/index-chunker.ts | 27 ++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/src/lib/index-chunker.test.ts b/src/lib/index-chunker.test.ts index a3fc7593b..6a834a5ef 100755 --- a/src/lib/index-chunker.test.ts +++ b/src/lib/index-chunker.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest" -import { chunkIndexByEntries, parsePrematchOutput } from "./index-chunker" +import { chunkIndexByEntries, parsePrematchOutput, assembleReducedIndex } from "./index-chunker" describe("chunkIndexByEntries", () => { it("returns empty array for empty index", () => { @@ -130,3 +130,67 @@ describe("parsePrematchOutput", () => { expect(parsePrematchOutput("[3, 3, 12]")).toEqual([3, 12]) }) }) + +describe("assembleReducedIndex", () => { + it("returns empty string for empty matches", () => { + const index = "## Concepts\n- [[a]] — desc a\n- [[b]] — desc b" + expect(assembleReducedIndex(index, [])).toBe("") + }) + + it("returns single matched entry with its category header", () => { + const index = [ + "# Index", "", + "## Concepts", + "- [[attention]] — Core mechanism", + "- [[transformer]] — Architecture", + ].join("\n") + const result = assembleReducedIndex(index, [1]) + expect(result).toContain("## Concepts") + expect(result).toContain("[[attention]]") + expect(result).not.toContain("[[transformer]]") + }) + + it("returns multiple matches preserving original order", () => { + const index = [ + "## Concepts", + "- [[a]] — desc a", + "- [[b]] — desc b", + "- [[c]] — desc c", + ].join("\n") + const result = assembleReducedIndex(index, [3, 1]) + expect(result.indexOf("[[a]]")).toBeLessThan(result.indexOf("[[c]]")) + }) + + it("deduplicates category headers", () => { + const index = [ + "## Concepts", + "- [[a]] — desc a", + "- [[b]] — desc b", + "## Entities", + "- [[c]] — desc c", + ].join("\n") + const result = assembleReducedIndex(index, [1, 2]) + const headerCount = (result.match(/## Concepts/g) ?? []).length + expect(headerCount).toBe(1) + }) + + it("handles matches from different categories", () => { + const index = [ + "## Concepts", + "- [[a]] — desc a", + "## Entities", + "- [[b]] — desc b", + ].join("\n") + const result = assembleReducedIndex(index, [1, 2]) + expect(result).toContain("## Concepts") + expect(result).toContain("## Entities") + expect(result).toContain("[[a]]") + expect(result).toContain("[[b]]") + }) + + it("ignores out-of-range numbers gracefully", () => { + const index = "## Concepts\n- [[a]] — desc a" + const result = assembleReducedIndex(index, [1, 999]) + expect(result).toContain("[[a]]") + }) +}) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 854dba672..3964abcfe 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -93,3 +93,30 @@ export function parsePrematchOutput(output: string): number[] { return [...new Set(numbers)] } + +/** + * Assemble a reduced index from the original index.md and matched entry numbers. + * Entry numbers are 1-based as assigned by chunkIndexByEntries. + * Preserves original index order and deduplicates category headers. + */ +export function assembleReducedIndex(index: string, matchedNumbers: number[]): string { + if (matchedNumbers.length === 0) return "" + + const { entries } = parseIndexEntries(index) + const matchSet = new Set(matchedNumbers) + const lines: string[] = [] + let currentCat = "" + + for (let i = 0; i < entries.length; i++) { + const entryNum = i + 1 + if (!matchSet.has(entryNum)) continue + + if (entries[i].category !== currentCat) { + currentCat = entries[i].category + lines.push(`## ${currentCat}`) + } + lines.push(entries[i].text) + } + + return lines.join("\n") +} From 2f7076411e8a25470e35084491751733babd1490 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 10:49:50 +0800 Subject: [PATCH 04/29] feat: add buildPrematchPrompt, parseIndexBlocks, and appendIndexEntries --- src/lib/index-chunker.test.ts | 155 +++++++++++++++++++++++++++++++++- src/lib/index-chunker.ts | 135 +++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 1 deletion(-) diff --git a/src/lib/index-chunker.test.ts b/src/lib/index-chunker.test.ts index 6a834a5ef..bc94415c1 100755 --- a/src/lib/index-chunker.test.ts +++ b/src/lib/index-chunker.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect } from "vitest" -import { chunkIndexByEntries, parsePrematchOutput, assembleReducedIndex } from "./index-chunker" +import { + chunkIndexByEntries, + parsePrematchOutput, + assembleReducedIndex, + buildPrematchPrompt, + parseIndexBlocks, + appendIndexEntries, + type ParsedIndexBlock, +} from "./index-chunker" describe("chunkIndexByEntries", () => { it("returns empty array for empty index", () => { @@ -194,3 +202,148 @@ describe("assembleReducedIndex", () => { expect(result).toContain("[[a]]") }) }) + +describe("buildPrematchPrompt", () => { + it("includes source document content", () => { + const prompt = buildPrematchPrompt("This is a source about transformers.", "## Concepts\n[1] attention — desc") + expect(prompt).toContain("This is a source about transformers.") + }) + + it("includes the index chunk", () => { + const chunk = "## Concepts\n[1] attention — desc\n[2] transformer — desc" + const prompt = buildPrematchPrompt("source content", chunk) + expect(prompt).toContain("[1] attention") + expect(prompt).toContain("[2] transformer") + }) + + it("includes strict output format instructions", () => { + const prompt = buildPrematchPrompt("src", "chunk") + expect(prompt).toContain("[3, 12, 47]") + expect(prompt).toContain("none") + expect(prompt).toContain("STRICT") + }) +}) + +describe("parseIndexBlocks", () => { + it("parses a single INDEX block", () => { + const text = [ + "---INDEX: Concepts---", + "rope — Rotary Position Embedding", + "flash-attention — IO-aware attention", + "---END INDEX---", + ].join("\n") + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].category).toBe("Concepts") + expect(blocks[0].entries).toEqual([ + "rope — Rotary Position Embedding", + "flash-attention — IO-aware attention", + ]) + }) + + it("parses multiple INDEX blocks", () => { + const text = [ + "---INDEX: Concepts---", + "rope — desc", + "---END INDEX---", + "", + "---INDEX: Entities---", + "openai — desc", + "---END INDEX---", + ].join("\n") + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(2) + expect(blocks[0].category).toBe("Concepts") + expect(blocks[1].category).toBe("Entities") + }) + + it("returns empty array when no INDEX blocks", () => { + expect(parseIndexBlocks("no blocks here")).toEqual([]) + expect(parseIndexBlocks("")).toEqual([]) + }) + + it("tolerates extra whitespace in markers", () => { + const text = "---INDEX: Concepts ---\nrope — desc\n---END INDEX---" + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].category).toBe("Concepts") + }) + + it("handles empty entries", () => { + const text = "---INDEX: Concepts---\n---END INDEX---" + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].entries).toEqual([]) + }) + + it("normalizes CRLF line endings", () => { + const text = "---INDEX: Concepts---\r\nrope — desc\r\n---END INDEX---" + const blocks = parseIndexBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].entries).toEqual(["rope — desc"]) + }) +}) + +describe("appendIndexEntries", () => { + it("appends entries to existing category", () => { + const index = [ + "# Index", "", + "## Concepts", + "- [[attention]] — Core mechanism", + "- [[transformer]] — Architecture", + ].join("\n") + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["rope — Rotary Position Embedding"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[attention]]") + expect(result).toContain("- [[transformer]]") + expect(result).toContain("- [[rope]] — Rotary Position Embedding") + expect(result.indexOf("[[transformer]]")).toBeLessThan(result.indexOf("rope")) + }) + + it("creates new category if it does not exist", () => { + const index = "## Concepts\n- [[a]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Entities", entries: ["openai — AI company"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("## Entities") + expect(result).toContain("- [[openai]] — AI company") + expect(result).toContain("## Concepts") + }) + + it("appends to multiple categories", () => { + const index = "## Concepts\n- [[a]] — desc\n## Entities\n- [[b]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["c — desc c"] }, + { category: "Entities", entries: ["d — desc d"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[c]] — desc c") + expect(result).toContain("- [[d]] — desc d") + }) + + it("handles entries that need slug format conversion", () => { + const index = "## Concepts\n- [[a]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["rope — desc"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[rope]] — desc") + }) + + it("preserves entries with no description", () => { + const index = "## Concepts\n- [[a]] — desc" + const blocks: ParsedIndexBlock[] = [ + { category: "Concepts", entries: ["rope"] }, + ] + const result = appendIndexEntries(index, blocks) + expect(result).toContain("- [[rope]]") + }) + + it("handles empty blocks array", () => { + const index = "## Concepts\n- [[a]] — desc" + expect(appendIndexEntries(index, [])).toBe(index) + }) +}) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 3964abcfe..ed5da4d2b 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -120,3 +120,138 @@ export function assembleReducedIndex(index: string, matchedNumbers: number[]): s return lines.join("\n") } + +/** + * Build the system prompt for a pre-match LLM call. + * The LLM reads the source document and a chunk of index entries, + * then outputs matching entry numbers. + */ +export function buildPrematchPrompt(sourceContent: string, chunk: string): string { + return [ + "You are a relevance matcher. Read the source document and determine", + "which wiki index entries are related to it.", + "", + "## Source Document", + sourceContent, + "", + "## Index Chunk", + "Below is a chunk of the wiki index. Each numbered item is a page entry.", + "For each item, determine whether it covers the same subject as any", + "entity or concept in the source document. A match means the page", + "is about the same entity, concept, method, or topic.", + "", + chunk, + "", + "## Output Format (STRICT)", + "", + "Output ONLY matching item numbers in bracket format: [3, 12, 47]", + "If no items match, output exactly: none", + "", + "Do not output explanations, reasoning, or any other text.", + ].join("\n") +} + +export interface ParsedIndexBlock { + category: string + entries: string[] +} + +const INDEX_BLOCK_REGEX = /---INDEX:\s*(.+?)\s*---\n([\s\S]*?)---END INDEX---/g + +/** + * Parse ---INDEX: Category--- blocks from LLM generation output. + * Pattern follows existing parseFileBlocks / parseReviewBlocks conventions. + */ +export function parseIndexBlocks(text: string): ParsedIndexBlock[] { + const normalized = text.replace(/\r\n/g, "\n") + const blocks: ParsedIndexBlock[] = [] + + for (const match of normalized.matchAll(INDEX_BLOCK_REGEX)) { + const category = match[1].trim() + const body = match[2].trim() + const entries = body + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + blocks.push({ category, entries }) + } + + return blocks +} + +/** + * Convert a raw entry line from an INDEX block into index.md format. + * Input: "rope — Rotary Position Embedding" or "rope" + * Output: "- [[rope]] — Rotary Position Embedding" or "- [[rope]]" + */ +function formatIndexEntry(rawEntry: string): string { + const trimmed = rawEntry.trim() + // Already has [[...]] format? Keep as-is (just ensure leading - [[) + if (/^\[\[/.test(trimmed)) { + return `- ${trimmed}` + } + // Split on em-dash or hyphen to separate slug from description + const dashMatch = trimmed.match(/^(.+?)\s+[—–-]\s+(.+)$/) + if (dashMatch) { + const slug = dashMatch[1].trim() + const desc = dashMatch[2].trim() + return `- [[${slug}]] — ${desc}` + } + // No description — just the slug + return `- [[${trimmed}]]` +} + +/** + * Programmatically append parsed INDEX block entries to existing index.md content. + * - Appends to existing category sections + * - Creates new category sections at the end if needed + */ +export function appendIndexEntries(indexContent: string, blocks: ParsedIndexBlock[]): string { + if (blocks.length === 0) return indexContent + + const lines = indexContent.split("\n") + const result = [...lines] + + for (const block of blocks) { + if (block.entries.length === 0) continue + + const categoryHeader = `## ${block.category}` + const formattedEntries = block.entries.map(formatIndexEntry) + + // Find the category section + let catStartIdx = -1 + for (let i = 0; i < result.length; i++) { + if (result[i].trim() === categoryHeader) { + catStartIdx = i + break + } + } + + if (catStartIdx >= 0) { + // Category exists — find the last entry line in this section + let insertIdx = catStartIdx + 1 + for (let i = catStartIdx + 1; i < result.length; i++) { + const line = result[i].trim() + if (/^##\s/.test(line)) break // Hit next category + if (/^[-*]\s+\[\[/.test(line) || /^\(none yet\)/.test(line)) { + insertIdx = i + 1 + } + } + // Remove "(none yet)" placeholder if present + const placeholderIdx = result.indexOf("(none yet)", catStartIdx) + if (placeholderIdx >= 0 && placeholderIdx < insertIdx) { + result.splice(placeholderIdx, 1) + insertIdx-- + } + // Insert entries + result.splice(insertIdx, 0, ...formattedEntries) + } else { + // Category doesn't exist — append at end + result.push("") + result.push(categoryHeader) + result.push(...formattedEntries) + } + } + + return result.join("\n") +} From c6c2f0f179bee2b65a11eb251ad8465290861bf2 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 10:55:31 +0800 Subject: [PATCH 05/29] feat: add runPrematchParallel for concurrent pre-match LLM calls --- src/lib/index-chunker.test.ts | 83 ++++++++++++++++++++++++++++++++++- src/lib/index-chunker.ts | 65 +++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/src/lib/index-chunker.test.ts b/src/lib/index-chunker.test.ts index bc94415c1..2160b625f 100755 --- a/src/lib/index-chunker.test.ts +++ b/src/lib/index-chunker.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest" +import { describe, it, expect, vi, beforeEach } from "vitest" import { chunkIndexByEntries, parsePrematchOutput, @@ -6,8 +6,17 @@ import { buildPrematchPrompt, parseIndexBlocks, appendIndexEntries, + runPrematchParallel, type ParsedIndexBlock, } from "./index-chunker" +import { streamChat } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" + +vi.mock("@/lib/llm-client", () => ({ + streamChat: vi.fn(), +})) + +const mockStreamChat = vi.mocked(streamChat) describe("chunkIndexByEntries", () => { it("returns empty array for empty index", () => { @@ -347,3 +356,75 @@ describe("appendIndexEntries", () => { expect(appendIndexEntries(index, [])).toBe(index) }) }) + +describe("runPrematchParallel", () => { + beforeEach(() => { + mockStreamChat.mockReset() + }) + + it("returns matched numbers from all chunks", async () => { + const chunks = ["## Concepts\n[1] attention — desc", "## Entities\n[51] openai — desc"] + // The prompt includes the chunk content, so we can differentiate by content + mockStreamChat.mockImplementation(async (_cfg, messages, callbacks) => { + const sys = String(messages[0]?.content ?? "") + if (sys.includes("openai")) { + callbacks?.onToken?.("[51]") + } else { + callbacks?.onToken?.("[1, 3]") + } + callbacks?.onDone?.() + }) + + const result = await runPrematchParallel( + chunks, + "source content", + {} as LlmConfig, + undefined, + ) + expect(result).toContain(1) + expect(result).toContain(3) + expect(result).toContain(51) + }) + + it("skips failed chunks and continues", async () => { + const chunks = ["chunk0-content", "chunk1-content"] + let callCount = 0 + mockStreamChat.mockImplementation(async (_cfg, _messages, callbacks) => { + callCount++ + if (callCount === 1) { + callbacks?.onError?.(new Error("network error")) + return + } + callbacks?.onToken?.("[42]") + callbacks?.onDone?.() + }) + + const result = await runPrematchParallel( + chunks, + "source content", + {} as LlmConfig, + undefined, + ) + expect(result).toEqual([42]) + }) + + it("returns empty when all chunks fail", async () => { + mockStreamChat.mockImplementation(async (_cfg, _messages, callbacks) => { + callbacks?.onError?.(new Error("total failure")) + }) + + const result = await runPrematchParallel( + ["c0", "c1"], + "source", + {} as LlmConfig, + undefined, + ) + expect(result).toEqual([]) + }) + + it("returns empty for empty chunks array", async () => { + const result = await runPrematchParallel([], "source", {} as LlmConfig, undefined) + expect(result).toEqual([]) + expect(mockStreamChat).not.toHaveBeenCalled() + }) +}) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index ed5da4d2b..83b1e34e9 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -4,6 +4,9 @@ * Entries are numbered sequentially with [N] prefix across all chunks. */ +import { streamChat } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" + interface IndexEntry { category: string text: string @@ -255,3 +258,65 @@ export function appendIndexEntries(indexContent: string, blocks: ParsedIndexBloc return result.join("\n") } + +const PREMATCH_CONCURRENCY = 8 + +/** + * Run pre-match LLM calls in parallel across all index chunks. + * Returns the union of all matched entry numbers. + * Failed chunks are logged and skipped (treated as 0 matches). + */ +export async function runPrematchParallel( + chunks: string[], + sourceContent: string, + llmConfig: LlmConfig, + signal: AbortSignal | undefined, +): Promise { + if (chunks.length === 0) return [] + + const results: number[][] = [] + + // Process in batches of PREMATCH_CONCURRENCY + for (let i = 0; i < chunks.length; i += PREMATCH_CONCURRENCY) { + if (signal?.aborted) break + const batch = chunks.slice(i, i + PREMATCH_CONCURRENCY) + + const batchResults = await Promise.all( + batch.map(async (chunk) => { + let output = "" + let hadError = false + + try { + await streamChat( + llmConfig, + [ + { role: "system", content: buildPrematchPrompt(sourceContent, chunk) }, + { role: "user", content: "Output matching item numbers for the chunk above." }, + ], + { + onToken: (token: string) => { output += token }, + onDone: () => {}, + onError: (err: Error) => { + hadError = true + console.warn(`[prematch] chunk failed: ${err.message}`) + }, + }, + signal, + { temperature: 0.1, max_tokens: 256 }, + ) + } catch (err) { + hadError = true + console.warn(`[prematch] chunk threw: ${err instanceof Error ? err.message : String(err)}`) + } + + if (hadError) return [] + return parsePrematchOutput(output) + }), + ) + + results.push(...batchResults) + } + + // Flatten and deduplicate + return [...new Set(results.flat())] +} From b281029b5a065ff69d3233d66af62dd5b9bdd48c Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 10:58:19 +0800 Subject: [PATCH 06/29] feat: replace index.md FILE output with INDEX blocks in generation prompt --- src/lib/ingest.ts | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 1a5c50a0e..0ba243b0c 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -2146,8 +2146,10 @@ export function buildGenerationPrompt( `1. A source summary page at **${summaryPath}** (MUST use this exact path)`, "2. Entity or schema-defined typed pages for key named things identified in the analysis. Prefer schema-defined directories when present; otherwise use wiki/entities/.", "3. Concept or schema-defined typed pages for key ideas, methods, techniques, and abstractions. Prefer schema-defined directories when present; otherwise use wiki/concepts/.", - "4. A log entry for wiki/log.md (just the new entry to append, format: ## [YYYY-MM-DD] ingest | Title)", - "Do not generate wiki/index.md or wiki/overview.md. The application maintains aggregate navigation separately so large wikis are never rewritten through model output.", + "4. INDEX blocks for new entries (see INDEX block format below) — do NOT output a complete wiki/index.md file", + "5. A log entry for wiki/log.md (just the new entry to append, format: ## [YYYY-MM-DD] ingest | Title)", + "6. OVERVIEW blocks for new sections (see OVERVIEW block format below) — do NOT output a complete wiki/overview.md file", + "Do not generate wiki/index.md or wiki/overview.md as complete files. The application maintains aggregate navigation separately so large wikis are never rewritten through model output.", "", "## Frontmatter Rules (CRITICAL — parser is strict)", "", @@ -2227,7 +2229,7 @@ export function buildGenerationPrompt( " SEARCH: automated technical debt detection AI generated code | software quality metrics LLM code generation | static analysis tools agentic software development", "", purpose ? `## Wiki Purpose\n${purpose}` : "", - index ? `## Current Wiki Index (preserve all existing entries, add new ones)\n${index}` : "", + index ? `## Current Wiki Index (for reference only — do NOT reproduce it)\n${index}` : "", overview ? `## Current Overview (update this to reflect the new source)\n${overview}` : "", "", // ── OUTPUT FORMAT MUST BE THE LAST SECTION — models weight recent instructions highest ── @@ -2252,6 +2254,17 @@ export function buildGenerationPrompt( "---END REVIEW---", "```", "", + "INDEX block template (for new index entries only):", + "```", + "---INDEX: CategoryName---", + "slug — one-line description", + "---END INDEX---", + "```", + "", + "Output one INDEX block per category that has new entries.", + "CategoryName must match an existing ## heading in the index.", + "Do NOT output ---FILE: wiki/index.md---. Use INDEX blocks instead.", + "", "## Output Requirements (STRICT — deviations will cause parse failure)", "", "1. The FIRST character of your response MUST be `-` (the opening of `---FILE:`).", @@ -2262,6 +2275,11 @@ export function buildGenerationPrompt( "6. Between blocks, use only blank lines — no prose.", "7. FILE block prose (body, explanations, descriptions, section text) must use the mandatory output language specified below. Preserve proper nouns, acronyms, model names, dataset names, tool/library names, code identifiers, URLs, file names, citation strings, paper titles, and technical terms with no widely-used localized equivalent in their standard original form, including in page names and section headings.", "", + "8. INDEX blocks appear AFTER all FILE blocks and BEFORE any REVIEW blocks.", + "9. Each INDEX block must specify a category name after ---INDEX:.", + "10. Each entry line in an INDEX block must start with a slug followed by \" — \" and a description.", + "11. Do NOT output ---FILE: wiki/index.md---. Use INDEX blocks instead.", + "", "If you start with anything other than `---FILE:`, the entire response will be discarded.", "", // Repeat the language directive at the very end so it wins the "most From 019077bf9998e8409672ebf507b122df58fe6c1e Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 11:02:36 +0800 Subject: [PATCH 07/29] feat: insert pre-match and INDEX block processing into autoIngest --- src/lib/ingest.ts | 125 +++++++++++++++++++++------------------------- 1 file changed, 58 insertions(+), 67 deletions(-) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 0ba243b0c..48fe5d0b6 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -42,6 +42,13 @@ import type { MultimodalConfig } from "@/stores/wiki-store" import { GENERATION_WIKI_TYPES } from "@/lib/wiki-page-types" import { computeContextBudget } from "@/lib/context-budget" import { refreshProjectFileTree } from "@/lib/project-file-tree-refresh" +import { + chunkIndexByEntries, + assembleReducedIndex, + runPrematchParallel, + parseIndexBlocks, + appendIndexEntries, +} from "./index-chunker" const LONG_SOURCE_MIN_BUDGET = 8_000 const LONG_SOURCE_MAX_SINGLE_PASS_BUDGET = 300_000 @@ -951,6 +958,34 @@ async function autoIngestImpl( } } + // ── Step 0.7: Pre-match index chunks ───────────────────── + // Split index into chunks, run parallel LLM calls to find + // entries relevant to this source, assemble a reduced index. + const CHUNK_SIZE = 50 + let reducedIndex = index + if (index.trim()) { + const chunks = chunkIndexByEntries(index, CHUNK_SIZE) + if (chunks.length > 1) { + activity.updateItem(activityId, { + detail: `Step 0.7: Pre-matching index (${chunks.length} chunks)...`, + }) + const matchedNumbers = await runPrematchParallel( + chunks, + sourceContext, + llmConfig, + signal, + ) + reducedIndex = assembleReducedIndex(index, matchedNumbers) + if (!reducedIndex) { + reducedIndex = "(no matching wiki entries found)" + } + console.log( + `[ingest:prematch] index reduced from ${index.length} to ${reducedIndex.length} chars ` + + `(${matchedNumbers.length} matches from ${chunks.length} chunks)`, + ) + } + } + // ── Step 1: Analysis ────────────────────────────────────────── // LLM reads the source and produces a structured analysis: // key entities, concepts, main arguments, connections to existing wiki, contradictions @@ -966,7 +1001,7 @@ async function autoIngestImpl( await streamChat( llmConfig, [ - { role: "system", content: buildAnalysisPrompt(purpose, index, sourceContext, schema) }, + { role: "system", content: buildAnalysisPrompt(purpose, reducedIndex, sourceContext, schema) }, { role: "user", content: `Analyze this source document:\n\n**File:** ${sourceIdentity}${folderContext ? `\n**Folder context:** ${folderContext}` : ""}\n\n---\n\n${sourceContext}` }, ], { @@ -998,7 +1033,7 @@ async function autoIngestImpl( await streamChat( llmConfig, [ - { role: "system", content: buildGenerationPrompt(schema, purpose, index, sourceIdentity, overview, sourceContext, sourceSummaryPath) }, + { role: "system", content: buildGenerationPrompt(schema, purpose, reducedIndex, sourceIdentity, overview, sourceContext, sourceSummaryPath) }, { role: "user", content: [ @@ -1111,15 +1146,28 @@ async function autoIngestImpl( const writeWarnings = writeResult.warnings const hardFailures = writeResult.hardFailures - try { - if (await updateWikiIndexDeterministically(pp, writtenPaths)) { - writtenPaths.push("wiki/index.md") - onFileWritten?.("wiki/index.md") + // ── Step 3.5: Process INDEX blocks ────────────────────── + // Parse ---INDEX: blocks from generation output and append + // entries to the existing index.md programmatically. + const indexBlocks = parseIndexBlocks(generation) + if (indexBlocks.length > 0) { + try { + const indexAbs = `${pp}/wiki/index.md` + const existingIndex = await tryReadFile(indexAbs) + const updatedIndex = appendIndexEntries(existingIndex, indexBlocks) + await writeFile(indexAbs, updatedIndex) + console.log( + `[ingest:index] appended ${indexBlocks.reduce((s, b) => s + b.entries.length, 0)} entries ` + + `across ${indexBlocks.length} categories to index.md`, + ) + if (!writtenPaths.includes("wiki/index.md")) { + writtenPaths.push("wiki/index.md") + } + } catch (err) { + writeWarnings.push( + `Failed to append INDEX blocks: ${err instanceof Error ? err.message : String(err)}`, + ) } - } catch (err) { - writeWarnings.push( - `Deterministic index update failed: ${err instanceof Error ? err.message : String(err)}`, - ) } const aggregateRepairPaths = aggregatePathsNeedingRepair(writtenPaths, writeWarnings) @@ -1429,63 +1477,6 @@ export function aggregatePathsNeedingRepair(writtenPaths: string[], warnings: st ) } -async function updateWikiIndexDeterministically( - projectPath: string, - writtenPaths: string[], -): Promise { - const candidates = Array.from(new Set(writtenPaths.map(normalizePath))).filter((path) => - path.startsWith("wiki/") - && path.endsWith(".md") - && !AGGREGATE_WIKI_PATHS.includes(path as (typeof AGGREGATE_WIKI_PATHS)[number]), - ) - if (candidates.length === 0) return false - - const indexPath = `${projectPath}/wiki/index.md` - const index = await readFile(indexPath).catch(() => "# Wiki Index\n") - const knownTargets = new Set( - Array.from(index.matchAll(/\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|[^\]]+)?\]\]/g)) - .map((match) => normalizeIndexTarget(match[1])), - ) - const additions: string[] = [] - for (const path of candidates) { - const target = path.replace(/^wiki\//, "").replace(/\.md$/i, "") - if (knownTargets.has(normalizeIndexTarget(target))) continue - const content = await readFile(`${projectPath}/${path}`).catch(() => "") - const parsed = parseFrontmatter(content) - const title = typeof parsed.frontmatter?.title === "string" - ? parsed.frontmatter.title.trim() - : getFileName(path).replace(/\.md$/i, "") - additions.push(`- [[${target}]] — ${title}`) - } - if (additions.length === 0) return false - - await writeFile(indexPath, updateBoundedRecentIndexSection(index, additions)) - return true -} - -function normalizeIndexTarget(target: string): string { - return normalizePath(target) - .replace(/^wiki\//i, "") - .replace(/\.md$/i, "") - .toLowerCase() -} - -export function updateBoundedRecentIndexSection(index: string, additions: string[]): string { - const section = "## Recently Updated" - const lines = index.trimEnd().split("\n") - const start = lines.findIndex((line) => line.trim() === section) - const prefix = start >= 0 ? lines.slice(0, start) : lines - const sectionEnd = start >= 0 - ? lines.findIndex((line, position) => position > start && /^##\s+/.test(line)) - : -1 - const existing = start >= 0 - ? lines.slice(start + 1, sectionEnd >= 0 ? sectionEnd : undefined).filter((line) => /^-\s+/.test(line)) - : [] - const suffix = sectionEnd >= 0 ? lines.slice(sectionEnd) : [] - const recent = Array.from(new Set([...additions, ...existing])).slice(0, 200) - return [...prefix, "", section, ...recent, ...(suffix.length ? ["", ...suffix] : []), ""].join("\n") -} - export function filterAggregateRepairOutput(text: string, allowedPaths: string[]): { text: string warnings: string[] From 62b188894296ed033016132d524fbde809b08f73 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 11:08:16 +0800 Subject: [PATCH 08/29] feat: remove index.md from aggregate repair, add fallback INDEX construction --- src/lib/ingest-parse.test.ts | 4 +-- src/lib/ingest.ts | 59 ++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) mode change 100644 => 100755 src/lib/ingest-parse.test.ts diff --git a/src/lib/ingest-parse.test.ts b/src/lib/ingest-parse.test.ts old mode 100644 new mode 100755 index 525b4d728..1c34c3d6d --- a/src/lib/ingest-parse.test.ts +++ b/src/lib/ingest-parse.test.ts @@ -115,14 +115,14 @@ describe("source summary media refs", () => { describe("aggregate repair targeting", () => { it("repairs only the append-only log and leaves deterministic aggregates to the app", () => { expect(aggregatePathsNeedingRepair( - ["wiki/index.md", "wiki/log.md"], + ["wiki/log.md"], ['FILE block "wiki/overview.md" was not closed before end of stream — likely truncation.'], )).toEqual([]) expect(aggregatePathsNeedingRepair(["wiki/index.md"], [])).toEqual(["wiki/log.md"]) expect(aggregatePathsNeedingRepair( - ["wiki/index.md", "wiki/overview.md", "wiki/log.md"], + ["wiki/overview.md", "wiki/log.md"], [], )).toEqual([]) }) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 48fe5d0b6..d21f497fa 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -39,7 +39,7 @@ import { } from "@/lib/extract-source-images" import { captionMarkdownImages, loadCaptionCache } from "@/lib/image-caption-pipeline" import type { MultimodalConfig } from "@/stores/wiki-store" -import { GENERATION_WIKI_TYPES } from "@/lib/wiki-page-types" +import { GENERATION_WIKI_TYPES, inferWikiTypeFromPath, wikiTypeLabel } from "@/lib/wiki-page-types" import { computeContextBudget } from "@/lib/context-budget" import { refreshProjectFileTree } from "@/lib/project-file-tree-refresh" import { @@ -62,7 +62,7 @@ const INGEST_GENERATION_TOKENS_256K = 24_576 const INGEST_GENERATION_TOKENS_512K = 32_768 const REVIEW_STAGE_MIN_SIGNAL_CHARS = 10_000 const REVIEW_STAGE_MIN_FILE_BLOCKS = 4 -const AGGREGATE_WIKI_PATHS = ["wiki/index.md", "wiki/overview.md", "wiki/log.md"] as const +const AGGREGATE_WIKI_PATHS = ["wiki/overview.md", "wiki/log.md"] as const function appendSavedImageRefsForCaption(content: string, images: SavedImage[]): string { if (images.length === 0) return content @@ -1170,6 +1170,32 @@ async function autoIngestImpl( } } + // ── Step 3.6: Fallback INDEX construction ─────────────── + // If no INDEX blocks were emitted, construct them from the + // written FILE blocks as a fallback. This replaces the old + // aggregate-repair-for-index.md path. + if (indexBlocks.length === 0 && writtenPaths.length > 1) { + try { + const fallbackBlocks = buildFallbackIndexBlocks(generation) + if (fallbackBlocks.length > 0) { + const indexAbs = `${pp}/wiki/index.md` + const existingIndex = await tryReadFile(indexAbs) + const updatedIndex = appendIndexEntries(existingIndex, fallbackBlocks) + await writeFile(indexAbs, updatedIndex) + writeWarnings.push("No INDEX blocks in generation — constructed fallback entries from FILE blocks") + if (!writtenPaths.includes("wiki/index.md")) { + writtenPaths.push("wiki/index.md") + } + console.log( + `[ingest:index-fallback] constructed ${fallbackBlocks.reduce((s, b) => s + b.entries.length, 0)} entries ` + + `across ${fallbackBlocks.length} categories from FILE blocks`, + ) + } + } catch (err) { + writeWarnings.push(`Fallback index construction failed: ${err instanceof Error ? err.message : String(err)}`) + } + } + const aggregateRepairPaths = aggregatePathsNeedingRepair(writtenPaths, writeWarnings) const repairableAggregatePaths = aggregateRepairPaths.filter((path) => isAggregateRepairSafe(path, index, overview, llmConfig.maxContextSize), @@ -1425,6 +1451,35 @@ export function isAppManagedAggregatePath(relativePath: string): boolean { return normalized === "wiki/index.md" || normalized === "wiki/overview.md" } +/** + * When generation output has no ---INDEX: blocks, construct fallback + * entries from the FILE blocks that were written. Extracts slug from + * path and title from frontmatter. + */ +function buildFallbackIndexBlocks( + generation: string, +): import("./index-chunker").ParsedIndexBlock[] { + const { blocks } = parseFileBlocks(generation) + const byCategory = new Map() + + for (const block of blocks) { + // Skip aggregate files and source summaries + if (block.path === "wiki/index.md" || block.path === "wiki/log.md" || block.path === "wiki/overview.md") continue + if (block.path.startsWith("wiki/sources/")) continue + + const type = inferWikiTypeFromPath(block.path) ?? "entity" + const category = wikiTypeLabel(type) + "s" + const slug = block.path.replace(/\.md$/, "").split("/").pop() ?? "" + const title = extractGeneratedPageTitle(block.content) ?? slug + + const entry = title === slug ? slug : `${slug} — ${title}` + if (!byCategory.has(category)) byCategory.set(category, []) + byCategory.get(category)!.push(entry) + } + + return [...byCategory.entries()].map(([category, entries]) => ({ category, entries })) +} + const CJK_OUTPUT_LANGUAGES = new Set(["Chinese", "Traditional Chinese", "Japanese", "Korean"]) function containsCjk(text: string): boolean { From 4bf88993b2e5ebdf7ccd21d063c8f1dec5bf40b6 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 11:11:42 +0800 Subject: [PATCH 09/29] feat: add prematch cache for index chunk matching results --- src/lib/ingest-cache.test.ts | 44 +++++++++++++++++++++++++++++++++++- src/lib/ingest-cache.ts | 31 +++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/lib/ingest-cache.test.ts mode change 100644 => 100755 src/lib/ingest-cache.ts diff --git a/src/lib/ingest-cache.test.ts b/src/lib/ingest-cache.test.ts old mode 100644 new mode 100755 index ed949cd63..69c578402 --- a/src/lib/ingest-cache.test.ts +++ b/src/lib/ingest-cache.test.ts @@ -7,7 +7,7 @@ vi.mock("@/commands/fs", () => ({ fileExists: vi.fn(), })) -import { checkIngestCache, saveIngestCache } from "./ingest-cache" +import { checkIngestCache, saveIngestCache, checkPrematchCache, savePrematchCache } from "./ingest-cache" import { readFile, writeFile, fileExists } from "@/commands/fs" const mockReadFile = vi.mocked(readFile) @@ -94,3 +94,45 @@ describe("ingest-cache — checkIngestCache", () => { expect(result).toBeNull() }) }) + +describe("prematch cache", () => { + beforeEach(() => { + mockReadFile.mockReset() + mockWriteFile.mockReset() + mockWriteFile.mockResolvedValue(undefined as unknown as void) + }) + + it("returns null when no cache entry exists", async () => { + mockReadFile.mockResolvedValue(JSON.stringify({ entries: {}, prematch: {} })) + const result = await checkPrematchCache("/project", "source-hash-abc", "chunk-hash-xyz") + expect(result).toBeNull() + }) + + it("returns cached result on hit", async () => { + let persisted = "" + mockReadFile.mockImplementation(async () => persisted || JSON.stringify({ entries: {}, prematch: {} })) + mockWriteFile.mockImplementation(async (_p, c) => { persisted = c }) + + await savePrematchCache("/project", "src-hash", "chunk-hash", [1, 3, 5]) + const result = await checkPrematchCache("/project", "src-hash", "chunk-hash") + expect(result).toEqual([1, 3, 5]) + }) + + it("returns null when prematch key not in cache", async () => { + let persisted = "" + mockReadFile.mockImplementation(async () => persisted || JSON.stringify({ entries: {}, prematch: {} })) + mockWriteFile.mockImplementation(async (_p, c) => { persisted = c }) + + await savePrematchCache("/project", "src-hash", "chunk-hash", [1, 3]) + // Different key should miss + const result = await checkPrematchCache("/project", "different-src", "chunk-hash") + expect(result).toBeNull() + }) + + it("handles cache file with no prematch field (backwards compat)", async () => { + // Old cache format without prematch field + mockReadFile.mockResolvedValue(JSON.stringify({ entries: {} })) + const result = await checkPrematchCache("/project", "src", "chunk") + expect(result).toBeNull() + }) +}) diff --git a/src/lib/ingest-cache.ts b/src/lib/ingest-cache.ts old mode 100644 new mode 100755 index eaa82d7d8..b8f2b8a9e --- a/src/lib/ingest-cache.ts +++ b/src/lib/ingest-cache.ts @@ -15,6 +15,7 @@ interface CacheEntry { interface CacheData { entries: Record // keyed by source filename + prematch?: Record // keyed by "sourceHash:chunkHash" } async function sha256(content: string): Promise { @@ -112,6 +113,36 @@ export async function saveIngestCache( await saveCache(projectPath, { entries: newEntries }) } +/** + * Check if a pre-match result is cached for a source+chunk combination. + * Returns the cached matched numbers, or null if not cached. + */ +export async function checkPrematchCache( + projectPath: string, + sourceHash: string, + chunkHash: string, +): Promise { + const cache = await loadCache(projectPath) + const key = `${sourceHash}:${chunkHash}` + return cache.prematch?.[key] ?? null +} + +/** + * Save a pre-match result to the cache. + */ +export async function savePrematchCache( + projectPath: string, + sourceHash: string, + chunkHash: string, + matchedNumbers: number[], +): Promise { + const cache = await loadCache(projectPath) + if (!cache.prematch) cache.prematch = {} + const key = `${sourceHash}:${chunkHash}` + cache.prematch[key] = matchedNumbers + await saveCache(projectPath, cache) +} + /** * Remove a source file entry from cache (e.g., when source is deleted). */ From cb7b1992b540e9edb1d6bbac51932c550ef13348 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 11:13:15 +0800 Subject: [PATCH 10/29] fix: preserve prematch cache when saving/removing ingest cache entries --- src/lib/ingest-cache.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/lib/ingest-cache.ts b/src/lib/ingest-cache.ts index b8f2b8a9e..a8f5b97e8 100755 --- a/src/lib/ingest-cache.ts +++ b/src/lib/ingest-cache.ts @@ -104,13 +104,12 @@ export async function saveIngestCache( ): Promise { const cache = await loadCache(projectPath) const hash = await sha256(sourceContent) - const newEntries = { ...cache.entries } - newEntries[sourceFileName] = { + cache.entries[sourceFileName] = { hash, timestamp: Date.now(), filesWritten, } - await saveCache(projectPath, { entries: newEntries }) + await saveCache(projectPath, cache) } /** @@ -151,9 +150,8 @@ export async function removeFromIngestCache( sourceFileName: string, ): Promise { const cache = await loadCache(projectPath) - const newEntries = { ...cache.entries } - delete newEntries[sourceFileName] - await saveCache(projectPath, { entries: newEntries }) + delete cache.entries[sourceFileName] + await saveCache(projectPath, cache) } /** Move a cache entry when an unchanged source is renamed inside raw/sources. */ From d46c5f364222d7aa46c19b2fa7c4904ddd49bea0 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 11:20:45 +0800 Subject: [PATCH 11/29] test: update ingest scenarios for INDEX block format --- src/lib/ingest.scenarios.test.ts | 2 +- .../scenarios/ingest-scenarios.ts | 39 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/lib/ingest.scenarios.test.ts b/src/lib/ingest.scenarios.test.ts index b9d1e9031..2e131dcd2 100644 --- a/src/lib/ingest.scenarios.test.ts +++ b/src/lib/ingest.scenarios.test.ts @@ -3,7 +3,7 @@ * * Each scenario materializes an initial project, a source document, and two * canned LLM responses (stage 1 analysis, stage 2 generation with FILE + - * REVIEW blocks). The runner mocks streamChat to emit them sequentially. + * INDEX + REVIEW blocks). The runner mocks streamChat to emit them sequentially. * * After ingest runs, the runner asserts: * - expected files exist on disk with expected substrings diff --git a/src/test-helpers/scenarios/ingest-scenarios.ts b/src/test-helpers/scenarios/ingest-scenarios.ts index 2966d26bf..90df89c16 100644 --- a/src/test-helpers/scenarios/ingest-scenarios.ts +++ b/src/test-helpers/scenarios/ingest-scenarios.ts @@ -2,13 +2,23 @@ import type { IngestScenario } from "./types" /** * Ingest scenarios drive autoIngest end-to-end. Two LLM responses per - * scenario (stage 1 analysis, stage 2 generation with FILE + REVIEW blocks). + * scenario (stage 1 analysis, stage 2 generation with FILE + INDEX + + * REVIEW blocks). * * FILE block format (what stage 2 must emit to write a wiki file): * ---FILE: wiki/path/to/page.md--- * (file content, usually with YAML frontmatter) * ---END FILE--- * + * INDEX block format (what stage 2 emits to append new entries to + * wiki/index.md instead of regenerating the whole file): + * ---INDEX: CategoryName--- + * slug — Short description + * ---END INDEX--- + * INDEX blocks appear AFTER all FILE blocks and BEFORE any REVIEW blocks. + * Only NEW entries are listed (pre-existing index entries are preserved + * by the append path, never re-emitted). + * * REVIEW block format (what stage 2 emits to inject a review item): * ---REVIEW: missing-page | Short title--- * Description. @@ -100,11 +110,16 @@ export const ingestScenarios: IngestScenario[] = [ "", "Paper introducing [[Rotary Position Embedding]].", "---END FILE---", + "", + "---INDEX: Concepts---", + "rope — Rotary Position Embedding for variable-length contexts", + "---END INDEX---", ].join("\n"), expected: { writtenPaths: [ "wiki/concepts/rope.md", "wiki/sources/rope-paper.md", + "wiki/index.md", ], fileContains: { "wiki/concepts/rope.md": [ @@ -112,6 +127,7 @@ export const ingestScenarios: IngestScenario[] = [ "[[attention]]", ], "wiki/sources/rope-paper.md": ["rope-paper.md"], + "wiki/index.md": ["[[attention]]", "[[rope]]"], }, reviewsCreated: [], }, @@ -146,6 +162,10 @@ export const ingestScenarios: IngestScenario[] = [ "FlashAttention is mentioned here.", "---END FILE---", "", + "---INDEX: Sources---", + "flash-attention — FlashAttention source summary", + "---END INDEX---", + "", "---REVIEW: missing-page | FlashAttention---", "The source introduces FlashAttention but no dedicated page exists.", "OPTIONS: Create page | Skip", @@ -157,7 +177,10 @@ export const ingestScenarios: IngestScenario[] = [ "---END REVIEW---", ].join("\n"), expected: { - writtenPaths: ["wiki/sources/flash-attention.md"], + writtenPaths: ["wiki/sources/flash-attention.md", "wiki/index.md"], + fileContains: { + "wiki/index.md": ["[[flash-attention]]"], + }, reviewsCreated: [ { type: "missing-page", titleContains: "FlashAttention" }, { type: "suggestion", titleContains: "IO-aware" }, @@ -205,14 +228,20 @@ export const ingestScenarios: IngestScenario[] = [ "", "Source for multi-head [[attention]].", "---END FILE---", + "", + "---INDEX: Concepts---", + "multi-head-attention — Parallel multi-head attention variant", + "---END INDEX---", ].join("\n"), expected: { writtenPaths: [ "wiki/concepts/multi-head-attention.md", "wiki/sources/multi-head.md", + "wiki/index.md", ], fileContains: { "wiki/concepts/multi-head-attention.md": ["[[attention]]"], + "wiki/index.md": ["[[attention]]", "[[multi-head-attention]]"], }, }, }, @@ -253,17 +282,23 @@ export const ingestScenarios: IngestScenario[] = [ "", "关于 [[Transformer]] 的综述。", "---END FILE---", + "", + "---INDEX: Concepts---", + "transformer — 基于注意力机制的神经网络架构", + "---END INDEX---", ].join("\n"), expected: { writtenPaths: [ "wiki/concepts/transformer.md", "wiki/sources/transformer-survey.md", + "wiki/index.md", ], fileContains: { "wiki/concepts/transformer.md": [ "title: Transformer", "[[注意力机制]]", ], + "wiki/index.md": ["[[注意力机制]]", "[[transformer]]"], }, }, }, From 98c3c7f5d46131876ab7e5da55cf13205369aaf6 Mon Sep 17 00:00:00 2001 From: wkh <746906141@qq.com> Date: Thu, 25 Jun 2026 11:40:46 +0800 Subject: [PATCH 12/29] chore: add .gitattributes and normalize line endings to LF --- .gitattributes | 2 ++ .github/workflows/build.yml | 0 .github/workflows/ci.yml | 0 .gitignore | 0 LICENSE | 0 README.md | 0 README_CN.md | 0 README_JA.md | 0 README_KO.md | 0 assets/1-deepresearch.jpg | Bin assets/2-ai_chat.jpg | Bin assets/3-knowledge_graph.jpg | Bin assets/4-chrome_extension_webclipper.jpg | Bin assets/5-obsidian_compatibility.jpg | Bin assets/kg_community.jpg | Bin assets/kg_insights.jpg | Bin assets/llm_wiki_arch.jpg | Bin assets/overview.jpg | Bin components.json | 0 extension/Readability.js | 0 extension/Turndown.js | 0 extension/icon128.png | Bin extension/icon16.png | Bin extension/icon48.png | Bin extension/manifest.json | 0 extension/popup.html | 0 extension/popup.js | 0 index.html | 0 llm-wiki.md | 0 logo.jpg | Bin mcp-server/README.md | 0 mcp-server/package-lock.json | 0 mcp-server/package.json | 0 mcp-server/src/api-client.ts | 0 mcp-server/src/index.ts | 0 mcp-server/src/version.ts | 0 mcp-server/test/api-client.test.ts | 0 mcp-server/test/version.test.ts | 0 mcp-server/tsconfig.json | 0 package-lock.json | 0 package.json | 0 plans/multimodal-images.md | 0 scripts/debug_ollama_tokens.py | 0 src-tauri/Cargo.lock | 0 src-tauri/Cargo.toml | 0 src-tauri/build.rs | 0 src-tauri/capabilities/default.json | 0 src-tauri/icons/128x128.png | Bin src-tauri/icons/128x128@2x.png | Bin src-tauri/icons/32x32.png | Bin src-tauri/icons/icon.icns | Bin src-tauri/icons/icon.ico | Bin src-tauri/icons/icon.png | Bin src-tauri/pdfium/SHA256SUMS | 0 src-tauri/pdfium/libpdfium-arm64.so | Bin src-tauri/pdfium/libpdfium-x86_64.dylib | Bin src-tauri/pdfium/libpdfium.dylib | Bin src-tauri/pdfium/libpdfium.so | Bin src-tauri/pdfium/pdfium.dll | Bin src-tauri/src/api_server.rs | 0 src-tauri/src/clip_server.rs | 0 src-tauri/src/commands/claude_cli.rs | 0 src-tauri/src/commands/cli_resolver.rs | 0 src-tauri/src/commands/codex_cli.rs | 0 src-tauri/src/commands/extract_images.rs | 0 src-tauri/src/commands/file_sync.rs | 0 src-tauri/src/commands/fs.rs | 0 src-tauri/src/commands/mod.rs | 0 src-tauri/src/commands/project.rs | 0 src-tauri/src/commands/search.rs | 0 src-tauri/src/commands/vectorstore.rs | 0 src-tauri/src/lib.rs | 0 src-tauri/src/main.rs | 0 src-tauri/src/panic_guard.rs | 0 src-tauri/src/proxy.rs | 0 src-tauri/src/tray.rs | 0 src-tauri/src/types/mod.rs | 0 src-tauri/src/types/wiki.rs | 0 src-tauri/tauri.conf.json | 0 src-tauri/tauri.linux.conf.json | 0 src-tauri/tauri.macos.conf.json | 0 src-tauri/tauri.windows.conf.json | 0 src-tauri/windows-app-manifest.xml | 0 src/App.tsx | 0 src/assets/logo.jpg | Bin src/commands/file-sync.ts | 0 src/commands/fs.test.ts | 0 src/commands/fs.ts | 0 src/components/chat/chat-input.tsx | 0 src/components/chat/chat-message.tsx | 0 src/components/chat/chat-panel.tsx | 0 src/components/editor/file-preview.tsx | 0 src/components/editor/frontmatter-panel.tsx | 0 src/components/editor/wiki-editor.tsx | 0 src/components/editor/wiki-reader.tsx | 0 src/components/error-boundary.tsx | 0 src/components/graph/graph-layout-worker.ts | 0 src/components/graph/graph-view.tsx | 0 src/components/layout/activity-panel.tsx | 0 src/components/layout/app-layout-visibility.test.ts | 0 src/components/layout/app-layout-visibility.ts | 0 src/components/layout/app-layout.tsx | 0 src/components/layout/content-area.tsx | 0 src/components/layout/file-tree.tsx | 0 src/components/layout/icon-sidebar.tsx | 0 src/components/layout/knowledge-tree.tsx | 0 src/components/layout/preview-panel.tsx | 0 src/components/layout/research-panel-nav.test.ts | 0 src/components/layout/research-panel-nav.ts | 0 src/components/layout/research-panel.tsx | 0 src/components/layout/sidebar-panel.tsx | 0 src/components/layout/update-banner.tsx | 0 src/components/lint/lint-view.test.ts | 0 src/components/lint/lint-view.tsx | 0 src/components/mermaid-diagram.test.tsx | 0 src/components/mermaid-diagram.tsx | 0 .../project/create-project-dialog.test.ts | 0 src/components/project/create-project-dialog.tsx | 0 src/components/project/template-picker.tsx | 0 src/components/project/welcome-screen.tsx | 0 src/components/review/review-view.tsx | 0 src/components/search/search-view.tsx | 0 src/components/settings/context-size-selector.tsx | 0 src/components/settings/llm-presets.ts | 0 src/components/settings/preset-resolver.test.ts | 0 src/components/settings/preset-resolver.ts | 0 src/components/settings/sections/about-section.tsx | 0 .../settings/sections/api-server-section.test.ts | 0 .../settings/sections/api-server-section.tsx | 0 .../settings/sections/changelog-section.tsx | 0 .../settings/sections/embedding-section.tsx | 0 .../settings/sections/general-section.tsx | 0 .../settings/sections/interface-section.tsx | 0 .../settings/sections/llm-provider-section.tsx | 0 .../settings/sections/maintenance-section.tsx | 0 src/components/settings/sections/mineru-section.tsx | 0 .../settings/sections/multimodal-section.tsx | 0 .../settings/sections/network-section.tsx | 0 src/components/settings/sections/output-section.tsx | 0 .../settings/sections/scheduled-import-section.tsx | 0 .../settings/sections/source-watch-section.tsx | 0 .../settings/sections/web-search-section.tsx | 0 src/components/settings/settings-types.ts | 0 src/components/settings/settings-view.tsx | 0 src/components/sources/sources-view.tsx | 0 src/components/ui/button.tsx | 0 src/components/ui/dialog.tsx | 0 src/components/ui/input.tsx | 0 src/components/ui/label.tsx | 0 src/components/ui/resizable.tsx | 0 src/components/ui/scroll-area.tsx | 0 src/components/ui/separator.tsx | 0 src/components/ui/tooltip.tsx | 0 src/i18n/en.json | 0 src/i18n/i18n-parity.test.ts | 0 src/i18n/index.ts | 0 src/i18n/zh.json | 0 src/index.css | 0 src/lib/__tests__/claude-cli-transport.test.ts | 0 src/lib/__tests__/dedup_embedding.test.ts | 0 src/lib/__tests__/llm-providers.test.ts | 0 src/lib/anytxt-search.test.ts | 0 src/lib/anytxt-search.ts | 0 src/lib/api-server-constants.ts | 0 src/lib/api-server.real-llm.test.ts | 0 src/lib/api-token.test.ts | 0 src/lib/api-token.ts | 0 src/lib/auto-save.test.ts | 0 src/lib/auto-save.ts | 0 src/lib/azure-openai.test.ts | 0 src/lib/azure-openai.ts | 0 src/lib/changelog.ts | 0 src/lib/chat-image-utils.ts | 0 src/lib/chat-save-to-wiki.test.ts | 0 src/lib/chat-save-to-wiki.ts | 0 src/lib/claude-cli-transport.ts | 0 src/lib/clip-watcher.ts | 0 src/lib/codex-cli-transport.test.ts | 0 src/lib/codex-cli-transport.ts | 0 src/lib/connection-tests.test.ts | 0 src/lib/connection-tests.ts | 0 src/lib/context-budget.test.ts | 0 src/lib/context-budget.ts | 0 src/lib/dedup-queue.test.ts | 0 src/lib/dedup-queue.ts | 0 src/lib/dedup-runner.test.ts | 0 src/lib/dedup-runner.ts | 0 src/lib/dedup-storage.ts | 0 src/lib/dedup.test.ts | 0 src/lib/dedup.ts | 0 src/lib/dedup_embedding.ts | 0 src/lib/deep-research.test.ts | 0 src/lib/deep-research.ts | 0 src/lib/detect-language.property.test.ts | 0 src/lib/detect-language.test.ts | 0 src/lib/detect-language.ts | 0 src/lib/embedding.real-llm.test.ts | 0 src/lib/embedding.test.ts | 0 src/lib/embedding.ts | 0 src/lib/endpoint-normalizer.test.ts | 0 src/lib/endpoint-normalizer.ts | 0 src/lib/enrich-wikilinks.real-llm.test.ts | 0 src/lib/enrich-wikilinks.scenarios.test.ts | 0 src/lib/enrich-wikilinks.test.ts | 0 src/lib/enrich-wikilinks.ts | 0 src/lib/extract-source-images.test.ts | 0 src/lib/extract-source-images.ts | 0 src/lib/file-types.test.ts | 0 src/lib/file-types.ts | 0 src/lib/frontmatter.test.ts | 0 src/lib/frontmatter.ts | 0 src/lib/graph-filters.test.ts | 0 src/lib/graph-filters.ts | 0 src/lib/graph-insights.ts | 0 src/lib/graph-relevance.ts | 0 src/lib/graph-search.test.ts | 0 src/lib/graph-search.ts | 0 src/lib/graph-visibility.test.ts | 0 src/lib/graph-visibility.ts | 0 src/lib/greeting-detector.test.ts | 0 src/lib/greeting-detector.ts | 0 src/lib/has-usable-llm.test.ts | 0 src/lib/has-usable-llm.ts | 0 src/lib/image-caption-pipeline.test.ts | 0 src/lib/image-caption-pipeline.ts | 0 src/lib/ingest-queue.integration.test.ts | 0 src/lib/ingest-queue.test.ts | 0 src/lib/ingest-queue.ts | 0 src/lib/ingest-sanitize.test.ts | 0 src/lib/ingest-sanitize.ts | 0 src/lib/ingest-source-path-collision.test.ts | 0 src/lib/ingest.prompt.test.ts | 0 src/lib/ingest.real-llm.test.ts | 0 src/lib/ingest.scenarios.test.ts | 0 src/lib/keyboard-utils.test.ts | 0 src/lib/keyboard-utils.ts | 0 src/lib/language-metadata.test.ts | 0 src/lib/language-metadata.ts | 0 src/lib/latex-to-unicode.ts | 0 src/lib/lint-fixes.test.ts | 0 src/lib/lint-fixes.ts | 0 src/lib/lint.real-llm.test.ts | 0 src/lib/lint.scenarios.test.ts | 0 src/lib/lint.test.ts | 0 src/lib/lint.ts | 0 src/lib/llm-client.real-llm.test.ts | 0 src/lib/llm-client.test.ts | 0 src/lib/llm-client.ts | 0 src/lib/llm-providers.test.ts | 0 src/lib/llm-providers.ts | 0 src/lib/markdown-image-resolver.test.ts | 0 src/lib/markdown-image-resolver.ts | 0 src/lib/mineru.test.ts | 0 src/lib/mineru.ts | 0 src/lib/optimize-research-topic.test.ts | 0 src/lib/optimize-research-topic.ts | 0 src/lib/output-language-options.ts | 0 src/lib/output-language.test.ts | 0 src/lib/output-language.ts | 0 src/lib/page-merge.test.ts | 0 src/lib/page-merge.ts | 0 src/lib/path-utils.property.test.ts | 0 src/lib/path-utils.test.ts | 0 src/lib/path-utils.ts | 0 src/lib/persist.integration.test.ts | 0 src/lib/persist.ts | 0 src/lib/project-file-sync.test.ts | 0 src/lib/project-file-sync.ts | 0 src/lib/project-identity.ts | 0 src/lib/project-mutex.test.ts | 0 src/lib/project-mutex.ts | 0 src/lib/project-store.test.ts | 0 src/lib/project-store.ts | 0 src/lib/proxy-config.test.ts | 0 src/lib/proxy-config.ts | 0 src/lib/raw-source-resolver.test.ts | 0 src/lib/raw-source-resolver.ts | 0 src/lib/reasoning-detector.test.ts | 0 src/lib/reasoning-detector.ts | 0 src/lib/reset-project-state.test.ts | 0 src/lib/reset-project-state.ts | 0 src/lib/review-create-page.test.ts | 0 src/lib/review-create-page.ts | 0 src/lib/review-utils.property.test.ts | 0 src/lib/review-utils.test.ts | 0 src/lib/review-utils.ts | 0 src/lib/scheduled-import.test.ts | 0 src/lib/scheduled-import.ts | 0 src/lib/search-rrf.test.ts | 0 src/lib/search.scenarios.test.ts | 0 src/lib/search.ts | 0 src/lib/source-delete-decision.test.ts | 0 src/lib/source-delete-decision.ts | 0 src/lib/source-identity.test.ts | 0 src/lib/source-identity.ts | 0 src/lib/source-lifecycle-delete.test.ts | 0 src/lib/source-lifecycle.test.ts | 0 src/lib/source-lifecycle.ts | 0 src/lib/source-watch-config.test.ts | 0 src/lib/source-watch-config.ts | 0 src/lib/source-watch-defaults.json | 0 src/lib/sources-merge.test.ts | 0 src/lib/sources-merge.ts | 0 src/lib/sources-tree-delete.test.ts | 0 src/lib/sources-tree-delete.ts | 0 src/lib/sweep-chained.real-llm.test.ts | 0 src/lib/sweep-reviews.property.test.ts | 0 src/lib/sweep-reviews.race.test.ts | 0 src/lib/sweep-reviews.real-llm.test.ts | 0 src/lib/sweep-reviews.scenarios.test.ts | 0 src/lib/sweep-reviews.test.ts | 0 src/lib/sweep-reviews.ts | 0 src/lib/tauri-fetch.test.ts | 0 src/lib/tauri-fetch.ts | 0 src/lib/templates.ts | 0 src/lib/text-chunker.test.ts | 0 src/lib/text-chunker.ts | 0 src/lib/theme.ts | 0 src/lib/update-check.test.ts | 0 src/lib/update-check.ts | 0 src/lib/utils.ts | 0 src/lib/vision-caption.real-llm.test.ts | 0 src/lib/vision-caption.test.ts | 0 src/lib/vision-caption.ts | 0 src/lib/vision.real-llm.test.ts | 0 src/lib/web-search.test.ts | 0 src/lib/web-search.ts | 0 src/lib/wiki-cleanup.test.ts | 0 src/lib/wiki-cleanup.ts | 0 src/lib/wiki-filename.test.ts | 0 src/lib/wiki-filename.ts | 0 src/lib/wiki-graph.ts | 0 src/lib/wiki-page-delete.test.ts | 0 src/lib/wiki-page-delete.ts | 0 src/lib/wiki-page-resolver.test.ts | 0 src/lib/wiki-page-resolver.ts | 0 src/lib/wiki-page-types.test.ts | 0 src/lib/wiki-page-types.ts | 0 src/lib/wiki-schema.test.ts | 0 src/lib/wiki-schema.ts | 0 src/lib/wiki-type-style.test.ts | 0 src/lib/wiki-type-style.ts | 0 src/lib/wikilink-transform.test.ts | 0 src/lib/wikilink-transform.ts | 0 src/main.tsx | 0 src/stores/activity-store.ts | 0 src/stores/chat-messages-to-llm.test.ts | 0 src/stores/chat-store.ts | 0 src/stores/file-sync-store.ts | 0 src/stores/lint-store.test.ts | 0 src/stores/lint-store.ts | 0 src/stores/research-store.ts | 0 src/stores/review-store.property.test.ts | 0 src/stores/review-store.test.ts | 0 src/stores/review-store.ts | 0 src/stores/update-store.ts | 0 src/stores/wiki-store.test.ts | 0 src/stores/wiki-store.ts | 0 src/stores/zoom-store.ts | 0 src/test-helpers/deferred.ts | 0 src/test-helpers/fs-temp.ts | 0 src/test-helpers/load-test-env.ts | 0 src/test-helpers/mock-stream-chat.ts | 0 src/test-helpers/real-content.ts | 0 src/test-helpers/scenarios/enrich-scenarios.ts | 0 src/test-helpers/scenarios/ingest-scenarios.ts | 0 src/test-helpers/scenarios/lint-scenarios.ts | 0 src/test-helpers/scenarios/materialize.ts | 0 src/test-helpers/scenarios/search-scenarios.ts | 0 src/test-helpers/scenarios/sweep-scenarios.ts | 0 src/test-helpers/scenarios/types.ts | 0 src/types/wiki.ts | 0 src/vite-env.d.ts | 0 tsconfig.app.json | 0 tsconfig.json | 0 tsconfig.node.json | 0 vite.config.ts | 0 377 files changed, 2 insertions(+) create mode 100755 .gitattributes mode change 100644 => 100755 .github/workflows/build.yml mode change 100644 => 100755 .github/workflows/ci.yml mode change 100644 => 100755 .gitignore mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md mode change 100644 => 100755 README_CN.md mode change 100644 => 100755 README_JA.md mode change 100644 => 100755 README_KO.md mode change 100644 => 100755 assets/1-deepresearch.jpg mode change 100644 => 100755 assets/2-ai_chat.jpg mode change 100644 => 100755 assets/3-knowledge_graph.jpg mode change 100644 => 100755 assets/4-chrome_extension_webclipper.jpg mode change 100644 => 100755 assets/5-obsidian_compatibility.jpg mode change 100644 => 100755 assets/kg_community.jpg mode change 100644 => 100755 assets/kg_insights.jpg mode change 100644 => 100755 assets/llm_wiki_arch.jpg mode change 100644 => 100755 assets/overview.jpg mode change 100644 => 100755 components.json mode change 100644 => 100755 extension/Readability.js mode change 100644 => 100755 extension/Turndown.js mode change 100644 => 100755 extension/icon128.png mode change 100644 => 100755 extension/icon16.png mode change 100644 => 100755 extension/icon48.png mode change 100644 => 100755 extension/manifest.json mode change 100644 => 100755 extension/popup.html mode change 100644 => 100755 extension/popup.js mode change 100644 => 100755 index.html mode change 100644 => 100755 llm-wiki.md mode change 100644 => 100755 logo.jpg mode change 100644 => 100755 mcp-server/README.md mode change 100644 => 100755 mcp-server/package-lock.json mode change 100644 => 100755 mcp-server/package.json mode change 100644 => 100755 mcp-server/src/api-client.ts mode change 100644 => 100755 mcp-server/src/index.ts mode change 100644 => 100755 mcp-server/src/version.ts mode change 100644 => 100755 mcp-server/test/api-client.test.ts mode change 100644 => 100755 mcp-server/test/version.test.ts mode change 100644 => 100755 mcp-server/tsconfig.json mode change 100644 => 100755 package-lock.json mode change 100644 => 100755 package.json mode change 100644 => 100755 plans/multimodal-images.md mode change 100644 => 100755 scripts/debug_ollama_tokens.py mode change 100644 => 100755 src-tauri/Cargo.lock mode change 100644 => 100755 src-tauri/Cargo.toml mode change 100644 => 100755 src-tauri/build.rs mode change 100644 => 100755 src-tauri/capabilities/default.json mode change 100644 => 100755 src-tauri/icons/128x128.png mode change 100644 => 100755 src-tauri/icons/128x128@2x.png mode change 100644 => 100755 src-tauri/icons/32x32.png mode change 100644 => 100755 src-tauri/icons/icon.icns mode change 100644 => 100755 src-tauri/icons/icon.ico mode change 100644 => 100755 src-tauri/icons/icon.png mode change 100644 => 100755 src-tauri/pdfium/SHA256SUMS mode change 100644 => 100755 src-tauri/pdfium/libpdfium-arm64.so mode change 100644 => 100755 src-tauri/pdfium/libpdfium-x86_64.dylib mode change 100644 => 100755 src-tauri/pdfium/libpdfium.dylib mode change 100644 => 100755 src-tauri/pdfium/libpdfium.so mode change 100644 => 100755 src-tauri/pdfium/pdfium.dll mode change 100644 => 100755 src-tauri/src/api_server.rs mode change 100644 => 100755 src-tauri/src/clip_server.rs mode change 100644 => 100755 src-tauri/src/commands/claude_cli.rs mode change 100644 => 100755 src-tauri/src/commands/cli_resolver.rs mode change 100644 => 100755 src-tauri/src/commands/codex_cli.rs mode change 100644 => 100755 src-tauri/src/commands/extract_images.rs mode change 100644 => 100755 src-tauri/src/commands/file_sync.rs mode change 100644 => 100755 src-tauri/src/commands/fs.rs mode change 100644 => 100755 src-tauri/src/commands/mod.rs mode change 100644 => 100755 src-tauri/src/commands/project.rs mode change 100644 => 100755 src-tauri/src/commands/search.rs mode change 100644 => 100755 src-tauri/src/commands/vectorstore.rs mode change 100644 => 100755 src-tauri/src/lib.rs mode change 100644 => 100755 src-tauri/src/main.rs mode change 100644 => 100755 src-tauri/src/panic_guard.rs mode change 100644 => 100755 src-tauri/src/proxy.rs mode change 100644 => 100755 src-tauri/src/tray.rs mode change 100644 => 100755 src-tauri/src/types/mod.rs mode change 100644 => 100755 src-tauri/src/types/wiki.rs mode change 100644 => 100755 src-tauri/tauri.conf.json mode change 100644 => 100755 src-tauri/tauri.linux.conf.json mode change 100644 => 100755 src-tauri/tauri.macos.conf.json mode change 100644 => 100755 src-tauri/tauri.windows.conf.json mode change 100644 => 100755 src-tauri/windows-app-manifest.xml mode change 100644 => 100755 src/App.tsx mode change 100644 => 100755 src/assets/logo.jpg mode change 100644 => 100755 src/commands/file-sync.ts mode change 100644 => 100755 src/commands/fs.test.ts mode change 100644 => 100755 src/commands/fs.ts mode change 100644 => 100755 src/components/chat/chat-input.tsx mode change 100644 => 100755 src/components/chat/chat-message.tsx mode change 100644 => 100755 src/components/chat/chat-panel.tsx mode change 100644 => 100755 src/components/editor/file-preview.tsx mode change 100644 => 100755 src/components/editor/frontmatter-panel.tsx mode change 100644 => 100755 src/components/editor/wiki-editor.tsx mode change 100644 => 100755 src/components/editor/wiki-reader.tsx mode change 100644 => 100755 src/components/error-boundary.tsx mode change 100644 => 100755 src/components/graph/graph-layout-worker.ts mode change 100644 => 100755 src/components/graph/graph-view.tsx mode change 100644 => 100755 src/components/layout/activity-panel.tsx mode change 100644 => 100755 src/components/layout/app-layout-visibility.test.ts mode change 100644 => 100755 src/components/layout/app-layout-visibility.ts mode change 100644 => 100755 src/components/layout/app-layout.tsx mode change 100644 => 100755 src/components/layout/content-area.tsx mode change 100644 => 100755 src/components/layout/file-tree.tsx mode change 100644 => 100755 src/components/layout/icon-sidebar.tsx mode change 100644 => 100755 src/components/layout/knowledge-tree.tsx mode change 100644 => 100755 src/components/layout/preview-panel.tsx mode change 100644 => 100755 src/components/layout/research-panel-nav.test.ts mode change 100644 => 100755 src/components/layout/research-panel-nav.ts mode change 100644 => 100755 src/components/layout/research-panel.tsx mode change 100644 => 100755 src/components/layout/sidebar-panel.tsx mode change 100644 => 100755 src/components/layout/update-banner.tsx mode change 100644 => 100755 src/components/lint/lint-view.test.ts mode change 100644 => 100755 src/components/lint/lint-view.tsx mode change 100644 => 100755 src/components/mermaid-diagram.test.tsx mode change 100644 => 100755 src/components/mermaid-diagram.tsx mode change 100644 => 100755 src/components/project/create-project-dialog.test.ts mode change 100644 => 100755 src/components/project/create-project-dialog.tsx mode change 100644 => 100755 src/components/project/template-picker.tsx mode change 100644 => 100755 src/components/project/welcome-screen.tsx mode change 100644 => 100755 src/components/review/review-view.tsx mode change 100644 => 100755 src/components/search/search-view.tsx mode change 100644 => 100755 src/components/settings/context-size-selector.tsx mode change 100644 => 100755 src/components/settings/llm-presets.ts mode change 100644 => 100755 src/components/settings/preset-resolver.test.ts mode change 100644 => 100755 src/components/settings/preset-resolver.ts mode change 100644 => 100755 src/components/settings/sections/about-section.tsx mode change 100644 => 100755 src/components/settings/sections/api-server-section.test.ts mode change 100644 => 100755 src/components/settings/sections/api-server-section.tsx mode change 100644 => 100755 src/components/settings/sections/changelog-section.tsx mode change 100644 => 100755 src/components/settings/sections/embedding-section.tsx mode change 100644 => 100755 src/components/settings/sections/general-section.tsx mode change 100644 => 100755 src/components/settings/sections/interface-section.tsx mode change 100644 => 100755 src/components/settings/sections/llm-provider-section.tsx mode change 100644 => 100755 src/components/settings/sections/maintenance-section.tsx mode change 100644 => 100755 src/components/settings/sections/mineru-section.tsx mode change 100644 => 100755 src/components/settings/sections/multimodal-section.tsx mode change 100644 => 100755 src/components/settings/sections/network-section.tsx mode change 100644 => 100755 src/components/settings/sections/output-section.tsx mode change 100644 => 100755 src/components/settings/sections/scheduled-import-section.tsx mode change 100644 => 100755 src/components/settings/sections/source-watch-section.tsx mode change 100644 => 100755 src/components/settings/sections/web-search-section.tsx mode change 100644 => 100755 src/components/settings/settings-types.ts mode change 100644 => 100755 src/components/settings/settings-view.tsx mode change 100644 => 100755 src/components/sources/sources-view.tsx mode change 100644 => 100755 src/components/ui/button.tsx mode change 100644 => 100755 src/components/ui/dialog.tsx mode change 100644 => 100755 src/components/ui/input.tsx mode change 100644 => 100755 src/components/ui/label.tsx mode change 100644 => 100755 src/components/ui/resizable.tsx mode change 100644 => 100755 src/components/ui/scroll-area.tsx mode change 100644 => 100755 src/components/ui/separator.tsx mode change 100644 => 100755 src/components/ui/tooltip.tsx mode change 100644 => 100755 src/i18n/en.json mode change 100644 => 100755 src/i18n/i18n-parity.test.ts mode change 100644 => 100755 src/i18n/index.ts mode change 100644 => 100755 src/i18n/zh.json mode change 100644 => 100755 src/index.css mode change 100644 => 100755 src/lib/__tests__/claude-cli-transport.test.ts mode change 100644 => 100755 src/lib/__tests__/dedup_embedding.test.ts mode change 100644 => 100755 src/lib/__tests__/llm-providers.test.ts mode change 100644 => 100755 src/lib/anytxt-search.test.ts mode change 100644 => 100755 src/lib/anytxt-search.ts mode change 100644 => 100755 src/lib/api-server-constants.ts mode change 100644 => 100755 src/lib/api-server.real-llm.test.ts mode change 100644 => 100755 src/lib/api-token.test.ts mode change 100644 => 100755 src/lib/api-token.ts mode change 100644 => 100755 src/lib/auto-save.test.ts mode change 100644 => 100755 src/lib/auto-save.ts mode change 100644 => 100755 src/lib/azure-openai.test.ts mode change 100644 => 100755 src/lib/azure-openai.ts mode change 100644 => 100755 src/lib/changelog.ts mode change 100644 => 100755 src/lib/chat-image-utils.ts mode change 100644 => 100755 src/lib/chat-save-to-wiki.test.ts mode change 100644 => 100755 src/lib/chat-save-to-wiki.ts mode change 100644 => 100755 src/lib/claude-cli-transport.ts mode change 100644 => 100755 src/lib/clip-watcher.ts mode change 100644 => 100755 src/lib/codex-cli-transport.test.ts mode change 100644 => 100755 src/lib/codex-cli-transport.ts mode change 100644 => 100755 src/lib/connection-tests.test.ts mode change 100644 => 100755 src/lib/connection-tests.ts mode change 100644 => 100755 src/lib/context-budget.test.ts mode change 100644 => 100755 src/lib/context-budget.ts mode change 100644 => 100755 src/lib/dedup-queue.test.ts mode change 100644 => 100755 src/lib/dedup-queue.ts mode change 100644 => 100755 src/lib/dedup-runner.test.ts mode change 100644 => 100755 src/lib/dedup-runner.ts mode change 100644 => 100755 src/lib/dedup-storage.ts mode change 100644 => 100755 src/lib/dedup.test.ts mode change 100644 => 100755 src/lib/dedup.ts mode change 100644 => 100755 src/lib/dedup_embedding.ts mode change 100644 => 100755 src/lib/deep-research.test.ts mode change 100644 => 100755 src/lib/deep-research.ts mode change 100644 => 100755 src/lib/detect-language.property.test.ts mode change 100644 => 100755 src/lib/detect-language.test.ts mode change 100644 => 100755 src/lib/detect-language.ts mode change 100644 => 100755 src/lib/embedding.real-llm.test.ts mode change 100644 => 100755 src/lib/embedding.test.ts mode change 100644 => 100755 src/lib/embedding.ts mode change 100644 => 100755 src/lib/endpoint-normalizer.test.ts mode change 100644 => 100755 src/lib/endpoint-normalizer.ts mode change 100644 => 100755 src/lib/enrich-wikilinks.real-llm.test.ts mode change 100644 => 100755 src/lib/enrich-wikilinks.scenarios.test.ts mode change 100644 => 100755 src/lib/enrich-wikilinks.test.ts mode change 100644 => 100755 src/lib/enrich-wikilinks.ts mode change 100644 => 100755 src/lib/extract-source-images.test.ts mode change 100644 => 100755 src/lib/extract-source-images.ts mode change 100644 => 100755 src/lib/file-types.test.ts mode change 100644 => 100755 src/lib/file-types.ts mode change 100644 => 100755 src/lib/frontmatter.test.ts mode change 100644 => 100755 src/lib/frontmatter.ts mode change 100644 => 100755 src/lib/graph-filters.test.ts mode change 100644 => 100755 src/lib/graph-filters.ts mode change 100644 => 100755 src/lib/graph-insights.ts mode change 100644 => 100755 src/lib/graph-relevance.ts mode change 100644 => 100755 src/lib/graph-search.test.ts mode change 100644 => 100755 src/lib/graph-search.ts mode change 100644 => 100755 src/lib/graph-visibility.test.ts mode change 100644 => 100755 src/lib/graph-visibility.ts mode change 100644 => 100755 src/lib/greeting-detector.test.ts mode change 100644 => 100755 src/lib/greeting-detector.ts mode change 100644 => 100755 src/lib/has-usable-llm.test.ts mode change 100644 => 100755 src/lib/has-usable-llm.ts mode change 100644 => 100755 src/lib/image-caption-pipeline.test.ts mode change 100644 => 100755 src/lib/image-caption-pipeline.ts mode change 100644 => 100755 src/lib/ingest-queue.integration.test.ts mode change 100644 => 100755 src/lib/ingest-queue.test.ts mode change 100644 => 100755 src/lib/ingest-queue.ts mode change 100644 => 100755 src/lib/ingest-sanitize.test.ts mode change 100644 => 100755 src/lib/ingest-sanitize.ts mode change 100644 => 100755 src/lib/ingest-source-path-collision.test.ts mode change 100644 => 100755 src/lib/ingest.prompt.test.ts mode change 100644 => 100755 src/lib/ingest.real-llm.test.ts mode change 100644 => 100755 src/lib/ingest.scenarios.test.ts mode change 100644 => 100755 src/lib/keyboard-utils.test.ts mode change 100644 => 100755 src/lib/keyboard-utils.ts mode change 100644 => 100755 src/lib/language-metadata.test.ts mode change 100644 => 100755 src/lib/language-metadata.ts mode change 100644 => 100755 src/lib/latex-to-unicode.ts mode change 100644 => 100755 src/lib/lint-fixes.test.ts mode change 100644 => 100755 src/lib/lint-fixes.ts mode change 100644 => 100755 src/lib/lint.real-llm.test.ts mode change 100644 => 100755 src/lib/lint.scenarios.test.ts mode change 100644 => 100755 src/lib/lint.test.ts mode change 100644 => 100755 src/lib/lint.ts mode change 100644 => 100755 src/lib/llm-client.real-llm.test.ts mode change 100644 => 100755 src/lib/llm-client.test.ts mode change 100644 => 100755 src/lib/llm-client.ts mode change 100644 => 100755 src/lib/llm-providers.test.ts mode change 100644 => 100755 src/lib/llm-providers.ts mode change 100644 => 100755 src/lib/markdown-image-resolver.test.ts mode change 100644 => 100755 src/lib/markdown-image-resolver.ts mode change 100644 => 100755 src/lib/mineru.test.ts mode change 100644 => 100755 src/lib/mineru.ts mode change 100644 => 100755 src/lib/optimize-research-topic.test.ts mode change 100644 => 100755 src/lib/optimize-research-topic.ts mode change 100644 => 100755 src/lib/output-language-options.ts mode change 100644 => 100755 src/lib/output-language.test.ts mode change 100644 => 100755 src/lib/output-language.ts mode change 100644 => 100755 src/lib/page-merge.test.ts mode change 100644 => 100755 src/lib/page-merge.ts mode change 100644 => 100755 src/lib/path-utils.property.test.ts mode change 100644 => 100755 src/lib/path-utils.test.ts mode change 100644 => 100755 src/lib/path-utils.ts mode change 100644 => 100755 src/lib/persist.integration.test.ts mode change 100644 => 100755 src/lib/persist.ts mode change 100644 => 100755 src/lib/project-file-sync.test.ts mode change 100644 => 100755 src/lib/project-file-sync.ts mode change 100644 => 100755 src/lib/project-identity.ts mode change 100644 => 100755 src/lib/project-mutex.test.ts mode change 100644 => 100755 src/lib/project-mutex.ts mode change 100644 => 100755 src/lib/project-store.test.ts mode change 100644 => 100755 src/lib/project-store.ts mode change 100644 => 100755 src/lib/proxy-config.test.ts mode change 100644 => 100755 src/lib/proxy-config.ts mode change 100644 => 100755 src/lib/raw-source-resolver.test.ts mode change 100644 => 100755 src/lib/raw-source-resolver.ts mode change 100644 => 100755 src/lib/reasoning-detector.test.ts mode change 100644 => 100755 src/lib/reasoning-detector.ts mode change 100644 => 100755 src/lib/reset-project-state.test.ts mode change 100644 => 100755 src/lib/reset-project-state.ts mode change 100644 => 100755 src/lib/review-create-page.test.ts mode change 100644 => 100755 src/lib/review-create-page.ts mode change 100644 => 100755 src/lib/review-utils.property.test.ts mode change 100644 => 100755 src/lib/review-utils.test.ts mode change 100644 => 100755 src/lib/review-utils.ts mode change 100644 => 100755 src/lib/scheduled-import.test.ts mode change 100644 => 100755 src/lib/scheduled-import.ts mode change 100644 => 100755 src/lib/search-rrf.test.ts mode change 100644 => 100755 src/lib/search.scenarios.test.ts mode change 100644 => 100755 src/lib/search.ts mode change 100644 => 100755 src/lib/source-delete-decision.test.ts mode change 100644 => 100755 src/lib/source-delete-decision.ts mode change 100644 => 100755 src/lib/source-identity.test.ts mode change 100644 => 100755 src/lib/source-identity.ts mode change 100644 => 100755 src/lib/source-lifecycle-delete.test.ts mode change 100644 => 100755 src/lib/source-lifecycle.test.ts mode change 100644 => 100755 src/lib/source-lifecycle.ts mode change 100644 => 100755 src/lib/source-watch-config.test.ts mode change 100644 => 100755 src/lib/source-watch-config.ts mode change 100644 => 100755 src/lib/source-watch-defaults.json mode change 100644 => 100755 src/lib/sources-merge.test.ts mode change 100644 => 100755 src/lib/sources-merge.ts mode change 100644 => 100755 src/lib/sources-tree-delete.test.ts mode change 100644 => 100755 src/lib/sources-tree-delete.ts mode change 100644 => 100755 src/lib/sweep-chained.real-llm.test.ts mode change 100644 => 100755 src/lib/sweep-reviews.property.test.ts mode change 100644 => 100755 src/lib/sweep-reviews.race.test.ts mode change 100644 => 100755 src/lib/sweep-reviews.real-llm.test.ts mode change 100644 => 100755 src/lib/sweep-reviews.scenarios.test.ts mode change 100644 => 100755 src/lib/sweep-reviews.test.ts mode change 100644 => 100755 src/lib/sweep-reviews.ts mode change 100644 => 100755 src/lib/tauri-fetch.test.ts mode change 100644 => 100755 src/lib/tauri-fetch.ts mode change 100644 => 100755 src/lib/templates.ts mode change 100644 => 100755 src/lib/text-chunker.test.ts mode change 100644 => 100755 src/lib/text-chunker.ts mode change 100644 => 100755 src/lib/theme.ts mode change 100644 => 100755 src/lib/update-check.test.ts mode change 100644 => 100755 src/lib/update-check.ts mode change 100644 => 100755 src/lib/utils.ts mode change 100644 => 100755 src/lib/vision-caption.real-llm.test.ts mode change 100644 => 100755 src/lib/vision-caption.test.ts mode change 100644 => 100755 src/lib/vision-caption.ts mode change 100644 => 100755 src/lib/vision.real-llm.test.ts mode change 100644 => 100755 src/lib/web-search.test.ts mode change 100644 => 100755 src/lib/web-search.ts mode change 100644 => 100755 src/lib/wiki-cleanup.test.ts mode change 100644 => 100755 src/lib/wiki-cleanup.ts mode change 100644 => 100755 src/lib/wiki-filename.test.ts mode change 100644 => 100755 src/lib/wiki-filename.ts mode change 100644 => 100755 src/lib/wiki-graph.ts mode change 100644 => 100755 src/lib/wiki-page-delete.test.ts mode change 100644 => 100755 src/lib/wiki-page-delete.ts mode change 100644 => 100755 src/lib/wiki-page-resolver.test.ts mode change 100644 => 100755 src/lib/wiki-page-resolver.ts mode change 100644 => 100755 src/lib/wiki-page-types.test.ts mode change 100644 => 100755 src/lib/wiki-page-types.ts mode change 100644 => 100755 src/lib/wiki-schema.test.ts mode change 100644 => 100755 src/lib/wiki-schema.ts mode change 100644 => 100755 src/lib/wiki-type-style.test.ts mode change 100644 => 100755 src/lib/wiki-type-style.ts mode change 100644 => 100755 src/lib/wikilink-transform.test.ts mode change 100644 => 100755 src/lib/wikilink-transform.ts mode change 100644 => 100755 src/main.tsx mode change 100644 => 100755 src/stores/activity-store.ts mode change 100644 => 100755 src/stores/chat-messages-to-llm.test.ts mode change 100644 => 100755 src/stores/chat-store.ts mode change 100644 => 100755 src/stores/file-sync-store.ts mode change 100644 => 100755 src/stores/lint-store.test.ts mode change 100644 => 100755 src/stores/lint-store.ts mode change 100644 => 100755 src/stores/research-store.ts mode change 100644 => 100755 src/stores/review-store.property.test.ts mode change 100644 => 100755 src/stores/review-store.test.ts mode change 100644 => 100755 src/stores/review-store.ts mode change 100644 => 100755 src/stores/update-store.ts mode change 100644 => 100755 src/stores/wiki-store.test.ts mode change 100644 => 100755 src/stores/wiki-store.ts mode change 100644 => 100755 src/stores/zoom-store.ts mode change 100644 => 100755 src/test-helpers/deferred.ts mode change 100644 => 100755 src/test-helpers/fs-temp.ts mode change 100644 => 100755 src/test-helpers/load-test-env.ts mode change 100644 => 100755 src/test-helpers/mock-stream-chat.ts mode change 100644 => 100755 src/test-helpers/real-content.ts mode change 100644 => 100755 src/test-helpers/scenarios/enrich-scenarios.ts mode change 100644 => 100755 src/test-helpers/scenarios/ingest-scenarios.ts mode change 100644 => 100755 src/test-helpers/scenarios/lint-scenarios.ts mode change 100644 => 100755 src/test-helpers/scenarios/materialize.ts mode change 100644 => 100755 src/test-helpers/scenarios/search-scenarios.ts mode change 100644 => 100755 src/test-helpers/scenarios/sweep-scenarios.ts mode change 100644 => 100755 src/test-helpers/scenarios/types.ts mode change 100644 => 100755 src/types/wiki.ts mode change 100644 => 100755 src/vite-env.d.ts mode change 100644 => 100755 tsconfig.app.json mode change 100644 => 100755 tsconfig.json mode change 100644 => 100755 tsconfig.node.json mode change 100644 => 100755 vite.config.ts diff --git a/.gitattributes b/.gitattributes new file mode 100755 index 000000000..c0b85c760 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# 统一行尾符为 LF，避免 Windows/WSL 跨平台换行符问题 +* text=auto eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml old mode 100644 new mode 100755 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/README_CN.md b/README_CN.md old mode 100644 new mode 100755 diff --git a/README_JA.md b/README_JA.md old mode 100644 new mode 100755 diff --git a/README_KO.md b/README_KO.md old mode 100644 new mode 100755 diff --git a/assets/1-deepresearch.jpg b/assets/1-deepresearch.jpg old mode 100644 new mode 100755 diff --git a/assets/2-ai_chat.jpg b/assets/2-ai_chat.jpg old mode 100644 new mode 100755 diff --git a/assets/3-knowledge_graph.jpg b/assets/3-knowledge_graph.jpg old mode 100644 new mode 100755 diff --git a/assets/4-chrome_extension_webclipper.jpg b/assets/4-chrome_extension_webclipper.jpg old mode 100644 new mode 100755 diff --git a/assets/5-obsidian_compatibility.jpg b/assets/5-obsidian_compatibility.jpg old mode 100644 new mode 100755 diff --git a/assets/kg_community.jpg b/assets/kg_community.jpg old mode 100644 new mode 100755 diff --git a/assets/kg_insights.jpg b/assets/kg_insights.jpg old mode 100644 new mode 100755 diff --git a/assets/llm_wiki_arch.jpg b/assets/llm_wiki_arch.jpg old mode 100644 new mode 100755 diff --git a/assets/overview.jpg b/assets/overview.jpg old mode 100644 new mode 100755 diff --git a/components.json b/components.json old mode 100644 new mode 100755 diff --git a/extension/Readability.js b/extension/Readability.js old mode 100644 new mode 100755 diff --git a/extension/Turndown.js b/extension/Turndown.js old mode 100644 new mode 100755 diff --git a/extension/icon128.png b/extension/icon128.png old mode 100644 new mode 100755 diff --git a/extension/icon16.png b/extension/icon16.png old mode 100644 new mode 100755 diff --git a/extension/icon48.png b/extension/icon48.png old mode 100644 new mode 100755 diff --git a/extension/manifest.json b/extension/manifest.json old mode 100644 new mode 100755 diff --git a/extension/popup.html b/extension/popup.html old mode 100644 new mode 100755 diff --git a/extension/popup.js b/extension/popup.js old mode 100644 new mode 100755 diff --git a/index.html b/index.html old mode 100644 new mode 100755 diff --git a/llm-wiki.md b/llm-wiki.md old mode 100644 new mode 100755 diff --git a/logo.jpg b/logo.jpg old mode 100644 new mode 100755 diff --git a/mcp-server/README.md b/mcp-server/README.md old mode 100644 new mode 100755 diff --git a/mcp-server/package-lock.json b/mcp-server/package-lock.json old mode 100644 new mode 100755 diff --git a/mcp-server/package.json b/mcp-server/package.json old mode 100644 new mode 100755 diff --git a/mcp-server/src/api-client.ts b/mcp-server/src/api-client.ts old mode 100644 new mode 100755 diff --git a/mcp-server/src/index.ts b/mcp-server/src/index.ts old mode 100644 new mode 100755 diff --git a/mcp-server/src/version.ts b/mcp-server/src/version.ts old mode 100644 new mode 100755 diff --git a/mcp-server/test/api-client.test.ts b/mcp-server/test/api-client.test.ts old mode 100644 new mode 100755 diff --git a/mcp-server/test/version.test.ts b/mcp-server/test/version.test.ts old mode 100644 new mode 100755 diff --git a/mcp-server/tsconfig.json b/mcp-server/tsconfig.json old mode 100644 new mode 100755 diff --git a/package-lock.json b/package-lock.json old mode 100644 new mode 100755 diff --git a/package.json b/package.json old mode 100644 new mode 100755 diff --git a/plans/multimodal-images.md b/plans/multimodal-images.md old mode 100644 new mode 100755 diff --git a/scripts/debug_ollama_tokens.py b/scripts/debug_ollama_tokens.py old mode 100644 new mode 100755 diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock old mode 100644 new mode 100755 diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml old mode 100644 new mode 100755 diff --git a/src-tauri/build.rs b/src-tauri/build.rs old mode 100644 new mode 100755 diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json old mode 100644 new mode 100755 diff --git a/src-tauri/icons/128x128.png b/src-tauri/icons/128x128.png old mode 100644 new mode 100755 diff --git a/src-tauri/icons/128x128@2x.png b/src-tauri/icons/128x128@2x.png old mode 100644 new mode 100755 diff --git a/src-tauri/icons/32x32.png b/src-tauri/icons/32x32.png old mode 100644 new mode 100755 diff --git a/src-tauri/icons/icon.icns b/src-tauri/icons/icon.icns old mode 100644 new mode 100755 diff --git a/src-tauri/icons/icon.ico b/src-tauri/icons/icon.ico old mode 100644 new mode 100755 diff --git a/src-tauri/icons/icon.png b/src-tauri/icons/icon.png old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/SHA256SUMS b/src-tauri/pdfium/SHA256SUMS old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium-arm64.so b/src-tauri/pdfium/libpdfium-arm64.so old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium-x86_64.dylib b/src-tauri/pdfium/libpdfium-x86_64.dylib old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium.dylib b/src-tauri/pdfium/libpdfium.dylib old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/libpdfium.so b/src-tauri/pdfium/libpdfium.so old mode 100644 new mode 100755 diff --git a/src-tauri/pdfium/pdfium.dll b/src-tauri/pdfium/pdfium.dll old mode 100644 new mode 100755 diff --git a/src-tauri/src/api_server.rs b/src-tauri/src/api_server.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/clip_server.rs b/src-tauri/src/clip_server.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/claude_cli.rs b/src-tauri/src/commands/claude_cli.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/cli_resolver.rs b/src-tauri/src/commands/cli_resolver.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/codex_cli.rs b/src-tauri/src/commands/codex_cli.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/extract_images.rs b/src-tauri/src/commands/extract_images.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/file_sync.rs b/src-tauri/src/commands/file_sync.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/fs.rs b/src-tauri/src/commands/fs.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/project.rs b/src-tauri/src/commands/project.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/search.rs b/src-tauri/src/commands/search.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/commands/vectorstore.rs b/src-tauri/src/commands/vectorstore.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/panic_guard.rs b/src-tauri/src/panic_guard.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/proxy.rs b/src-tauri/src/proxy.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/tray.rs b/src-tauri/src/tray.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/types/mod.rs b/src-tauri/src/types/mod.rs old mode 100644 new mode 100755 diff --git a/src-tauri/src/types/wiki.rs b/src-tauri/src/types/wiki.rs old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.linux.conf.json b/src-tauri/tauri.linux.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.macos.conf.json b/src-tauri/tauri.macos.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/tauri.windows.conf.json b/src-tauri/tauri.windows.conf.json old mode 100644 new mode 100755 diff --git a/src-tauri/windows-app-manifest.xml b/src-tauri/windows-app-manifest.xml old mode 100644 new mode 100755 diff --git a/src/App.tsx b/src/App.tsx old mode 100644 new mode 100755 diff --git a/src/assets/logo.jpg b/src/assets/logo.jpg old mode 100644 new mode 100755 diff --git a/src/commands/file-sync.ts b/src/commands/file-sync.ts old mode 100644 new mode 100755 diff --git a/src/commands/fs.test.ts b/src/commands/fs.test.ts old mode 100644 new mode 100755 diff --git a/src/commands/fs.ts b/src/commands/fs.ts old mode 100644 new mode 100755 diff --git a/src/components/chat/chat-input.tsx b/src/components/chat/chat-input.tsx old mode 100644 new mode 100755 diff --git a/src/components/chat/chat-message.tsx b/src/components/chat/chat-message.tsx old mode 100644 new mode 100755 diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/file-preview.tsx b/src/components/editor/file-preview.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/frontmatter-panel.tsx b/src/components/editor/frontmatter-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/wiki-editor.tsx b/src/components/editor/wiki-editor.tsx old mode 100644 new mode 100755 diff --git a/src/components/editor/wiki-reader.tsx b/src/components/editor/wiki-reader.tsx old mode 100644 new mode 100755 diff --git a/src/components/error-boundary.tsx b/src/components/error-boundary.tsx old mode 100644 new mode 100755 diff --git a/src/components/graph/graph-layout-worker.ts b/src/components/graph/graph-layout-worker.ts old mode 100644 new mode 100755 diff --git a/src/components/graph/graph-view.tsx b/src/components/graph/graph-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/activity-panel.tsx b/src/components/layout/activity-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/app-layout-visibility.test.ts b/src/components/layout/app-layout-visibility.test.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/app-layout-visibility.ts b/src/components/layout/app-layout-visibility.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/app-layout.tsx b/src/components/layout/app-layout.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/content-area.tsx b/src/components/layout/content-area.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/file-tree.tsx b/src/components/layout/file-tree.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/icon-sidebar.tsx b/src/components/layout/icon-sidebar.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/knowledge-tree.tsx b/src/components/layout/knowledge-tree.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/preview-panel.tsx b/src/components/layout/preview-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/research-panel-nav.test.ts b/src/components/layout/research-panel-nav.test.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/research-panel-nav.ts b/src/components/layout/research-panel-nav.ts old mode 100644 new mode 100755 diff --git a/src/components/layout/research-panel.tsx b/src/components/layout/research-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/sidebar-panel.tsx b/src/components/layout/sidebar-panel.tsx old mode 100644 new mode 100755 diff --git a/src/components/layout/update-banner.tsx b/src/components/layout/update-banner.tsx old mode 100644 new mode 100755 diff --git a/src/components/lint/lint-view.test.ts b/src/components/lint/lint-view.test.ts old mode 100644 new mode 100755 diff --git a/src/components/lint/lint-view.tsx b/src/components/lint/lint-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/mermaid-diagram.test.tsx b/src/components/mermaid-diagram.test.tsx old mode 100644 new mode 100755 diff --git a/src/components/mermaid-diagram.tsx b/src/components/mermaid-diagram.tsx old mode 100644 new mode 100755 diff --git a/src/components/project/create-project-dialog.test.ts b/src/components/project/create-project-dialog.test.ts old mode 100644 new mode 100755 diff --git a/src/components/project/create-project-dialog.tsx b/src/components/project/create-project-dialog.tsx old mode 100644 new mode 100755 diff --git a/src/components/project/template-picker.tsx b/src/components/project/template-picker.tsx old mode 100644 new mode 100755 diff --git a/src/components/project/welcome-screen.tsx b/src/components/project/welcome-screen.tsx old mode 100644 new mode 100755 diff --git a/src/components/review/review-view.tsx b/src/components/review/review-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/search/search-view.tsx b/src/components/search/search-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/context-size-selector.tsx b/src/components/settings/context-size-selector.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/llm-presets.ts b/src/components/settings/llm-presets.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/preset-resolver.test.ts b/src/components/settings/preset-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/preset-resolver.ts b/src/components/settings/preset-resolver.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/about-section.tsx b/src/components/settings/sections/about-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/api-server-section.test.ts b/src/components/settings/sections/api-server-section.test.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/api-server-section.tsx b/src/components/settings/sections/api-server-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/changelog-section.tsx b/src/components/settings/sections/changelog-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/embedding-section.tsx b/src/components/settings/sections/embedding-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/general-section.tsx b/src/components/settings/sections/general-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/interface-section.tsx b/src/components/settings/sections/interface-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/llm-provider-section.tsx b/src/components/settings/sections/llm-provider-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/maintenance-section.tsx b/src/components/settings/sections/maintenance-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/mineru-section.tsx b/src/components/settings/sections/mineru-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/multimodal-section.tsx b/src/components/settings/sections/multimodal-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/network-section.tsx b/src/components/settings/sections/network-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/output-section.tsx b/src/components/settings/sections/output-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/scheduled-import-section.tsx b/src/components/settings/sections/scheduled-import-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/source-watch-section.tsx b/src/components/settings/sections/source-watch-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/sections/web-search-section.tsx b/src/components/settings/sections/web-search-section.tsx old mode 100644 new mode 100755 diff --git a/src/components/settings/settings-types.ts b/src/components/settings/settings-types.ts old mode 100644 new mode 100755 diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/sources/sources-view.tsx b/src/components/sources/sources-view.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/button.tsx b/src/components/ui/button.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/dialog.tsx b/src/components/ui/dialog.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/input.tsx b/src/components/ui/input.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/label.tsx b/src/components/ui/label.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/resizable.tsx b/src/components/ui/resizable.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/scroll-area.tsx b/src/components/ui/scroll-area.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/separator.tsx b/src/components/ui/separator.tsx old mode 100644 new mode 100755 diff --git a/src/components/ui/tooltip.tsx b/src/components/ui/tooltip.tsx old mode 100644 new mode 100755 diff --git a/src/i18n/en.json b/src/i18n/en.json old mode 100644 new mode 100755 diff --git a/src/i18n/i18n-parity.test.ts b/src/i18n/i18n-parity.test.ts old mode 100644 new mode 100755 diff --git a/src/i18n/index.ts b/src/i18n/index.ts old mode 100644 new mode 100755 diff --git a/src/i18n/zh.json b/src/i18n/zh.json old mode 100644 new mode 100755 diff --git a/src/index.css b/src/index.css old mode 100644 new mode 100755 diff --git a/src/lib/__tests__/claude-cli-transport.test.ts b/src/lib/__tests__/claude-cli-transport.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/__tests__/dedup_embedding.test.ts b/src/lib/__tests__/dedup_embedding.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/__tests__/llm-providers.test.ts b/src/lib/__tests__/llm-providers.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/anytxt-search.test.ts b/src/lib/anytxt-search.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/anytxt-search.ts b/src/lib/anytxt-search.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-server-constants.ts b/src/lib/api-server-constants.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-server.real-llm.test.ts b/src/lib/api-server.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-token.test.ts b/src/lib/api-token.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/api-token.ts b/src/lib/api-token.ts old mode 100644 new mode 100755 diff --git a/src/lib/auto-save.test.ts b/src/lib/auto-save.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/auto-save.ts b/src/lib/auto-save.ts old mode 100644 new mode 100755 diff --git a/src/lib/azure-openai.test.ts b/src/lib/azure-openai.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/azure-openai.ts b/src/lib/azure-openai.ts old mode 100644 new mode 100755 diff --git a/src/lib/changelog.ts b/src/lib/changelog.ts old mode 100644 new mode 100755 diff --git a/src/lib/chat-image-utils.ts b/src/lib/chat-image-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/chat-save-to-wiki.test.ts b/src/lib/chat-save-to-wiki.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/chat-save-to-wiki.ts b/src/lib/chat-save-to-wiki.ts old mode 100644 new mode 100755 diff --git a/src/lib/claude-cli-transport.ts b/src/lib/claude-cli-transport.ts old mode 100644 new mode 100755 diff --git a/src/lib/clip-watcher.ts b/src/lib/clip-watcher.ts old mode 100644 new mode 100755 diff --git a/src/lib/codex-cli-transport.test.ts b/src/lib/codex-cli-transport.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/codex-cli-transport.ts b/src/lib/codex-cli-transport.ts old mode 100644 new mode 100755 diff --git a/src/lib/connection-tests.test.ts b/src/lib/connection-tests.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/connection-tests.ts b/src/lib/connection-tests.ts old mode 100644 new mode 100755 diff --git a/src/lib/context-budget.test.ts b/src/lib/context-budget.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/context-budget.ts b/src/lib/context-budget.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-queue.test.ts b/src/lib/dedup-queue.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-queue.ts b/src/lib/dedup-queue.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-runner.test.ts b/src/lib/dedup-runner.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-runner.ts b/src/lib/dedup-runner.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup-storage.ts b/src/lib/dedup-storage.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup.test.ts b/src/lib/dedup.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup.ts b/src/lib/dedup.ts old mode 100644 new mode 100755 diff --git a/src/lib/dedup_embedding.ts b/src/lib/dedup_embedding.ts old mode 100644 new mode 100755 diff --git a/src/lib/deep-research.test.ts b/src/lib/deep-research.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/deep-research.ts b/src/lib/deep-research.ts old mode 100644 new mode 100755 diff --git a/src/lib/detect-language.property.test.ts b/src/lib/detect-language.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/detect-language.test.ts b/src/lib/detect-language.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/detect-language.ts b/src/lib/detect-language.ts old mode 100644 new mode 100755 diff --git a/src/lib/embedding.real-llm.test.ts b/src/lib/embedding.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/embedding.test.ts b/src/lib/embedding.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/embedding.ts b/src/lib/embedding.ts old mode 100644 new mode 100755 diff --git a/src/lib/endpoint-normalizer.test.ts b/src/lib/endpoint-normalizer.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/endpoint-normalizer.ts b/src/lib/endpoint-normalizer.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.real-llm.test.ts b/src/lib/enrich-wikilinks.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.scenarios.test.ts b/src/lib/enrich-wikilinks.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.test.ts b/src/lib/enrich-wikilinks.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/enrich-wikilinks.ts b/src/lib/enrich-wikilinks.ts old mode 100644 new mode 100755 diff --git a/src/lib/extract-source-images.test.ts b/src/lib/extract-source-images.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/extract-source-images.ts b/src/lib/extract-source-images.ts old mode 100644 new mode 100755 diff --git a/src/lib/file-types.test.ts b/src/lib/file-types.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/file-types.ts b/src/lib/file-types.ts old mode 100644 new mode 100755 diff --git a/src/lib/frontmatter.test.ts b/src/lib/frontmatter.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/frontmatter.ts b/src/lib/frontmatter.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-filters.test.ts b/src/lib/graph-filters.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-filters.ts b/src/lib/graph-filters.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-insights.ts b/src/lib/graph-insights.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-relevance.ts b/src/lib/graph-relevance.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-search.test.ts b/src/lib/graph-search.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-search.ts b/src/lib/graph-search.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-visibility.test.ts b/src/lib/graph-visibility.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/graph-visibility.ts b/src/lib/graph-visibility.ts old mode 100644 new mode 100755 diff --git a/src/lib/greeting-detector.test.ts b/src/lib/greeting-detector.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/greeting-detector.ts b/src/lib/greeting-detector.ts old mode 100644 new mode 100755 diff --git a/src/lib/has-usable-llm.test.ts b/src/lib/has-usable-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/has-usable-llm.ts b/src/lib/has-usable-llm.ts old mode 100644 new mode 100755 diff --git a/src/lib/image-caption-pipeline.test.ts b/src/lib/image-caption-pipeline.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/image-caption-pipeline.ts b/src/lib/image-caption-pipeline.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-queue.integration.test.ts b/src/lib/ingest-queue.integration.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-queue.test.ts b/src/lib/ingest-queue.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-queue.ts b/src/lib/ingest-queue.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-sanitize.test.ts b/src/lib/ingest-sanitize.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-sanitize.ts b/src/lib/ingest-sanitize.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest-source-path-collision.test.ts b/src/lib/ingest-source-path-collision.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest.prompt.test.ts b/src/lib/ingest.prompt.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest.real-llm.test.ts b/src/lib/ingest.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/ingest.scenarios.test.ts b/src/lib/ingest.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/keyboard-utils.test.ts b/src/lib/keyboard-utils.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/keyboard-utils.ts b/src/lib/keyboard-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/language-metadata.test.ts b/src/lib/language-metadata.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/language-metadata.ts b/src/lib/language-metadata.ts old mode 100644 new mode 100755 diff --git a/src/lib/latex-to-unicode.ts b/src/lib/latex-to-unicode.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint-fixes.test.ts b/src/lib/lint-fixes.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint-fixes.ts b/src/lib/lint-fixes.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.real-llm.test.ts b/src/lib/lint.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.scenarios.test.ts b/src/lib/lint.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.test.ts b/src/lib/lint.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/lint.ts b/src/lib/lint.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-client.real-llm.test.ts b/src/lib/llm-client.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-client.test.ts b/src/lib/llm-client.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-client.ts b/src/lib/llm-client.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-providers.test.ts b/src/lib/llm-providers.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/llm-providers.ts b/src/lib/llm-providers.ts old mode 100644 new mode 100755 diff --git a/src/lib/markdown-image-resolver.test.ts b/src/lib/markdown-image-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/markdown-image-resolver.ts b/src/lib/markdown-image-resolver.ts old mode 100644 new mode 100755 diff --git a/src/lib/mineru.test.ts b/src/lib/mineru.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/mineru.ts b/src/lib/mineru.ts old mode 100644 new mode 100755 diff --git a/src/lib/optimize-research-topic.test.ts b/src/lib/optimize-research-topic.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/optimize-research-topic.ts b/src/lib/optimize-research-topic.ts old mode 100644 new mode 100755 diff --git a/src/lib/output-language-options.ts b/src/lib/output-language-options.ts old mode 100644 new mode 100755 diff --git a/src/lib/output-language.test.ts b/src/lib/output-language.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/output-language.ts b/src/lib/output-language.ts old mode 100644 new mode 100755 diff --git a/src/lib/page-merge.test.ts b/src/lib/page-merge.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/page-merge.ts b/src/lib/page-merge.ts old mode 100644 new mode 100755 diff --git a/src/lib/path-utils.property.test.ts b/src/lib/path-utils.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/path-utils.test.ts b/src/lib/path-utils.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/path-utils.ts b/src/lib/path-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/persist.integration.test.ts b/src/lib/persist.integration.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/persist.ts b/src/lib/persist.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-file-sync.test.ts b/src/lib/project-file-sync.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-file-sync.ts b/src/lib/project-file-sync.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-identity.ts b/src/lib/project-identity.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-mutex.test.ts b/src/lib/project-mutex.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-mutex.ts b/src/lib/project-mutex.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-store.test.ts b/src/lib/project-store.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/project-store.ts b/src/lib/project-store.ts old mode 100644 new mode 100755 diff --git a/src/lib/proxy-config.test.ts b/src/lib/proxy-config.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/proxy-config.ts b/src/lib/proxy-config.ts old mode 100644 new mode 100755 diff --git a/src/lib/raw-source-resolver.test.ts b/src/lib/raw-source-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/raw-source-resolver.ts b/src/lib/raw-source-resolver.ts old mode 100644 new mode 100755 diff --git a/src/lib/reasoning-detector.test.ts b/src/lib/reasoning-detector.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/reasoning-detector.ts b/src/lib/reasoning-detector.ts old mode 100644 new mode 100755 diff --git a/src/lib/reset-project-state.test.ts b/src/lib/reset-project-state.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/reset-project-state.ts b/src/lib/reset-project-state.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-create-page.test.ts b/src/lib/review-create-page.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-create-page.ts b/src/lib/review-create-page.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-utils.property.test.ts b/src/lib/review-utils.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-utils.test.ts b/src/lib/review-utils.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/review-utils.ts b/src/lib/review-utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/scheduled-import.test.ts b/src/lib/scheduled-import.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/scheduled-import.ts b/src/lib/scheduled-import.ts old mode 100644 new mode 100755 diff --git a/src/lib/search-rrf.test.ts b/src/lib/search-rrf.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/search.scenarios.test.ts b/src/lib/search.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/search.ts b/src/lib/search.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-delete-decision.test.ts b/src/lib/source-delete-decision.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-delete-decision.ts b/src/lib/source-delete-decision.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-identity.test.ts b/src/lib/source-identity.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-identity.ts b/src/lib/source-identity.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-lifecycle-delete.test.ts b/src/lib/source-lifecycle-delete.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-lifecycle.test.ts b/src/lib/source-lifecycle.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-watch-config.test.ts b/src/lib/source-watch-config.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-watch-config.ts b/src/lib/source-watch-config.ts old mode 100644 new mode 100755 diff --git a/src/lib/source-watch-defaults.json b/src/lib/source-watch-defaults.json old mode 100644 new mode 100755 diff --git a/src/lib/sources-merge.test.ts b/src/lib/sources-merge.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sources-merge.ts b/src/lib/sources-merge.ts old mode 100644 new mode 100755 diff --git a/src/lib/sources-tree-delete.test.ts b/src/lib/sources-tree-delete.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sources-tree-delete.ts b/src/lib/sources-tree-delete.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-chained.real-llm.test.ts b/src/lib/sweep-chained.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.property.test.ts b/src/lib/sweep-reviews.property.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.race.test.ts b/src/lib/sweep-reviews.race.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.real-llm.test.ts b/src/lib/sweep-reviews.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.scenarios.test.ts b/src/lib/sweep-reviews.scenarios.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.test.ts b/src/lib/sweep-reviews.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/sweep-reviews.ts b/src/lib/sweep-reviews.ts old mode 100644 new mode 100755 diff --git a/src/lib/tauri-fetch.test.ts b/src/lib/tauri-fetch.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/tauri-fetch.ts b/src/lib/tauri-fetch.ts old mode 100644 new mode 100755 diff --git a/src/lib/templates.ts b/src/lib/templates.ts old mode 100644 new mode 100755 diff --git a/src/lib/text-chunker.test.ts b/src/lib/text-chunker.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/text-chunker.ts b/src/lib/text-chunker.ts old mode 100644 new mode 100755 diff --git a/src/lib/theme.ts b/src/lib/theme.ts old mode 100644 new mode 100755 diff --git a/src/lib/update-check.test.ts b/src/lib/update-check.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/update-check.ts b/src/lib/update-check.ts old mode 100644 new mode 100755 diff --git a/src/lib/utils.ts b/src/lib/utils.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision-caption.real-llm.test.ts b/src/lib/vision-caption.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision-caption.test.ts b/src/lib/vision-caption.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision-caption.ts b/src/lib/vision-caption.ts old mode 100644 new mode 100755 diff --git a/src/lib/vision.real-llm.test.ts b/src/lib/vision.real-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/web-search.test.ts b/src/lib/web-search.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/web-search.ts b/src/lib/web-search.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-cleanup.test.ts b/src/lib/wiki-cleanup.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-cleanup.ts b/src/lib/wiki-cleanup.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-filename.test.ts b/src/lib/wiki-filename.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-filename.ts b/src/lib/wiki-filename.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-graph.ts b/src/lib/wiki-graph.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-delete.test.ts b/src/lib/wiki-page-delete.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-delete.ts b/src/lib/wiki-page-delete.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-resolver.test.ts b/src/lib/wiki-page-resolver.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-resolver.ts b/src/lib/wiki-page-resolver.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-types.test.ts b/src/lib/wiki-page-types.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-page-types.ts b/src/lib/wiki-page-types.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-schema.test.ts b/src/lib/wiki-schema.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-schema.ts b/src/lib/wiki-schema.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-type-style.test.ts b/src/lib/wiki-type-style.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wiki-type-style.ts b/src/lib/wiki-type-style.ts old mode 100644 new mode 100755 diff --git a/src/lib/wikilink-transform.test.ts b/src/lib/wikilink-transform.test.ts old mode 100644 new mode 100755 diff --git a/src/lib/wikilink-transform.ts b/src/lib/wikilink-transform.ts old mode 100644 new mode 100755 diff --git a/src/main.tsx b/src/main.tsx old mode 100644 new mode 100755 diff --git a/src/stores/activity-store.ts b/src/stores/activity-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/chat-messages-to-llm.test.ts b/src/stores/chat-messages-to-llm.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/chat-store.ts b/src/stores/chat-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/file-sync-store.ts b/src/stores/file-sync-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/lint-store.test.ts b/src/stores/lint-store.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/lint-store.ts b/src/stores/lint-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/research-store.ts b/src/stores/research-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/review-store.property.test.ts b/src/stores/review-store.property.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/review-store.test.ts b/src/stores/review-store.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/review-store.ts b/src/stores/review-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/update-store.ts b/src/stores/update-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/wiki-store.test.ts b/src/stores/wiki-store.test.ts old mode 100644 new mode 100755 diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts old mode 100644 new mode 100755 diff --git a/src/stores/zoom-store.ts b/src/stores/zoom-store.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/deferred.ts b/src/test-helpers/deferred.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/fs-temp.ts b/src/test-helpers/fs-temp.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/load-test-env.ts b/src/test-helpers/load-test-env.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/mock-stream-chat.ts b/src/test-helpers/mock-stream-chat.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/real-content.ts b/src/test-helpers/real-content.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/enrich-scenarios.ts b/src/test-helpers/scenarios/enrich-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/ingest-scenarios.ts b/src/test-helpers/scenarios/ingest-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/lint-scenarios.ts b/src/test-helpers/scenarios/lint-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/materialize.ts b/src/test-helpers/scenarios/materialize.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/search-scenarios.ts b/src/test-helpers/scenarios/search-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/sweep-scenarios.ts b/src/test-helpers/scenarios/sweep-scenarios.ts old mode 100644 new mode 100755 diff --git a/src/test-helpers/scenarios/types.ts b/src/test-helpers/scenarios/types.ts old mode 100644 new mode 100755 diff --git a/src/types/wiki.ts b/src/types/wiki.ts old mode 100644 new mode 100755 diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts old mode 100644 new mode 100755 diff --git a/tsconfig.app.json b/tsconfig.app.json old mode 100644 new mode 100755 diff --git a/tsconfig.json b/tsconfig.json old mode 100644 new mode 100755 diff --git a/tsconfig.node.json b/tsconfig.node.json old mode 100644 new mode 100755 diff --git a/vite.config.ts b/vite.config.ts old mode 100644 new mode 100755 From 96eb5a48da25939a90e23376d7f0abd3055461ac Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 10:15:17 +0800 Subject: [PATCH 13/29] fix: prevent model from generating FILE block for index.md - Remove literal '---FILE: wiki/index.md---' from prompt that taught the model the exact pattern to output (classic prompt anti-pattern) - Add parser defense: silently skip wiki/index.md FILE blocks to prevent overwriting index.md (which is updated via INDEX blocks) - Add reasoning: { mode: 'off' } to prematch LLM calls, consistent with all other ingest pipeline calls --- src/lib/index-chunker.ts | 2 +- src/lib/ingest.ts | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 83b1e34e9..307b66b9a 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -302,7 +302,7 @@ export async function runPrematchParallel( }, }, signal, - { temperature: 0.1, max_tokens: 256 }, + { temperature: 0.1, max_tokens: 256, reasoning: { mode: "off" } }, ) } catch (err) { hadError = true diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index d21f497fa..44e62d28a 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -520,6 +520,13 @@ export function parseFileBlocks(text: string): ParseFileBlocksResult { i++ } + // index.md is updated exclusively via INDEX blocks (append mode). + // If the model produces a FILE block for it (despite explicit prompt + // instructions not to), silently skip — no warning, no truncation error. + if (path === "wiki/index.md") { + continue + } + if (!closed) { // H2 fix (partial): we can't fabricate content the LLM never // sent, but we surface the drop instead of silently hiding it. @@ -2309,7 +2316,7 @@ export function buildGenerationPrompt( "", "Output one INDEX block per category that has new entries.", "CategoryName must match an existing ## heading in the index.", - "Do NOT output ---FILE: wiki/index.md---. Use INDEX blocks instead.", + "Do NOT output a FILE block for the index page. The index is updated ONLY via INDEX blocks.", "", "## Output Requirements (STRICT — deviations will cause parse failure)", "", @@ -2324,7 +2331,7 @@ export function buildGenerationPrompt( "8. INDEX blocks appear AFTER all FILE blocks and BEFORE any REVIEW blocks.", "9. Each INDEX block must specify a category name after ---INDEX:.", "10. Each entry line in an INDEX block must start with a slug followed by \" — \" and a description.", - "11. Do NOT output ---FILE: wiki/index.md---. Use INDEX blocks instead.", + "11. Do NOT output a FILE block for the index page. The index is updated ONLY via INDEX blocks.", "", "If you start with anything other than `---FILE:`, the entire response will be discarded.", "", From 97797856660744ce0ce5f74d1195206172343b61 Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 11:15:10 +0800 Subject: [PATCH 14/29] feat: add overview section chunking --- src/lib/overview-blocks.test.ts | 59 ++++++++++++++++++++++++++++ src/lib/overview-blocks.ts | 69 +++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 src/lib/overview-blocks.test.ts create mode 100644 src/lib/overview-blocks.ts diff --git a/src/lib/overview-blocks.test.ts b/src/lib/overview-blocks.test.ts new file mode 100644 index 000000000..7dfaea8d6 --- /dev/null +++ b/src/lib/overview-blocks.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest" +import { chunkOverviewBySections } from "./overview-blocks" + +describe("chunkOverviewBySections", () => { + it("returns empty array for empty overview", () => { + expect(chunkOverviewBySections("", 2000)).toEqual([]) + }) + + it("returns single chunk when overview is small", () => { + const overview = [ + "# Overview", + "", + "## 操作系统", + "操作系统是管理硬件资源的软件。", + "", + "## 网络", + "计算机网络是互联互通的系统。", + ].join("\n") + const chunks = chunkOverviewBySections(overview, 2000) + expect(chunks).toHaveLength(1) + expect(chunks[0]).toContain("## 操作系统") + expect(chunks[0]).toContain("## 网络") + }) + + it("splits into multiple chunks when overview exceeds maxChunkChars", () => { + const sections: string[] = ["# Overview", ""] + for (let i = 0; i < 5; i++) { + sections.push(`## Section ${i}`) + sections.push("x".repeat(600)) + sections.push("") + } + const overview = sections.join("\n") + const chunks = chunkOverviewBySections(overview, 1000) + expect(chunks.length).toBeGreaterThan(1) + }) + + it("preserves section headings in each chunk", () => { + const overview = [ + "# Overview", + "", + "## 操作系统", + "内容A", + "", + "## 网络", + "内容B", + ].join("\n") + const chunks = chunkOverviewBySections(overview, 50) + expect(chunks.length).toBeGreaterThanOrEqual(2) + expect(chunks.some((c) => c.includes("## 操作系统"))).toBe(true) + expect(chunks.some((c) => c.includes("## 网络"))).toBe(true) + }) + + it("handles overview with no ## headings (single chunk)", () => { + const overview = "Just some prose without headings." + const chunks = chunkOverviewBySections(overview, 2000) + expect(chunks).toHaveLength(1) + expect(chunks[0]).toBe(overview) + }) +}) diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts new file mode 100644 index 000000000..a87910fdc --- /dev/null +++ b/src/lib/overview-blocks.ts @@ -0,0 +1,69 @@ +/** + * Overview section chunking — symmetric to index-chunker.ts but for prose overview.md. + * + * Splits an overview into `##` sections, numbers each with an [N] prefix for + * prematch reference, and groups sections into chunks that stay under + * maxChunkChars. Later tasks add prematch, OVERVIEW block parsing and + * incremental append on top of this file. + */ + +interface OverviewSection { + heading: string | null + content: string +} + +function parseOverviewSections(overview: string): OverviewSection[] { + const lines = overview.split("\n") + const sections: OverviewSection[] = [] + let currentLines: string[] = [] + + for (const line of lines) { + if (/^##\s/.test(line)) { + if (currentLines.length > 0) { + sections.push({ heading: extractHeading(currentLines[0]), content: currentLines.join("\n") }) + } + currentLines = [line] + } else { + currentLines.push(line) + } + } + if (currentLines.length > 0) { + sections.push({ heading: extractHeading(currentLines[0]), content: currentLines.join("\n") }) + } + + return sections +} + +function extractHeading(line: string): string | null { + const match = line.match(/^##\s+(.+)$/) + return match ? match[1].trim() : null +} + +export function chunkOverviewBySections(overview: string, maxChunkChars: number): string[] { + if (!overview.trim()) return [] + + const sections = parseOverviewSections(overview) + if (!sections.some((s) => s.heading !== null)) { + return [overview] + } + const chunks: string[] = [] + let current = "" + let sectionNum = 0 + + for (const section of sections) { + const prefix = section.heading + ? `[${++sectionNum}] ## ${section.heading}` + : `[${++sectionNum}] (preamble)` + const numbered = prefix + "\n" + section.content.replace(/^##\s.*$/m, "").trim() + + if (current.length + numbered.length + 2 > maxChunkChars && current) { + chunks.push(current) + current = numbered + } else { + current = current ? `${current}\n\n${numbered}` : numbered + } + } + + if (current) chunks.push(current) + return chunks +} From c9a3a047ae51e213b8c5dfc9ef6ca71538484c1c Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 11:20:47 +0800 Subject: [PATCH 15/29] feat: add overview prematch prompt, parsing, assembly, and parallel runner --- src/lib/overview-blocks.test.ts | 106 +++++++++++++++++++++++- src/lib/overview-blocks.ts | 139 ++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 2 deletions(-) diff --git a/src/lib/overview-blocks.test.ts b/src/lib/overview-blocks.test.ts index 7dfaea8d6..e0701e9c6 100644 --- a/src/lib/overview-blocks.test.ts +++ b/src/lib/overview-blocks.test.ts @@ -1,5 +1,33 @@ -import { describe, it, expect } from "vitest" -import { chunkOverviewBySections } from "./overview-blocks" +import { describe, it, expect, vi, beforeEach } from "vitest" +import { + chunkOverviewBySections, + parseOverviewPrematchOutput, + buildOverviewPrematchPrompt, + assembleReducedOverview, + runOverviewPrematchParallel, +} from "./overview-blocks" +import { streamChat } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" + +vi.mock("@/lib/llm-client", () => ({ + streamChat: vi.fn(), +})) + +const mockStreamChat = vi.mocked(streamChat) +const mockLlmConfig = { model: "test", provider: "openai" } as unknown as LlmConfig + +function mockStreamResponse(text: string) { + return async (_config: unknown, _messages: unknown, callbacks: { + onToken: (t: string) => void; onDone: () => void; onError: (e: Error) => void + }) => { + for (const char of text) callbacks.onToken(char) + callbacks.onDone() + } +} + +beforeEach(() => { + mockStreamChat.mockReset() +}) describe("chunkOverviewBySections", () => { it("returns empty array for empty overview", () => { @@ -57,3 +85,77 @@ describe("chunkOverviewBySections", () => { expect(chunks[0]).toBe(overview) }) }) + +describe("parseOverviewPrematchOutput", () => { + it("parses bracket section names", () => { + expect(parseOverviewPrematchOutput("[操作系统, 进程管理]")).toEqual(["操作系统", "进程管理"]) + }) + it("returns empty for none", () => { + expect(parseOverviewPrematchOutput("none")).toEqual([]) + }) + it("returns empty for empty string", () => { + expect(parseOverviewPrematchOutput("")).toEqual([]) + }) + it("handles extra whitespace", () => { + expect(parseOverviewPrematchOutput("[ 操作系统 , 网络 ]")).toEqual(["操作系统", "网络"]) + }) +}) + +describe("buildOverviewPrematchPrompt", () => { + it("includes source content and chunk", () => { + const prompt = buildOverviewPrematchPrompt("source text", "chunk text") + expect(prompt).toContain("source text") + expect(prompt).toContain("chunk text") + expect(prompt).toContain("section names") + }) +}) + +describe("assembleReducedOverview", () => { + it("returns empty string for no matches", () => { + expect(assembleReducedOverview("any", [])).toBe("") + }) + it("assembles only matched sections", () => { + const overview = [ + "# Overview", "", + "## 操作系统", "OS content", "", + "## 网络", "Network content", "", + "## 数据库", "DB content", + ].join("\n") + const result = assembleReducedOverview(overview, ["操作系统", "数据库"]) + expect(result).toContain("## 操作系统") + expect(result).toContain("OS content") + expect(result).toContain("## 数据库") + expect(result).toContain("DB content") + expect(result).not.toContain("## 网络") + expect(result).not.toContain("Network content") + }) +}) + +describe("runOverviewPrematchParallel", () => { + it("returns empty array for empty chunks", async () => { + const result = await runOverviewPrematchParallel([], "source", mockLlmConfig, undefined) + expect(result).toEqual([]) + }) + it("collects matched section names from all chunks", async () => { + mockStreamChat + .mockImplementationOnce(mockStreamResponse("[操作系统, 进程管理]")) + .mockImplementationOnce(mockStreamResponse("[网络]")) + const result = await runOverviewPrematchParallel(["chunk1", "chunk2"], "source", mockLlmConfig, undefined) + expect(result.sort()).toEqual(["操作系统", "网络", "进程管理"]) + }) + it("handles none response", async () => { + mockStreamChat.mockImplementationOnce(mockStreamResponse("none")) + const result = await runOverviewPrematchParallel(["chunk1"], "source", mockLlmConfig, undefined) + expect(result).toEqual([]) + }) + it("handles LLM error gracefully", async () => { + mockStreamChat.mockImplementationOnce(async (_c, _m, callbacks: { + onToken: (t: string) => void; onDone: () => void; onError: (e: Error) => void + }) => { + callbacks.onError(new Error("LLM failed")) + callbacks.onDone() + }) + const result = await runOverviewPrematchParallel(["chunk1"], "source", mockLlmConfig, undefined) + expect(result).toEqual([]) + }) +}) diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index a87910fdc..62c189eab 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -7,6 +7,9 @@ * incremental append on top of this file. */ +import { streamChat } from "@/lib/llm-client" +import type { LlmConfig } from "@/stores/wiki-store" + interface OverviewSection { heading: string | null content: string @@ -67,3 +70,139 @@ export function chunkOverviewBySections(overview: string, maxChunkChars: number) if (current) chunks.push(current) return chunks } + +/** + * Parse prematch LLM output into section names. + * Tolerant: handles [操作系统, 进程管理], none, 无, surrounding text. + * Returns deduplicated section names, or empty array if none. + */ +export function parseOverviewPrematchOutput(output: string): string[] { + const trimmed = output.trim() + if (!trimmed) return [] + + const lower = trimmed.toLowerCase() + if (lower === "none" || trimmed === "无") return [] + + // Try to extract bracketed names first: [操作系统, 进程管理] + const bracketMatch = trimmed.match(/\[([^\]]+)\]/) + const source = bracketMatch ? bracketMatch[1] : trimmed + + return source + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0) +} + +/** + * Build the system prompt for an overview pre-match LLM call. + * The LLM reads the source document and a chunk of overview sections, + * then outputs matching section names. + */ +export function buildOverviewPrematchPrompt(sourceContent: string, chunk: string): string { + return [ + "You are a relevance matcher. Read the source document and determine", + "which overview sections are related to it.", + "", + "## Source Document", + sourceContent, + "", + "## Overview Sections", + "Below is a chunk of the wiki overview. Each section starts with [N] and a heading.", + "For each section, determine whether it covers the same subject as any", + "topic in the source document. A match means the section discusses the", + "same entity, concept, method, or topic area.", + "", + chunk, + "", + "## Output Format (STRICT)", + "", + "Output ONLY matching section names in bracket format: [操作系统, 进程管理]", + "Use the exact heading text (without the ## prefix).", + "If no sections match, output exactly: none", + "", + "Do not output explanations, reasoning, or any other text.", + ].join("\n") +} + +/** + * Assemble a reduced overview from the original overview.md and matched section names. + * Only includes sections whose heading matches. Preserves original section content. + */ +export function assembleReducedOverview(overview: string, matchedSections: string[]): string { + if (matchedSections.length === 0) return "" + + const matchSet = new Set(matchedSections.map((s) => s.trim())) + const sections = parseOverviewSections(overview) + const lines: string[] = [] + + for (const section of sections) { + if (section.heading && matchSet.has(section.heading)) { + lines.push(section.content.trim()) + lines.push("") + } + } + + return lines.join("\n").trim() +} + +const OVERVIEW_PREMATCH_CONCURRENCY = 8 + +/** + * Run pre-match LLM calls in parallel across all overview chunks. + * Returns the union of all matched section names. + * Failed chunks are logged and skipped (treated as 0 matches). + */ +export async function runOverviewPrematchParallel( + chunks: string[], + sourceContent: string, + llmConfig: LlmConfig, + signal: AbortSignal | undefined, +): Promise { + if (chunks.length === 0) return [] + + const results: string[][] = [] + + // Process in batches of OVERVIEW_PREMATCH_CONCURRENCY + for (let i = 0; i < chunks.length; i += OVERVIEW_PREMATCH_CONCURRENCY) { + if (signal?.aborted) break + const batch = chunks.slice(i, i + OVERVIEW_PREMATCH_CONCURRENCY) + + const batchResults = await Promise.all( + batch.map(async (chunk) => { + let output = "" + let hadError = false + + try { + await streamChat( + llmConfig, + [ + { role: "system", content: buildOverviewPrematchPrompt(sourceContent, chunk) }, + { role: "user", content: "Output matching section names for the chunk above." }, + ], + { + onToken: (token: string) => { output += token }, + onDone: () => {}, + onError: (err: Error) => { + hadError = true + console.warn(`[overview-prematch] chunk failed: ${err.message}`) + }, + }, + signal, + { temperature: 0.1, max_tokens: 256, reasoning: { mode: "off" } }, + ) + } catch (err) { + hadError = true + console.warn(`[overview-prematch] chunk threw: ${err instanceof Error ? err.message : String(err)}`) + } + + if (hadError) return [] + return parseOverviewPrematchOutput(output) + }), + ) + + results.push(...batchResults) + } + + // Flatten and deduplicate + return [...new Set(results.flat())] +} From 3eeeeaf9bac32d1e284045e58e05fec7d3d83e29 Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 11:27:38 +0800 Subject: [PATCH 16/29] feat: add OVERVIEW block parsing, content appending, and initial creation --- src/lib/overview-blocks.test.ts | 121 ++++++++++++++++++++++++++++++++ src/lib/overview-blocks.ts | 111 +++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) diff --git a/src/lib/overview-blocks.test.ts b/src/lib/overview-blocks.test.ts index e0701e9c6..cca69a86c 100644 --- a/src/lib/overview-blocks.test.ts +++ b/src/lib/overview-blocks.test.ts @@ -5,6 +5,10 @@ import { buildOverviewPrematchPrompt, assembleReducedOverview, runOverviewPrematchParallel, + parseOverviewBlocks, + appendOverviewContent, + createInitialOverview, + type ParsedOverviewBlock, } from "./overview-blocks" import { streamChat } from "@/lib/llm-client" import type { LlmConfig } from "@/stores/wiki-store" @@ -159,3 +163,120 @@ describe("runOverviewPrematchParallel", () => { expect(result).toEqual([]) }) }) + +describe("parseOverviewBlocks", () => { + it("parses a single OVERVIEW block", () => { + const text = [ + "---OVERVIEW: 操作系统---", + "操作系统是管理硬件资源的软件。", + "", + "它负责进程调度、内存管理和文件系统。", + "---END OVERVIEW---", + ].join("\n") + const blocks = parseOverviewBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].section).toBe("操作系统") + expect(blocks[0].content).toContain("操作系统是管理硬件资源的软件。") + }) + + it("parses multiple OVERVIEW blocks", () => { + const text = [ + "---OVERVIEW: 操作系统---", + "OS content", + "---END OVERVIEW---", + "", + "---OVERVIEW: 网络---", + "Network content", + "---END OVERVIEW---", + ].join("\n") + const blocks = parseOverviewBlocks(text) + expect(blocks).toHaveLength(2) + expect(blocks[0].section).toBe("操作系统") + expect(blocks[1].section).toBe("网络") + }) + + it("returns empty array when no OVERVIEW blocks", () => { + expect(parseOverviewBlocks("just some text")).toEqual([]) + expect(parseOverviewBlocks("")).toEqual([]) + }) + + it("handles extra whitespace in section name", () => { + const text = "---OVERVIEW: 操作系统 ---\ncontent\n---END OVERVIEW---" + const blocks = parseOverviewBlocks(text) + expect(blocks[0].section).toBe("操作系统") + }) +}) + +describe("appendOverviewContent", () => { + it("appends to existing section", () => { + const existing = [ + "# Overview", "", + "## 操作系统", "OS original content", "", + "## 网络", "Network content", + ].join("\n") + const blocks: ParsedOverviewBlock[] = [ + { section: "操作系统", content: "New OS paragraph." }, + ] + const result = appendOverviewContent(existing, blocks) + expect(result).toContain("## 操作系统") + expect(result).toContain("OS original content") + expect(result).toContain("New OS paragraph.") + expect(result.indexOf("OS original content")).toBeLessThan(result.indexOf("New OS paragraph.")) + expect(result).toContain("## 网络") + expect(result).toContain("Network content") + }) + + it("creates new section at end when section does not exist", () => { + const existing = ["# Overview", "", "## 操作系统", "OS content"].join("\n") + const blocks: ParsedOverviewBlock[] = [ + { section: "数据库", content: "DB content." }, + ] + const result = appendOverviewContent(existing, blocks) + expect(result).toContain("## 数据库") + expect(result).toContain("DB content.") + expect(result.indexOf("## 操作系统")).toBeLessThan(result.indexOf("## 数据库")) + }) + + it("handles multiple blocks", () => { + const existing = "# Overview\n\n## 操作系统\nOS content" + const blocks: ParsedOverviewBlock[] = [ + { section: "操作系统", content: "OS new." }, + { section: "网络", content: "Network new." }, + ] + const result = appendOverviewContent(existing, blocks) + expect(result).toContain("OS new.") + expect(result).toContain("## 网络") + expect(result).toContain("Network new.") + }) + + it("returns original when no blocks", () => { + const existing = "# Overview\n\n## A\ncontent" + expect(appendOverviewContent(existing, [])).toBe(existing) + }) +}) + +describe("createInitialOverview", () => { + it("creates overview with frontmatter and sections", () => { + const blocks: ParsedOverviewBlock[] = [ + { section: "操作系统", content: "OS content." }, + ] + const result = createInitialOverview(blocks, "2026-06-26") + expect(result).toContain("type: overview") + expect(result).toContain("---") + expect(result).toContain("# Overview") + expect(result).toContain("## 操作系统") + expect(result).toContain("OS content.") + expect(result).toContain("created: 2026-06-26") + expect(result).toContain("updated: 2026-06-26") + }) + + it("handles multiple sections", () => { + const blocks: ParsedOverviewBlock[] = [ + { section: "A", content: "Content A." }, + { section: "B", content: "Content B." }, + ] + const result = createInitialOverview(blocks, "2026-06-26") + expect(result).toContain("## A") + expect(result).toContain("## B") + }) +}) diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index 62c189eab..dc4d977f8 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -8,6 +8,7 @@ */ import { streamChat } from "@/lib/llm-client" +import { currentWikiDate } from "@/lib/ingest" import type { LlmConfig } from "@/stores/wiki-store" interface OverviewSection { @@ -206,3 +207,113 @@ export async function runOverviewPrematchParallel( // Flatten and deduplicate return [...new Set(results.flat())] } + +export interface ParsedOverviewBlock { + section: string + content: string +} + +const OVERVIEW_BLOCK_REGEX = /---OVERVIEW:\s*(.+?)\s*---\n([\s\S]*?)---END OVERVIEW---/g + +/** + * Parse ---OVERVIEW: SectionName--- blocks from LLM generation output. + * Symmetric to parseIndexBlocks in index-chunker.ts. + */ +export function parseOverviewBlocks(text: string): ParsedOverviewBlock[] { + const normalized = text.replace(/\r\n/g, "\n") + const blocks: ParsedOverviewBlock[] = [] + for (const match of normalized.matchAll(OVERVIEW_BLOCK_REGEX)) { + const section = match[1].trim() + const content = match[2].trim() + blocks.push({ section, content }) + } + return blocks +} + +/** + * Programmatically append parsed OVERVIEW block content to existing overview.md. + * - Appends to existing `## Section` (before the next ## or EOF) + * - Creates a new `## Section` at the end if it does not exist + * - Updates the `updated:` field in frontmatter to today's date (when present) + * - Returns the original content unchanged when no blocks are provided + * Symmetric to appendIndexEntries in index-chunker.ts. + */ +export function appendOverviewContent( + overviewContent: string, + blocks: ParsedOverviewBlock[], +): string { + if (blocks.length === 0) return overviewContent + + const lines = overviewContent.split("\n") + const result = [...lines] + + for (const block of blocks) { + if (!block.content.trim()) continue + const sectionHeader = `## ${block.section}` + + let sectionStartIdx = -1 + for (let i = 0; i < result.length; i++) { + if (result[i].trim() === sectionHeader) { + sectionStartIdx = i + break + } + } + + if (sectionStartIdx >= 0) { + let insertIdx = result.length + for (let i = sectionStartIdx + 1; i < result.length; i++) { + if (/^##\s/.test(result[i].trim())) { + insertIdx = i + break + } + } + result.splice(insertIdx, 0, "", block.content) + } else { + result.push("", sectionHeader, block.content) + } + } + + // Update frontmatter `updated:` date if present + if (result[0]?.trim() === "---") { + for (let i = 1; i < result.length; i++) { + if (result[i].trim() === "---") break + if (/^updated:/.test(result[i])) { + result[i] = `updated: ${currentWikiDate()}` + break + } + } + } + + return result.join("\n") +} + +/** + * Create the initial overview.md with frontmatter for the first ingest. + * Symmetric to the index.md bootstrap, but for prose overview sections. + */ +export function createInitialOverview( + blocks: ParsedOverviewBlock[], + date?: string, +): string { + const d = date ?? currentWikiDate() + const lines: string[] = [ + "---", + "type: overview", + 'title: "Overview"', + `created: ${d}`, + `updated: ${d}`, + "tags: []", + "related: []", + "---", + "", + "# Overview", + ] + + for (const block of blocks) { + lines.push("") + lines.push(`## ${block.section}`) + lines.push(block.content) + } + + return lines.join("\n") +} From c230847f6ee07bea0c0363bb46408801bc407f6a Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 11:33:16 +0800 Subject: [PATCH 17/29] feat: integrate OVERVIEW blocks into ingest pipeline - Skip wiki/overview.md FILE blocks in parser (Task 6) - Replace overview FILE block instruction with OVERVIEW block format in prompt (Task 7) - Add overview prematch (Step 0.8) and OVERVIEW block processing (Step 3.6) (Task 8) - Remove overview.md from aggregate repair (Task 9) --- src/lib/ingest-parse.test.ts | 61 +++++++++++++++------------ src/lib/ingest.ts | 82 +++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 35 deletions(-) diff --git a/src/lib/ingest-parse.test.ts b/src/lib/ingest-parse.test.ts index 1c34c3d6d..fa72b5bc9 100755 --- a/src/lib/ingest-parse.test.ts +++ b/src/lib/ingest-parse.test.ts @@ -30,7 +30,6 @@ import { rewriteIngestPathFromTitleForTargetLanguage, canonicalizeSourcesField, isAppManagedAggregatePath, - updateBoundedRecentIndexSection, } from "./ingest" // ── Happy paths ───────────────────────────────────────────────────── @@ -92,6 +91,22 @@ describe("parseFileBlocks — canonical shapes", () => { ].join("\n") expect(parseFileBlocks(text).blocks).toHaveLength(1) }) + + it("silently skips FILE block for wiki/overview.md", () => { + const text = [ + "---FILE: wiki/overview.md---", + "# Overview", + "This should be skipped.", + "---END FILE---", + "", + "---FILE: wiki/entities/foo.md---", + "This should be kept.", + "---END FILE---", + ].join("\n") + const { blocks } = parseFileBlocks(text) + expect(blocks).toHaveLength(1) + expect(blocks[0].path).toBe("wiki/entities/foo.md") + }) }) describe("source summary media refs", () => { @@ -113,24 +128,31 @@ describe("source summary media refs", () => { }) describe("aggregate repair targeting", () => { - it("repairs only the append-only log and leaves deterministic aggregates to the app", () => { + it("requests missing aggregate pages and aggregate pages dropped by truncation warnings", () => { + // wiki/log.md is the only aggregate repair target — overview.md is + // now maintained incrementally via OVERVIEW blocks, so a truncation + // warning for it must NOT trigger a full-overwrite repair. + expect(aggregatePathsNeedingRepair( + ["wiki/entities/foo.md"], + ['FILE block "wiki/log.md" was not closed before end of stream — likely truncation.'], + )).toEqual(["wiki/log.md"]) + expect(aggregatePathsNeedingRepair( ["wiki/log.md"], - ['FILE block "wiki/overview.md" was not closed before end of stream — likely truncation.'], + [], )).toEqual([]) - expect(aggregatePathsNeedingRepair(["wiki/index.md"], [])).toEqual(["wiki/log.md"]) - + // overview.md truncation is no longer an aggregate-repair trigger. expect(aggregatePathsNeedingRepair( - ["wiki/overview.md", "wiki/log.md"], - [], + ["wiki/log.md"], + ['FILE block "wiki/overview.md" was not closed before end of stream — likely truncation.'], )).toEqual([]) }) it("filters aggregate repair output to the requested aggregate paths only", () => { const raw = [ - "---FILE: wiki/overview.md---", - "# Overview", + "---FILE: wiki/log.md---", + "# Log", "---END FILE---", "", "---FILE: wiki/sources/should-not-touch.md---", @@ -142,9 +164,9 @@ describe("aggregate repair targeting", () => { "---END FILE---", ].join("\n") - const filtered = filterAggregateRepairOutput(raw, ["wiki/overview.md"]) + const filtered = filterAggregateRepairOutput(raw, ["wiki/log.md"]) - expect(filtered.text).toContain("---FILE: wiki/overview.md---") + expect(filtered.text).toContain("---FILE: wiki/log.md---") expect(filtered.text).not.toContain("should-not-touch") expect(filtered.text).not.toContain("wiki/entities/stray.md") expect(filtered.warnings.join("\n")).toContain("Dropped 2 non-aggregate") @@ -664,21 +686,4 @@ describe("application-managed aggregate boundaries", () => { expect(isAppManagedAggregatePath("wiki\\overview.MD")).toBe(true) expect(isAppManagedAggregatePath("wiki/entities/index.md")).toBe(false) }) - - it("bounds recent entries and preserves following sections", () => { - const existing = [ - "# Wiki Index", - "", - "## Recently Updated", - ...Array.from({ length: 205 }, (_, index) => `- [[old-${index}]] — Old ${index}`), - "", - "## Other", - "Keep me", - ].join("\n") - const result = updateBoundedRecentIndexSection(existing, ["- [[new]] — New"]) - const recent = result.split("## Recently Updated")[1].split("## Other")[0] - expect(recent.match(/^- \[\[/gm)).toHaveLength(200) - expect(recent).toContain("[[new]]") - expect(result).toContain("## Other\nKeep me") - }) }) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 44e62d28a..796b22d45 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -49,6 +49,14 @@ import { parseIndexBlocks, appendIndexEntries, } from "./index-chunker" +import { + chunkOverviewBySections, + runOverviewPrematchParallel, + assembleReducedOverview, + parseOverviewBlocks, + appendOverviewContent, + createInitialOverview, +} from "@/lib/overview-blocks" const LONG_SOURCE_MIN_BUDGET = 8_000 const LONG_SOURCE_MAX_SINGLE_PASS_BUDGET = 300_000 @@ -62,7 +70,7 @@ const INGEST_GENERATION_TOKENS_256K = 24_576 const INGEST_GENERATION_TOKENS_512K = 32_768 const REVIEW_STAGE_MIN_SIGNAL_CHARS = 10_000 const REVIEW_STAGE_MIN_FILE_BLOCKS = 4 -const AGGREGATE_WIKI_PATHS = ["wiki/overview.md", "wiki/log.md"] as const +const AGGREGATE_WIKI_PATHS = ["wiki/log.md"] as const function appendSavedImageRefsForCaption(content: string, images: SavedImage[]): string { if (images.length === 0) return content @@ -520,10 +528,11 @@ export function parseFileBlocks(text: string): ParseFileBlocksResult { i++ } - // index.md is updated exclusively via INDEX blocks (append mode). - // If the model produces a FILE block for it (despite explicit prompt + // index.md and overview.md are updated exclusively via INDEX/OVERVIEW + // blocks (append mode). + // If the model produces a FILE block for either (despite explicit prompt // instructions not to), silently skip — no warning, no truncation error. - if (path === "wiki/index.md") { + if (path === "wiki/index.md" || path === "wiki/overview.md") { continue } @@ -993,6 +1002,31 @@ async function autoIngestImpl( } } + // ── Step 0.8: Pre-match overview sections ───────────────────── + let reducedOverview = overview + if (overview.trim()) { + const overviewChunks = chunkOverviewBySections(overview, 2000) + if (overviewChunks.length > 1) { + activity.updateItem(activityId, { + detail: `Step 0.8: Pre-matching overview (${overviewChunks.length} chunks)...`, + }) + const matchedSections = await runOverviewPrematchParallel( + overviewChunks, + sourceContext, + llmConfig, + signal, + ) + reducedOverview = assembleReducedOverview(overview, matchedSections) + if (!reducedOverview) { + reducedOverview = "(no matching overview sections found)" + } + console.log( + `[ingest:overview-prematch] overview reduced from ${overview.length} to ${reducedOverview.length} chars ` + + `(${matchedSections.length} sections matched from ${overviewChunks.length} chunks)`, + ) + } + } + // ── Step 1: Analysis ────────────────────────────────────────── // LLM reads the source and produces a structured analysis: // key entities, concepts, main arguments, connections to existing wiki, contradictions @@ -1040,7 +1074,7 @@ async function autoIngestImpl( await streamChat( llmConfig, [ - { role: "system", content: buildGenerationPrompt(schema, purpose, reducedIndex, sourceIdentity, overview, sourceContext, sourceSummaryPath) }, + { role: "system", content: buildGenerationPrompt(schema, purpose, reducedIndex, sourceIdentity, reducedOverview, sourceContext, sourceSummaryPath) }, { role: "user", content: [ @@ -1177,6 +1211,29 @@ async function autoIngestImpl( } } + // ── Step 3.6: Process OVERVIEW blocks ────────────────────── + const overviewBlocks = parseOverviewBlocks(generation) + if (overviewBlocks.length > 0) { + try { + const overviewAbs = `${pp}/wiki/overview.md` + const existingOverview = await tryReadFile(overviewAbs) + const updatedOverview = existingOverview + ? appendOverviewContent(existingOverview, overviewBlocks) + : createInitialOverview(overviewBlocks) + await writeFile(overviewAbs, updatedOverview) + console.log( + `[ingest:overview] appended ${overviewBlocks.length} section(s) to overview.md`, + ) + if (!writtenPaths.includes("wiki/overview.md")) { + writtenPaths.push("wiki/overview.md") + } + } catch (err) { + writeWarnings.push( + `Failed to append OVERVIEW blocks: ${err instanceof Error ? err.message : String(err)}`, + ) + } + } + // ── Step 3.6: Fallback INDEX construction ─────────────── // If no INDEX blocks were emitted, construct them from the // written FILE blocks as a fallback. This replaces the old @@ -2162,7 +2219,7 @@ export function buildGenerationPrompt( purpose: string, index: string, sourceFileName: string, - overview?: string, + reducedOverview?: string, sourceContent: string = "", sourceSummaryPath?: string, ): string { @@ -2283,7 +2340,7 @@ export function buildGenerationPrompt( "", purpose ? `## Wiki Purpose\n${purpose}` : "", index ? `## Current Wiki Index (for reference only — do NOT reproduce it)\n${index}` : "", - overview ? `## Current Overview (update this to reflect the new source)\n${overview}` : "", + reducedOverview ? `## Current Overview (relevant sections only — for reference)\n${reducedOverview}` : "", "", // ── OUTPUT FORMAT MUST BE THE LAST SECTION — models weight recent instructions highest ── "## Output Format (MUST FOLLOW EXACTLY — this is how the parser reads your response)", @@ -2318,6 +2375,17 @@ export function buildGenerationPrompt( "CategoryName must match an existing ## heading in the index.", "Do NOT output a FILE block for the index page. The index is updated ONLY via INDEX blocks.", "", + "OVERVIEW block template (for overview paragraphs):", + "```", + "---OVERVIEW: SectionName---", + "1-2 paragraphs about this source's key topics.", + "---END OVERVIEW---", + "```", + "", + "Output one OVERVIEW block. SectionName should match an existing ## heading", + "in the overview context below, or create a new section name if the topic is new.", + "Do NOT output a FILE block for the overview page. Use OVERVIEW blocks instead.", + "", "## Output Requirements (STRICT — deviations will cause parse failure)", "", "1. The FIRST character of your response MUST be `-` (the opening of `---FILE:`).", From 888c64151f0d0ac87a9d81f6466cd1e29f3a2766 Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 11:54:45 +0800 Subject: [PATCH 18/29] perf: move prematch before budget calculation for accurate sourceBudget stableContextLength now uses reducedIndex/reducedOverview lengths instead of full index/overview. This prevents over-trimming the source budget when prematch has already significantly reduced the context. --- src/lib/ingest.ts | 64 +++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 796b22d45..43d671e88 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -946,37 +946,11 @@ async function autoIngestImpl( } } - const stableContextLength = schema.length + purpose.length + index.length + overview.length - const sourceBudget = computeIngestSourceBudget(llmConfig.maxContextSize, stableContextLength) - let sourceContext = enrichedSourceContent - let precomputedAnalysis = "" - let longSourceCheckpointPath: string | undefined - - if (enrichedSourceContent.length > sourceBudget) { - const longSourcePlan = await analyzeLongSourceInChunks( - pp, - llmConfig, - purpose, - schema, - index, - sourceIdentity, - sourceSummarySlug, - folderContext, - enrichedSourceContent, - sourceBudget, - activityId, - signal, - ) - if (longSourcePlan.chunked) { - sourceContext = longSourcePlan.sourceContext - precomputedAnalysis = longSourcePlan.analysis - longSourceCheckpointPath = longSourcePlan.checkpointPath - } - } - // ── Step 0.7: Pre-match index chunks ───────────────────── // Split index into chunks, run parallel LLM calls to find // entries relevant to this source, assemble a reduced index. + // Runs BEFORE budget calculation so stableContextLength reflects + // the actual (reduced) context size, not the full index length. const CHUNK_SIZE = 50 let reducedIndex = index if (index.trim()) { @@ -987,7 +961,7 @@ async function autoIngestImpl( }) const matchedNumbers = await runPrematchParallel( chunks, - sourceContext, + enrichedSourceContent, llmConfig, signal, ) @@ -1012,7 +986,7 @@ async function autoIngestImpl( }) const matchedSections = await runOverviewPrematchParallel( overviewChunks, - sourceContext, + enrichedSourceContent, llmConfig, signal, ) @@ -1027,6 +1001,36 @@ async function autoIngestImpl( } } + // Compute budget with REDUCED sizes (post-prematch) so sourceBudget + // is not artificially constrained by the full index/overview length. + const stableContextLength = schema.length + purpose.length + reducedIndex.length + reducedOverview.length + const sourceBudget = computeIngestSourceBudget(llmConfig.maxContextSize, stableContextLength) + let sourceContext = enrichedSourceContent + let precomputedAnalysis = "" + let longSourceCheckpointPath: string | undefined + + if (enrichedSourceContent.length > sourceBudget) { + const longSourcePlan = await analyzeLongSourceInChunks( + pp, + llmConfig, + purpose, + schema, + reducedIndex, + sourceIdentity, + sourceSummarySlug, + folderContext, + enrichedSourceContent, + sourceBudget, + activityId, + signal, + ) + if (longSourcePlan.chunked) { + sourceContext = longSourcePlan.sourceContext + precomputedAnalysis = longSourcePlan.analysis + longSourceCheckpointPath = longSourcePlan.checkpointPath + } + } + // ── Step 1: Analysis ────────────────────────────────────────── // LLM reads the source and produces a structured analysis: // key entities, concepts, main arguments, connections to existing wiki, contradictions From 4e2103fffc44e18d26b1bbd17e47ccbc87442d3b Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 11:59:44 +0800 Subject: [PATCH 19/29] fix: increase DeepSeek preset suggestedContextSize from 64K to 1M DeepSeek V4 Flash supports up to 1M token context. The previous 64K char setting caused computeIngestGenerationMaxTokens to return the minimum 8192 tokens, truncating generation output. --- src/components/settings/llm-presets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/settings/llm-presets.ts b/src/components/settings/llm-presets.ts index 879eaa116..34c5e5871 100755 --- a/src/components/settings/llm-presets.ts +++ b/src/components/settings/llm-presets.ts @@ -182,7 +182,7 @@ export const LLM_PRESETS: LlmPreset[] = [ "deepseek-chat", "deepseek-reasoner", ], - suggestedContextSize: 64000, + suggestedContextSize: 1000000, }, { id: "atlascloud", From f49e3e2c8802e0ee84270fa14dae68ee8287f570 Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 15:09:23 +0800 Subject: [PATCH 20/29] perf: increase prematch chunk sizes and remove hard limits - Increase CHUNK_SIZE from 50 to 200 entries per chunk - Increase overview prematch chunk from 2000 to 8000 chars - Remove max_tokens: 256 limit from prematch calls (let model decide) - Remove Math.min(8_192) cap from computeIngestReviewMaxTokens --- src/lib/index-chunker.ts | 2 +- src/lib/ingest.ts | 6 +++--- src/lib/overview-blocks.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 307b66b9a..9a55e1c93 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -302,7 +302,7 @@ export async function runPrematchParallel( }, }, signal, - { temperature: 0.1, max_tokens: 256, reasoning: { mode: "off" } }, + { temperature: 0.1, reasoning: { mode: "off" } }, ) } catch (err) { hadError = true diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 43d671e88..25f41664d 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -951,7 +951,7 @@ async function autoIngestImpl( // entries relevant to this source, assemble a reduced index. // Runs BEFORE budget calculation so stableContextLength reflects // the actual (reduced) context size, not the full index length. - const CHUNK_SIZE = 50 + const CHUNK_SIZE = 200 let reducedIndex = index if (index.trim()) { const chunks = chunkIndexByEntries(index, CHUNK_SIZE) @@ -979,7 +979,7 @@ async function autoIngestImpl( // ── Step 0.8: Pre-match overview sections ───────────────────── let reducedOverview = overview if (overview.trim()) { - const overviewChunks = chunkOverviewBySections(overview, 2000) + const overviewChunks = chunkOverviewBySections(overview, 8000) if (overviewChunks.length > 1) { activity.updateItem(activityId, { detail: `Step 0.8: Pre-matching overview (${overviewChunks.length} chunks)...`, @@ -2576,7 +2576,7 @@ export function computeIngestGenerationMaxTokens(maxContextSize: number | undefi } export function computeIngestReviewMaxTokens(maxContextSize: number | undefined): number { - return Math.min(8_192, Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize) / 2))) + return Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize) / 2)) } function splitOversizedBlock(block: string, targetChars: number): string[] { diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index dc4d977f8..3ea641bd9 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -189,7 +189,7 @@ export async function runOverviewPrematchParallel( }, }, signal, - { temperature: 0.1, max_tokens: 256, reasoning: { mode: "off" } }, + { temperature: 0.1, reasoning: { mode: "off" } }, ) } catch (err) { hadError = true From 0080790801e095866b9990fadae7fa4e04a36fa9 Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 17:24:45 +0800 Subject: [PATCH 21/29] fix: always run prematch even for single chunk Previously prematch was skipped when index/overview fit in one chunk (<=200 entries or <=8000 chars). This meant those entries were never filtered, wasting context. Now prematch runs for any chunked content, reducing even small indexes to only relevant entries. --- src/lib/ingest.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 25f41664d..0b7066ba8 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -955,7 +955,7 @@ async function autoIngestImpl( let reducedIndex = index if (index.trim()) { const chunks = chunkIndexByEntries(index, CHUNK_SIZE) - if (chunks.length > 1) { + if (chunks.length > 0) { activity.updateItem(activityId, { detail: `Step 0.7: Pre-matching index (${chunks.length} chunks)...`, }) @@ -980,7 +980,7 @@ async function autoIngestImpl( let reducedOverview = overview if (overview.trim()) { const overviewChunks = chunkOverviewBySections(overview, 8000) - if (overviewChunks.length > 1) { + if (overviewChunks.length > 0) { activity.updateItem(activityId, { detail: `Step 0.8: Pre-matching overview (${overviewChunks.length} chunks)...`, }) From 15139de82362b1b0dbdd8afe5920d8a677bac85f Mon Sep 17 00:00:00 2001 From: wkh Date: Fri, 26 Jun 2026 17:53:40 +0800 Subject: [PATCH 22/29] feat: paragraph-level overview prematch Change overview prematch from ##-section granularity to paragraph-level. Each paragraph (split by \n\n within a ## section) is individually numbered and matched by the LLM. assembleReducedOverview groups matched paragraphs under their section headings for generation prompt injection. - Add parseOverviewParagraphs helper - Update chunkOverviewBySections to number paragraphs globally - Update buildOverviewPrematchPrompt for paragraph numbers - Update parseOverviewPrematchOutput to return number[] - Update runOverviewPrematchParallel return type - Update assembleReducedOverview to accept paragraph numbers - Update ingest.ts call site variable names and log messages - Update scenario tests to include prematch mock response --- src/lib/ingest.prompt.test.ts | 2 +- src/lib/ingest.scenarios.test.ts | 10 ++- src/lib/ingest.ts | 14 +++- src/lib/overview-blocks.test.ts | 46 ++++------- src/lib/overview-blocks.ts | 133 ++++++++++++++++++++----------- 5 files changed, 121 insertions(+), 84 deletions(-) diff --git a/src/lib/ingest.prompt.test.ts b/src/lib/ingest.prompt.test.ts index 05ad17922..097497b55 100755 --- a/src/lib/ingest.prompt.test.ts +++ b/src/lib/ingest.prompt.test.ts @@ -203,7 +203,7 @@ describe("long-source ingest planning", () => { expect(computeIngestGenerationMaxTokens(128_000)).toBe(16_384) expect(computeIngestGenerationMaxTokens(256_000)).toBe(24_576) expect(computeIngestGenerationMaxTokens(1_000_000)).toBe(32_768) - expect(computeIngestReviewMaxTokens(1_000_000)).toBe(8_192) + expect(computeIngestReviewMaxTokens(1_000_000)).toBe(16_384) }) it("scales source budget from the configured context window instead of a fixed 50k cap", () => { diff --git a/src/lib/ingest.scenarios.test.ts b/src/lib/ingest.scenarios.test.ts index 2e131dcd2..eb17b9c91 100755 --- a/src/lib/ingest.scenarios.test.ts +++ b/src/lib/ingest.scenarios.test.ts @@ -104,7 +104,12 @@ async function setup(scenario: IngestScenario): Promise { maxContextSize: 128000, }) - // Queue up the two sequenced LLM responses + // Queue up the sequenced LLM responses: + // 1. Index prematch: BASIC_INDEX has 1 entry ([[attention]]), so [1] matches. + // 2. Overview prematch: skipped if no overview.md exists. + // 3. Analysis (Stage 1) + // 4. Generation (Stage 2) + // 5. Review suggestion (if triggered) const analysis = await fs.readFile( path.join(FIXTURES_ROOT, scenario.name, "llm-analysis.txt"), "utf-8", @@ -113,7 +118,7 @@ async function setup(scenario: IngestScenario): Promise { path.join(FIXTURES_ROOT, scenario.name, "llm-generation.txt"), "utf-8", ) - pendingResponses = [analysis, generation] + pendingResponses = ["[1]", analysis, generation] return { tmp } } @@ -244,6 +249,7 @@ describe("ingest scenarios (fixture-driven)", () => { }) pendingResponses = [ + "[1]", "analysis", [ "---FILE: wiki/sources/schema-routing.md---", diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 0b7066ba8..93f93fc75 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -969,6 +969,12 @@ async function autoIngestImpl( if (!reducedIndex) { reducedIndex = "(no matching wiki entries found)" } + // If prematch returned nothing meaningful, keep the full index + // as fallback — an empty reduced index would make the model + // skip all page generation. + if (reducedIndex.length < 10) { + reducedIndex = index + } console.log( `[ingest:prematch] index reduced from ${index.length} to ${reducedIndex.length} chars ` + `(${matchedNumbers.length} matches from ${chunks.length} chunks)`, @@ -984,19 +990,19 @@ async function autoIngestImpl( activity.updateItem(activityId, { detail: `Step 0.8: Pre-matching overview (${overviewChunks.length} chunks)...`, }) - const matchedSections = await runOverviewPrematchParallel( + const matchedParagraphs = await runOverviewPrematchParallel( overviewChunks, enrichedSourceContent, llmConfig, signal, ) - reducedOverview = assembleReducedOverview(overview, matchedSections) + reducedOverview = assembleReducedOverview(overview, matchedParagraphs) if (!reducedOverview) { - reducedOverview = "(no matching overview sections found)" + reducedOverview = "(no matching overview paragraphs found)" } console.log( `[ingest:overview-prematch] overview reduced from ${overview.length} to ${reducedOverview.length} chars ` + - `(${matchedSections.length} sections matched from ${overviewChunks.length} chunks)`, + `(${matchedParagraphs.length} paragraphs matched from ${overviewChunks.length} chunks)`, ) } } diff --git a/src/lib/overview-blocks.test.ts b/src/lib/overview-blocks.test.ts index cca69a86c..d7e724338 100644 --- a/src/lib/overview-blocks.test.ts +++ b/src/lib/overview-blocks.test.ts @@ -50,8 +50,10 @@ describe("chunkOverviewBySections", () => { ].join("\n") const chunks = chunkOverviewBySections(overview, 2000) expect(chunks).toHaveLength(1) - expect(chunks[0]).toContain("## 操作系统") - expect(chunks[0]).toContain("## 网络") + expect(chunks[0]).toContain("[1]") + expect(chunks[0]).toContain("操作系统是管理硬件资源的软件。") + expect(chunks[0]).toContain("[2]") + expect(chunks[0]).toContain("计算机网络是互联互通的系统。") }) it("splits into multiple chunks when overview exceeds maxChunkChars", () => { @@ -66,33 +68,18 @@ describe("chunkOverviewBySections", () => { expect(chunks.length).toBeGreaterThan(1) }) - it("preserves section headings in each chunk", () => { - const overview = [ - "# Overview", - "", - "## 操作系统", - "内容A", - "", - "## 网络", - "内容B", - ].join("\n") - const chunks = chunkOverviewBySections(overview, 50) - expect(chunks.length).toBeGreaterThanOrEqual(2) - expect(chunks.some((c) => c.includes("## 操作系统"))).toBe(true) - expect(chunks.some((c) => c.includes("## 网络"))).toBe(true) - }) - it("handles overview with no ## headings (single chunk)", () => { const overview = "Just some prose without headings." const chunks = chunkOverviewBySections(overview, 2000) expect(chunks).toHaveLength(1) - expect(chunks[0]).toBe(overview) + expect(chunks[0]).toContain("[1]") + expect(chunks[0]).toContain("Just some prose without headings.") }) }) describe("parseOverviewPrematchOutput", () => { - it("parses bracket section names", () => { - expect(parseOverviewPrematchOutput("[操作系统, 进程管理]")).toEqual(["操作系统", "进程管理"]) + it("parses bracket paragraph numbers", () => { + expect(parseOverviewPrematchOutput("[2, 5, 12]")).toEqual([2, 5, 12]) }) it("returns empty for none", () => { expect(parseOverviewPrematchOutput("none")).toEqual([]) @@ -101,7 +88,7 @@ describe("parseOverviewPrematchOutput", () => { expect(parseOverviewPrematchOutput("")).toEqual([]) }) it("handles extra whitespace", () => { - expect(parseOverviewPrematchOutput("[ 操作系统 , 网络 ]")).toEqual(["操作系统", "网络"]) + expect(parseOverviewPrematchOutput("[ 2 , 5 ]")).toEqual([2, 5]) }) }) @@ -110,7 +97,7 @@ describe("buildOverviewPrematchPrompt", () => { const prompt = buildOverviewPrematchPrompt("source text", "chunk text") expect(prompt).toContain("source text") expect(prompt).toContain("chunk text") - expect(prompt).toContain("section names") + expect(prompt).toContain("paragraph numbers") }) }) @@ -118,14 +105,15 @@ describe("assembleReducedOverview", () => { it("returns empty string for no matches", () => { expect(assembleReducedOverview("any", [])).toBe("") }) - it("assembles only matched sections", () => { + it("assembles only matched paragraphs", () => { const overview = [ "# Overview", "", "## 操作系统", "OS content", "", "## 网络", "Network content", "", "## 数据库", "DB content", ].join("\n") - const result = assembleReducedOverview(overview, ["操作系统", "数据库"]) + // Paragraphs: [1] preamble, [2] 操作系统, [3] 网络, [4] 数据库 + const result = assembleReducedOverview(overview, [2, 4]) expect(result).toContain("## 操作系统") expect(result).toContain("OS content") expect(result).toContain("## 数据库") @@ -140,12 +128,12 @@ describe("runOverviewPrematchParallel", () => { const result = await runOverviewPrematchParallel([], "source", mockLlmConfig, undefined) expect(result).toEqual([]) }) - it("collects matched section names from all chunks", async () => { + it("collects matched paragraph numbers from all chunks", async () => { mockStreamChat - .mockImplementationOnce(mockStreamResponse("[操作系统, 进程管理]")) - .mockImplementationOnce(mockStreamResponse("[网络]")) + .mockImplementationOnce(mockStreamResponse("[1, 3]")) + .mockImplementationOnce(mockStreamResponse("[5]")) const result = await runOverviewPrematchParallel(["chunk1", "chunk2"], "source", mockLlmConfig, undefined) - expect(result.sort()).toEqual(["操作系统", "网络", "进程管理"]) + expect(result.sort()).toEqual([1, 3, 5]) }) it("handles none response", async () => { mockStreamChat.mockImplementationOnce(mockStreamResponse("none")) diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index 3ea641bd9..628f787dd 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -16,6 +16,12 @@ interface OverviewSection { content: string } +interface OverviewParagraph { + sectionHeading: string | null + number: number + text: string +} + function parseOverviewSections(overview: string): OverviewSection[] { const lines = overview.split("\n") const sections: OverviewSection[] = [] @@ -43,104 +49,135 @@ function extractHeading(line: string): string | null { return match ? match[1].trim() : null } +/** Parse overview into globally-numbered paragraphs (split by \n\n within ## sections). */ +function parseOverviewParagraphs(overview: string): OverviewParagraph[] { + const sections = parseOverviewSections(overview) + const paragraphs: OverviewParagraph[] = [] + let num = 1 + + for (const section of sections) { + // Split section body by one or more blank lines + let body = section.heading + ? section.content.replace(/^##\s.*$/m, "").trim() + : section.content.trim() + if (!body) continue + + const parts = body.split(/\n\n+/).map((p) => p.trim()).filter((p) => p.length > 0) + for (const part of parts) { + paragraphs.push({ sectionHeading: section.heading, number: num++, text: part }) + } + } + + return paragraphs +} + +/** + * Chunk overview paragraphs into groups, numbered globally for prematch. + * Each chunk is prefixed with paragraph numbers for LLM reference. + */ export function chunkOverviewBySections(overview: string, maxChunkChars: number): string[] { if (!overview.trim()) return [] - const sections = parseOverviewSections(overview) - if (!sections.some((s) => s.heading !== null)) { - return [overview] - } + const paragraphs = parseOverviewParagraphs(overview) + if (paragraphs.length === 0) return [] + if (paragraphs.length === 1) return [`[${paragraphs[0].number}] ${paragraphs[0].text}`] + const chunks: string[] = [] let current = "" - let sectionNum = 0 - - for (const section of sections) { - const prefix = section.heading - ? `[${++sectionNum}] ## ${section.heading}` - : `[${++sectionNum}] (preamble)` - const numbered = prefix + "\n" + section.content.replace(/^##\s.*$/m, "").trim() - if (current.length + numbered.length + 2 > maxChunkChars && current) { + for (const p of paragraphs) { + const line = `[${p.number}] ${p.text}` + if (current && current.length + line.length + 2 > maxChunkChars) { chunks.push(current) - current = numbered + current = line } else { - current = current ? `${current}\n\n${numbered}` : numbered + current = current ? `${current}\n\n${line}` : line } } - if (current) chunks.push(current) return chunks } /** - * Parse prematch LLM output into section names. - * Tolerant: handles [操作系统, 进程管理], none, 无, surrounding text. - * Returns deduplicated section names, or empty array if none. + * Parse prematch LLM output into paragraph numbers. + * Tolerant: handles [2, 5, 12], none, surrounding text. + * Returns deduplicated paragraph numbers, or empty array if none. */ -export function parseOverviewPrematchOutput(output: string): string[] { +export function parseOverviewPrematchOutput(output: string): number[] { const trimmed = output.trim() if (!trimmed) return [] const lower = trimmed.toLowerCase() if (lower === "none" || trimmed === "无") return [] - // Try to extract bracketed names first: [操作系统, 进程管理] + // Try to extract bracketed numbers first: [2, 5, 12] const bracketMatch = trimmed.match(/\[([^\]]+)\]/) const source = bracketMatch ? bracketMatch[1] : trimmed - return source + const numbers = source .split(",") - .map((s) => s.trim()) - .filter((s) => s.length > 0) + .map((s) => parseInt(s.trim(), 10)) + .filter((n) => Number.isFinite(n) && n > 0) + + return [...new Set(numbers)] } /** * Build the system prompt for an overview pre-match LLM call. - * The LLM reads the source document and a chunk of overview sections, - * then outputs matching section names. + * The LLM reads the source document and a chunk of overview paragraphs, + * then outputs matching paragraph numbers. */ export function buildOverviewPrematchPrompt(sourceContent: string, chunk: string): string { return [ "You are a relevance matcher. Read the source document and determine", - "which overview sections are related to it.", + "which overview paragraphs are related to it.", "", "## Source Document", sourceContent, "", - "## Overview Sections", - "Below is a chunk of the wiki overview. Each section starts with [N] and a heading.", - "For each section, determine whether it covers the same subject as any", - "topic in the source document. A match means the section discusses the", + "## Overview Paragraphs", + "Below is a chunk of the wiki overview. Each numbered item is a paragraph.", + "For each paragraph, determine whether it covers the same subject as any", + "topic in the source document. A match means the paragraph discusses the", "same entity, concept, method, or topic area.", "", chunk, "", "## Output Format (STRICT)", "", - "Output ONLY matching section names in bracket format: [操作系统, 进程管理]", - "Use the exact heading text (without the ## prefix).", - "If no sections match, output exactly: none", + "Output ONLY matching paragraph numbers in bracket format: [2, 5, 12]", + "If no paragraphs match, output exactly: none", "", "Do not output explanations, reasoning, or any other text.", ].join("\n") } /** - * Assemble a reduced overview from the original overview.md and matched section names. - * Only includes sections whose heading matches. Preserves original section content. + * Assemble a reduced overview from the original overview.md and matched paragraph numbers. + * Only includes matched paragraphs, grouped under their ## section headings. */ -export function assembleReducedOverview(overview: string, matchedSections: string[]): string { - if (matchedSections.length === 0) return "" +export function assembleReducedOverview(overview: string, matchedParagraphs: number[]): string { + if (matchedParagraphs.length === 0) return "" + + const matchSet = new Set(matchedParagraphs) + const paragraphs = parseOverviewParagraphs(overview) + const grouped = new Map() + + for (const p of paragraphs) { + if (matchSet.has(p.number)) { + const key = p.sectionHeading + if (!grouped.has(key)) grouped.set(key, []) + grouped.get(key)!.push(p.text) + } + } - const matchSet = new Set(matchedSections.map((s) => s.trim())) - const sections = parseOverviewSections(overview) - const lines: string[] = [] + if (grouped.size === 0) return "" - for (const section of sections) { - if (section.heading && matchSet.has(section.heading)) { - lines.push(section.content.trim()) - lines.push("") - } + const lines: string[] = [] + for (const [heading, texts] of grouped) { + if (heading) lines.push(`## ${heading}`) + for (const t of texts) lines.push(t) + lines.push("") } return lines.join("\n").trim() @@ -158,10 +195,10 @@ export async function runOverviewPrematchParallel( sourceContent: string, llmConfig: LlmConfig, signal: AbortSignal | undefined, -): Promise { +): Promise { if (chunks.length === 0) return [] - const results: string[][] = [] + const results: number[][] = [] // Process in batches of OVERVIEW_PREMATCH_CONCURRENCY for (let i = 0; i < chunks.length; i += OVERVIEW_PREMATCH_CONCURRENCY) { @@ -178,7 +215,7 @@ export async function runOverviewPrematchParallel( llmConfig, [ { role: "system", content: buildOverviewPrematchPrompt(sourceContent, chunk) }, - { role: "user", content: "Output matching section names for the chunk above." }, + { role: "user", content: "Output matching paragraph numbers for the chunk above." }, ], { onToken: (token: string) => { output += token }, From 5a4bb2b68314a12525096a9585a7d16eb4cfa10c Mon Sep 17 00:00:00 2001 From: wkh Date: Sun, 28 Jun 2026 11:12:27 +0800 Subject: [PATCH 23/29] feat: reasoning max for prematch, increased chunk sizes and max_tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Index/overview prematch: reasoning max for precise matching - Index chunk size: 200 → 500 entries per chunk - Overview chunk size: 8000 → 16000 chars per chunk - Add INGEST_GENERATION_TOKENS_1M = 384000 for 1M context models - computeIngestReviewMaxTokens: remove /2 cap - Analysis and long-source: max_tokens 4096 → computeIngestReviewMaxTokens - Review prompt: 'output nothing' → 'output exactly: none' - Overview prematch prompt: 'related to' → 'would need to be UPDATED' - DeepSeek V4: thinking=enabled for reasoning max, reasoning_effort for high/max - Tests updated for new token limits and DeepSeek flash thinking behavior --- .gitignore | 1 + src/lib/index-chunker.ts | 2 +- src/lib/ingest.prompt.test.ts | 4 ++-- src/lib/ingest.ts | 14 ++++++++------ src/lib/llm-providers.test.ts | 35 +++++++++++++++++++++++++++++++++-- src/lib/overview-blocks.ts | 4 ++-- 6 files changed, 47 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 9a9f924ca..130b1b243 100755 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ pnpm-debug.log* # Brainstorm assets (not tracked in source control) .superpowers/ .claude/ +.playwright-mcp/ diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 9a55e1c93..38cc56861 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -302,7 +302,7 @@ export async function runPrematchParallel( }, }, signal, - { temperature: 0.1, reasoning: { mode: "off" } }, + { temperature: 0.1, reasoning: { mode: "max" } }, ) } catch (err) { hadError = true diff --git a/src/lib/ingest.prompt.test.ts b/src/lib/ingest.prompt.test.ts index 097497b55..4db0f0daf 100755 --- a/src/lib/ingest.prompt.test.ts +++ b/src/lib/ingest.prompt.test.ts @@ -202,8 +202,8 @@ describe("long-source ingest planning", () => { expect(computeIngestGenerationMaxTokens(64_000)).toBe(8_192) expect(computeIngestGenerationMaxTokens(128_000)).toBe(16_384) expect(computeIngestGenerationMaxTokens(256_000)).toBe(24_576) - expect(computeIngestGenerationMaxTokens(1_000_000)).toBe(32_768) - expect(computeIngestReviewMaxTokens(1_000_000)).toBe(16_384) + expect(computeIngestGenerationMaxTokens(1_000_000)).toBe(384_000) + expect(computeIngestReviewMaxTokens(1_000_000)).toBe(384_000) }) it("scales source budget from the configured context window instead of a fixed 50k cap", () => { diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 93f93fc75..05dc1c5bf 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -68,6 +68,7 @@ const INGEST_GENERATION_TOKENS_DEFAULT = 8_192 const INGEST_GENERATION_TOKENS_128K = 16_384 const INGEST_GENERATION_TOKENS_256K = 24_576 const INGEST_GENERATION_TOKENS_512K = 32_768 +const INGEST_GENERATION_TOKENS_1M = 384_000 const REVIEW_STAGE_MIN_SIGNAL_CHARS = 10_000 const REVIEW_STAGE_MIN_FILE_BLOCKS = 4 const AGGREGATE_WIKI_PATHS = ["wiki/log.md"] as const @@ -951,7 +952,7 @@ async function autoIngestImpl( // entries relevant to this source, assemble a reduced index. // Runs BEFORE budget calculation so stableContextLength reflects // the actual (reduced) context size, not the full index length. - const CHUNK_SIZE = 200 + const CHUNK_SIZE = 500 let reducedIndex = index if (index.trim()) { const chunks = chunkIndexByEntries(index, CHUNK_SIZE) @@ -985,7 +986,7 @@ async function autoIngestImpl( // ── Step 0.8: Pre-match overview sections ───────────────────── let reducedOverview = overview if (overview.trim()) { - const overviewChunks = chunkOverviewBySections(overview, 8000) + const overviewChunks = chunkOverviewBySections(overview, 16000) if (overviewChunks.length > 0) { activity.updateItem(activityId, { detail: `Step 0.8: Pre-matching overview (${overviewChunks.length} chunks)...`, @@ -1063,7 +1064,7 @@ async function autoIngestImpl( }, }, signal, - { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: 4096 }, + { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: computeIngestReviewMaxTokens(llmConfig.maxContextSize) }, ) } @@ -1152,7 +1153,7 @@ async function autoIngestImpl( }, { role: "user", - content: "Emit only high-value REVIEW blocks for follow-up research or unresolved knowledge gaps. Output nothing if there are none.", + content: "Emit only high-value REVIEW blocks for follow-up research or unresolved knowledge gaps. If there are none, output exactly: none", }, ], { @@ -2575,6 +2576,7 @@ export function computeIngestSourceBudget( export function computeIngestGenerationMaxTokens(maxContextSize: number | undefined): number { const { maxCtx } = computeContextBudget(maxContextSize) + if (maxCtx >= 1_000_000) return INGEST_GENERATION_TOKENS_1M if (maxCtx >= 512_000) return INGEST_GENERATION_TOKENS_512K if (maxCtx >= 256_000) return INGEST_GENERATION_TOKENS_256K if (maxCtx >= 128_000) return INGEST_GENERATION_TOKENS_128K @@ -2582,7 +2584,7 @@ export function computeIngestGenerationMaxTokens(maxContextSize: number | undefi } export function computeIngestReviewMaxTokens(maxContextSize: number | undefined): number { - return Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize) / 2)) + return Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize))) } function splitOversizedBlock(block: string, targetChars: number): string[] { @@ -2942,7 +2944,7 @@ async function analyzeLongSourceInChunks( }, }, signal, - { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: 4096 }, + { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: computeIngestReviewMaxTokens(llmConfig.maxContextSize) }, ) throwIfIngestAborted(signal, activityId) diff --git a/src/lib/llm-providers.test.ts b/src/lib/llm-providers.test.ts index 885fd9719..f065b6bff 100755 --- a/src/lib/llm-providers.test.ts +++ b/src/lib/llm-providers.test.ts @@ -540,10 +540,10 @@ describe("reasoning controls", () => { expect(body.reasoning).toBeUndefined() }) - it("maps DeepSeek V4 reasoning off to thinking disabled for structured tasks", () => { + it("maps DeepSeek V4 pro reasoning off to thinking disabled for structured tasks", () => { const cfg = mkConfig({ provider: "custom", - model: "deepseek-v4-flash", + model: "deepseek-v4-pro", customEndpoint: "https://api.deepseek.com/v1", apiMode: "chat_completions", }) @@ -556,6 +556,37 @@ describe("reasoning controls", () => { expect(body.reasoning).toBeUndefined() }) + it("maps DeepSeek V4 flash reasoning off to thinking disabled", () => { + const cfg = mkConfig({ + provider: "custom", + model: "deepseek-v4-flash", + customEndpoint: "https://api.deepseek.com/v1", + apiMode: "chat_completions", + }) + const body = getProviderConfig(cfg).buildBody( + [{ role: "user", content: "hi" }], + { reasoning: { mode: "off" } }, + ) as Record + + expect(body.thinking).toEqual({ type: "disabled" }) + }) + + it("sends thinking=enabled for flash when reasoning mode is max", () => { + const cfg = mkConfig({ + provider: "custom", + model: "deepseek-v4-flash", + customEndpoint: "https://api.deepseek.com/v1", + apiMode: "chat_completions", + }) + const body = getProviderConfig(cfg).buildBody( + [{ role: "user", content: "hi" }], + { reasoning: { mode: "max" } }, + ) as Record + + expect(body.thinking).toEqual({ type: "enabled" }) + expect(body.reasoning_effort).toBe("max") + }) + it("maps DeepSeek V4 high reasoning to thinking enabled plus reasoning_effort", () => { const cfg = mkConfig({ provider: "custom", diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index 628f787dd..680f1f6ce 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -130,7 +130,7 @@ export function parseOverviewPrematchOutput(output: string): number[] { export function buildOverviewPrematchPrompt(sourceContent: string, chunk: string): string { return [ "You are a relevance matcher. Read the source document and determine", - "which overview paragraphs are related to it.", + "which overview paragraphs would need to be UPDATED based on it.", "", "## Source Document", sourceContent, @@ -226,7 +226,7 @@ export async function runOverviewPrematchParallel( }, }, signal, - { temperature: 0.1, reasoning: { mode: "off" } }, + { temperature: 0.1, reasoning: { mode: "max" } }, ) } catch (err) { hadError = true From 01ab7e005ce5e20db0ea0ff8b83f4fe4fe5170f3 Mon Sep 17 00:00:00 2001 From: wkh Date: Wed, 1 Jul 2026 10:47:02 +0800 Subject: [PATCH 24/29] =?UTF-8?q?perf:=20aggregate-repair=20=E2=80=94=20re?= =?UTF-8?q?move=20unused=20full=20index/overview=20context?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregate repair only writes log.md now (index/overview are updated incrementally via INDEX/OVERVIEW blocks in the generation step). - Pass empty strings instead of full index (125K chars) and full overview (185K chars) as context - Remove prompt instructions for generating complete index/overview --- src/lib/ingest.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 05dc1c5bf..4ee2ae6ae 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -1297,8 +1297,8 @@ async function autoIngestImpl( content: buildAggregateRepairPrompt( repairableAggregatePaths, purpose, - index, - overview, + "", + "", sourceIdentity, analysis, sourceContext, @@ -2509,8 +2509,6 @@ function buildAggregateRepairPrompt( "", "Rules:", `- Use today's date ${today} for log entries and frontmatter dates.`, - "- For wiki/index.md: output the complete updated index, preserving existing entries and adding the new source-derived entries.", - "- For wiki/overview.md: output the complete updated overview, reflecting the full wiki plus this new source.", "- For wiki/log.md: output only the new log entry to append, format `## [YYYY-MM-DD] ingest | Title`.", "- Output only FILE blocks. Nothing else.", "", From 6073b61c62de69e612765885090623c907d8581b Mon Sep 17 00:00:00 2001 From: wkh Date: Wed, 1 Jul 2026 10:47:56 +0800 Subject: [PATCH 25/29] =?UTF-8?q?perf:=20review-suggestion=20=E2=80=94=20s?= =?UTF-8?q?plit=20missing-page=20detection=20to=20code,=20use=20reducedInd?= =?UTF-8?q?ex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add detectCodeReviewItems: pure-code missing-page/duplicate detection by scanning generation output wikilinks against wiki index and current FILE blocks. 100% accurate, 0 LLM tokens. - LLM prompt narrowed to suggestion + contradiction only (no longer asks about missing-page or duplicate). - Pass reducedIndex (prematch-filtered) instead of full wiki index to the LLM. Preserves cross-page suggestion capability at minimal cost. - Review input drops from ~105K tokens to ~12K tokens (89% reduction). --- src/lib/ingest.ts | 79 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index 4ee2ae6ae..db125c870 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -1143,7 +1143,7 @@ async function autoIngestImpl( role: "system", content: buildReviewSuggestionPrompt( purpose, - index, + reducedIndex, sourceIdentity, analysis, sourceContext, @@ -1406,11 +1406,12 @@ async function autoIngestImpl( } } - // ── Step 4: Parse review items ──────────────────────────────── + // ── Step 4: Parse review items ──────────────────────────────── throwIfIngestAborted(signal, activityId) const reviewItems = [ ...parseReviewBlocks(generation, sp), ...parseReviewBlocks(reviewSuggestionOutput, sp), + ...detectCodeReviewItems(generation, index, sp), ] if (reviewItems.length > 0) { useReviewStore.getState().addItems(reviewItems) @@ -2087,6 +2088,66 @@ function isOwnedOnlyBySource(content: string, sourceIdentity: string): boolean { const REVIEW_BLOCK_REGEX = /---REVIEW:\s*(\w[\w-]*)\s*\|\s*(.+?)\s*---\n([\s\S]*?)---END REVIEW---/g +/** + * Scan generation output for wikilinks and detect missing pages / duplicates + * purely from the wiki index — no LLM call needed. + */ +function detectCodeReviewItems( + generation: string, + index: string, + sourcePath: string, +): Omit[] { + if (!generation.trim() || !index.trim()) return [] + + // Collect existing slugs from the wiki index + const existingSlugs = new Set() + for (const m of index.matchAll(/\[\[([^\]|#]+)(?:\|[^\]]*)?\]\]/g)) { + existingSlugs.add(m[1].trim().toLowerCase()) + } + + // Also collect slugs from FILE blocks in the generation output — + // pages being created by this ingest should not be reported as missing. + // Match: ---FILE: wiki/.../slug.md AND title: "Title Name" inside frontmatter. + const FILE_BLOCK_REGEX = /---FILE:\s*(wiki\/[^\n]+?\.md)[^\n]*\n([\s\S]*?)---END FILE---/g + for (const fb of generation.matchAll(FILE_BLOCK_REGEX)) { + const filePath = fb[1].trim() + const frontmatter = fb[2] + // Add file-stem slug (wiki/concepts/rope → rope) + const stem = filePath.replace(/^wiki\/[^/]+\/(.+)\.md$/, "$1") + existingSlugs.add(stem.toLowerCase()) + // Add frontmatter title + const titleMatch = frontmatter.match(/^title:\s*"?(.+?)"?\s*$/m) + if (titleMatch) existingSlugs.add(titleMatch[1].trim().toLowerCase()) + } + + // Extract referenced slugs from the generation output + const wikilinkRegex = /\[\[([^\]|#]+)(?:\|[^\]]*)?\]\]/g + const referencedSlugs = new Map() + for (const m of generation.matchAll(wikilinkRegex)) { + const slug = m[1].trim() + const key = slug.toLowerCase() + if (!referencedSlugs.has(key)) referencedSlugs.set(key, slug) + } + + const items: Omit[] = [] + for (const [key, original] of referencedSlugs) { + if (!existingSlugs.has(key)) { + items.push({ + type: "missing-page", + title: original, + description: `Page "[[${original}]]" is referenced but does not exist in the wiki. Created automatically from source ingestion.`, + sourcePath, + options: [ + { label: "Create Page", action: "Create Page" }, + { label: "Skip", action: "Skip" }, + ], + }) + } + } + return items +} + + function parseReviewBlocks( text: string, sourcePath: string, @@ -2425,7 +2486,7 @@ export function buildGenerationPrompt( function buildReviewSuggestionPrompt( purpose: string, - index: string, + reducedIndex: string, sourceIdentity: string, analysis: string, sourceContext: string, @@ -2434,7 +2495,6 @@ function buildReviewSuggestionPrompt( ): string { const { maxCtx } = computeContextBudget(maxContextSize) const sectionCap = Math.max(4_000, Math.floor(maxCtx * 0.15)) - const indexCap = Math.max(3_000, Math.floor(sectionCap * 0.8)) return [ "You are identifying high-value follow-up research items for a personal wiki.", "Do not output chain-of-thought, hidden reasoning, or explanatory preamble.", @@ -2442,16 +2502,13 @@ function buildReviewSuggestionPrompt( languageRule(sourceContext), "", "Your job is NOT to generate wiki pages. The wiki page generation already happened.", - "Output only REVIEW blocks for unresolved knowledge gaps that deserve human attention or Deep Research.", + "Missing-page and duplicate checks are handled automatically. Your focus:", "", - "Create REVIEW blocks only for genuinely useful follow-up work:", - "- missing-page: an important entity/concept is referenced but still lacks a dedicated page", "- suggestion: a research question, source type, or comparison that would materially improve the wiki", "- contradiction: a conflict or tension that requires user judgment", - "- duplicate: likely duplicate pages/names that need user review", "", - "Prefer 1-5 high-signal reviews. If there is nothing worth reviewing, output nothing.", - "For suggestion and missing-page reviews, include a SEARCH line with 2-3 keyword-rich web search queries separated by ` | `.", + "Prefer 1-5 high-signal reviews. If there is nothing worth reviewing, output exactly: none.", + "For suggestion reviews, include a SEARCH line with 2-3 keyword-rich web search queries separated by ` | `.", "Use only these options: OPTIONS: Create Page | Skip", "", "REVIEW block template:", @@ -2467,7 +2524,7 @@ function buildReviewSuggestionPrompt( "Return REVIEW blocks only. Do not output FILE blocks. Do not wrap the response in markdown fences.", "", purpose ? `## Wiki Purpose\n${purpose}` : "", - index ? `## Current Wiki Index\n${trimLongText(index, indexCap)}` : "", + reducedIndex ? `## Relevant Wiki Index\n${reducedIndex}` : "", "", `## Source\n${sourceIdentity}`, "", From 3428fe1d99aae9b79a301522f8a37b691458b2a4 Mon Sep 17 00:00:00 2001 From: wkh Date: Wed, 1 Jul 2026 11:04:51 +0800 Subject: [PATCH 26/29] =?UTF-8?q?perf:=20prematch=20reasoning=20max=20?= =?UTF-8?q?=E2=86=92=20high,=20other=20stages=20stay=20off?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeepSeek API only supports high/max for reasoning_effort. Using high instead of max reduces thinking cost while maintaining accuracy for prematch discrimination. Non-prematch stages keep off. --- src/lib/index-chunker.ts | 2 +- src/lib/overview-blocks.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 38cc56861..9058f4093 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -302,7 +302,7 @@ export async function runPrematchParallel( }, }, signal, - { temperature: 0.1, reasoning: { mode: "max" } }, + { temperature: 0.1, reasoning: { mode: "high" } }, ) } catch (err) { hadError = true diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index 680f1f6ce..2a1a34c8d 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -226,7 +226,7 @@ export async function runOverviewPrematchParallel( }, }, signal, - { temperature: 0.1, reasoning: { mode: "max" } }, + { temperature: 0.1, reasoning: { mode: "high" } }, ) } catch (err) { hadError = true From c38f7b2dc779fa66db072bee0102a9ed30aee493 Mon Sep 17 00:00:00 2001 From: wkh Date: Wed, 1 Jul 2026 11:30:45 +0800 Subject: [PATCH 27/29] fix: treat reasoning-only response as 'none' in prematch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When reasoning=high, the model may produce reasoning_content but no content — meaning it concluded no paragraphs match. Treat this as a valid empty result instead of logging a warning. --- src/lib/index-chunker.ts | 3 ++- src/lib/overview-blocks.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/index-chunker.ts b/src/lib/index-chunker.ts index 9058f4093..90a301306 100755 --- a/src/lib/index-chunker.ts +++ b/src/lib/index-chunker.ts @@ -4,7 +4,7 @@ * Entries are numbered sequentially with [N] prefix across all chunks. */ -import { streamChat } from "@/lib/llm-client" +import { streamChat, isReasoningOnlyResponseError } from "@/lib/llm-client" import type { LlmConfig } from "@/stores/wiki-store" interface IndexEntry { @@ -305,6 +305,7 @@ export async function runPrematchParallel( { temperature: 0.1, reasoning: { mode: "high" } }, ) } catch (err) { + if (isReasoningOnlyResponseError(err)) return [] hadError = true console.warn(`[prematch] chunk threw: ${err instanceof Error ? err.message : String(err)}`) } diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index 2a1a34c8d..c2ff8749e 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -7,7 +7,7 @@ * incremental append on top of this file. */ -import { streamChat } from "@/lib/llm-client" +import { streamChat, isReasoningOnlyResponseError } from "@/lib/llm-client" import { currentWikiDate } from "@/lib/ingest" import type { LlmConfig } from "@/stores/wiki-store" @@ -229,6 +229,7 @@ export async function runOverviewPrematchParallel( { temperature: 0.1, reasoning: { mode: "high" } }, ) } catch (err) { + if (isReasoningOnlyResponseError(err)) return [] hadError = true console.warn(`[overview-prematch] chunk threw: ${err instanceof Error ? err.message : String(err)}`) } From 92c4366ae300b9c8388cdec1664cd054e912fc56 Mon Sep 17 00:00:00 2001 From: wkh Date: Wed, 1 Jul 2026 17:19:13 +0800 Subject: [PATCH 28/29] chore: add .worktrees/ to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 130b1b243..18d808c28 100755 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ pnpm-debug.log* .superpowers/ .claude/ .playwright-mcp/ + +# Worktrees +.worktrees/ From d8be245bc3da794f10f79402a56c2b6544cac695 Mon Sep 17 00:00:00 2001 From: wkh Date: Wed, 1 Jul 2026 21:30:14 +0800 Subject: [PATCH 29/29] fix: avoid duplicate overview paragraphs on re-ingest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Prematch prompt: tell model not to match paragraphs that already cover the topic adequately — re-ingesting the same topic does not require an update. - Generation OVERVIEW instruction: if the source only restates information already in the overview context, output NO OVERVIEW block. - Remove leftover text from previous incomplete revert. --- src/lib/ingest.ts | 2 +- src/lib/overview-blocks.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts index db125c870..3894f1f73 100644 --- a/src/lib/ingest.ts +++ b/src/lib/ingest.ts @@ -2412,7 +2412,7 @@ export function buildGenerationPrompt( "", purpose ? `## Wiki Purpose\n${purpose}` : "", index ? `## Current Wiki Index (for reference only — do NOT reproduce it)\n${index}` : "", - reducedOverview ? `## Current Overview (relevant sections only — for reference)\n${reducedOverview}` : "", + reducedOverview ? `## Current Overview (relevant sections only — for reference)\nIf the source only restates information already covered in this context,\noutput NO OVERVIEW block — skip it entirely.\n\n${reducedOverview}` : "", "", // ── OUTPUT FORMAT MUST BE THE LAST SECTION — models weight recent instructions highest ── "## Output Format (MUST FOLLOW EXACTLY — this is how the parser reads your response)", diff --git a/src/lib/overview-blocks.ts b/src/lib/overview-blocks.ts index c2ff8749e..89824f206 100644 --- a/src/lib/overview-blocks.ts +++ b/src/lib/overview-blocks.ts @@ -137,9 +137,10 @@ export function buildOverviewPrematchPrompt(sourceContent: string, chunk: string "", "## Overview Paragraphs", "Below is a chunk of the wiki overview. Each numbered item is a paragraph.", - "For each paragraph, determine whether it covers the same subject as any", - "topic in the source document. A match means the paragraph discusses the", - "same entity, concept, method, or topic area.", + "For each paragraph, determine whether the source adds NEW information that", + "would meaningfully change or extend it. Do NOT match a paragraph if it", + "already covers the topic adequately and the source only restates the same", + "facts — re-ingesting the same topic does not require an update.", "", chunk, "",