diff --git a/src/context/analyzer.ts b/src/context/analyzer.ts index 57f2631..36bc2e2 100644 --- a/src/context/analyzer.ts +++ b/src/context/analyzer.ts @@ -69,61 +69,389 @@ const STOP_WORDS = new Set([ "each", "every", "both", + "can", + "could", + "may", + "might", + "must", + "shall", + "should", + "will", + "would", + "just", + "now", + "then", + "there", + "here", + "also", + "only", + "very", + "really", + "quite", + "rather", + "some", + "any", + "many", + "much", "few", "more", "most", - "other", - "some", + "less", + "least", + "than", "such", + "other", + "another", + "same", + "different", + "new", + "old", + "good", + "bad", + "big", + "small", + "large", + "short", + "long", + "high", + "low", + "right", + "wrong", + "true", + "false", + "yes", "no", - "nor", "not", - "only", - "own", - "same", - "so", - "than", - "too", - "very", - "just", - "can", - "should", + "never", + "always", + "often", + "sometimes", + "usually", + "generally", + "specifically", + "particularly", + "actually", + "really", + "certainly", + "definitely", + "probably", + "possibly", + "maybe", + "perhaps", + "like", + "such", + "well", + "even", + "though", + "although", + "because", + "since", + "while", + "during", + "after", + "before", + "until", + "since", + "ago", + "later", + "earlier", + "soon", "now", - "into", - "also", - "been", - "being", - "do", - "does", - "did", - "doing", - "would", - "could", - "might", - "must", - "shall", - "may", - "add", - "added", - "adding", + "then", + "when", + "where", + "why", + "how", + "what", + "which", + "who", + "whose", + "whom", + "that", + "this", + "these", + "those", + "i", + "me", + "my", + "myself", + "you", + "your", + "yourself", + "he", + "him", + "his", + "himself", + "she", + "her", + "hers", + "herself", + "it", + "its", + "itself", + "we", + "us", + "our", + "ourselves", + "they", + "them", + "their", + "themselves", + "one", + "ones", + "someone", + "somebody", + "something", + "everyone", + "everybody", + "everything", + "nobody", + "nothing", + "none", + "anybody", + "anything", + "each", + "either", + "neither", + "both", + "all", + "any", + "some", + "many", + "much", + "few", + "little", + "more", + "most", + "least", + "less", + "than", + "such", + "other", + "another", + "same", + "different", + "changing", +]); + +// Generic keywords that don't provide meaningful cluster themes +const GENERIC_KEYWORDS = new Set([ + // Source/tool specific + "opencode", + "background", + "zsh", + "bash", + "terminal", + "shell", + "command", + "commands", + "git", + "file", + "files", + "vscode", + "cursor", + "editor", + "code", + "coding", + "development", + "programming", + "software", + + // Generic work terms + "project", + "projects", + "work", + "working", + "task", + "tasks", + "item", + "items", + "activity", + "activities", + "session", + "sessions", + "window", + "windows", + "tab", + "tabs", + "page", + "pages", + + // Generic data/tech terms + "data", + "database", + "db", + "api", + "apis", + "server", + "servers", + "client", + "clients", + "request", + "requests", + "response", + "responses", + "error", + "errors", + + // Generic issue/PR terms + "issue", + "issues", + "bug", + "bugs", "fix", - "fixed", - "fixing", + "fixes", + "feature", + "features", "update", - "updated", - "updating", - "implement", - "implemented", - "implementing", + "updates", + "change", + "changes", + "commit", + "commits", + "push", + "pull", + "merge", + "branch", + "branches", + "repository", + "repositories", + "repo", + "repos", + + // GitHub specific + "github", + "pr", + "pull", + "request", + "opened", + "merged", + "closed", + + // User/repo names (these are too generic in PR titles) + "jvalentini", + "worklog", + "main", + "master", + "develop", + "development", + + // Common verbs that don't help clustering + "add", + "added", "create", "created", - "creating", + "update", + "updated", "remove", "removed", - "removing", - "change", - "changed", - "changing", + "delete", + "deleted", + "implement", + "implemented", + "improve", + "improved", + "fix", + "fixed", + "ensure", + "ensures", + "make", + "made", + "set", + "setting", + "get", + "getting", + "run", + "running", + "start", + "started", + "stop", + "stopped", + "enable", + "enabled", + "disable", + "disabled", + + // PR/commit specific terms + "pr", + "pull", + "request", + "requests", + "merge", + "merges", + "merged", + "commit", + "commits", + "branch", + "branches", + + // More generic action words + "trigger", + "triggers", + "triggered", + "handle", + "handles", + "handled", + "support", + "supports", + "supported", + "resolve", + "resolves", + "resolved", + "address", + "addresses", + "addressed", + + // Tool/framework names that are too generic + "release", + "please", + "npm", + "bun", + "node", + "typescript", + "ts", + "js", + "javascript", + "react", + "vue", + "angular", + "next", + "nuxt", + "express", + "fastify", + "docker", + "kubernetes", + "aws", + "gcp", + "azure", + "postgres", + "postgresql", + "mysql", + "mongodb", + "redis", + "elasticsearch", + + // Even more generic terms + "new", + "old", + "good", + "bad", + "better", + "best", + "simple", + "complex", + "easy", + "hard", + "quick", + "fast", + "slow", + "small", + "large", + "big", + "little", + "major", + "minor", + "patch", ]); function tokenize(text: string): string[] { @@ -223,7 +551,28 @@ export function computeSimilarityMatrix(items: WorkItem[]): number[][] { } else { const vecJ = vectors[j]; if (vecJ) { - row[j] = cosineSimilarity(vecI, vecJ); + let similarity = cosineSimilarity(vecI, vecJ); + + // Boost similarity for clearly related items + const itemI = items[i]; + const itemJ = items[j]; + if (itemI && itemJ) { + // GitHub PR events with same PR number should be highly similar + const prMatchI = itemI.title.match(/PR #(\d+)/); + const prMatchJ = itemJ.title.match(/PR #(\d+)/); + if (prMatchI && prMatchJ && prMatchI[1] === prMatchJ[1]) { + similarity = Math.max(similarity, 0.9); + } + + // Same commit hash should be highly similar + const commitMatchI = itemI.title.match(/([a-f0-9]{7,40})/); + const commitMatchJ = itemJ.title.match(/([a-f0-9]{7,40})/); + if (commitMatchI && commitMatchJ && commitMatchI[1] === commitMatchJ[1]) { + similarity = Math.max(similarity, 0.95); + } + } + + row[j] = similarity; } } } @@ -313,13 +662,75 @@ function computeClusterCoherence(items: WorkItem[]): number { } function generateThemeLabel(keywords: string[], items: WorkItem[]): string { - if (keywords.length === 0) { + // Filter out generic keywords that don't provide meaningful themes + const meaningfulKeywords = keywords.filter((k) => !GENERIC_KEYWORDS.has(k.toLowerCase())); + + if (meaningfulKeywords.length === 0) { + // If no meaningful keywords, try to generate a theme from item content + const allText = items.map((i) => `${i.title} ${i.description ?? ""}`).join(" "); + const titleTokens = tokenize(allText).filter((t) => !GENERIC_KEYWORDS.has(t.toLowerCase())); + + if (titleTokens.length > 0) { + // Use the most common meaningful words from all content + const tokenCounts = new Map(); + for (const token of titleTokens) { + tokenCounts.set(token, (tokenCounts.get(token) ?? 0) + 1); + } + const sortedTokens = [...tokenCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3); // Get top 3 most meaningful terms + + // Try to find the most specific/unique terms + const meaningfulTokens = sortedTokens + .map(([token]) => token) + .filter((token) => token.length > 3) // Prefer longer, more specific words + .slice(0, 2); // Take at most 2 + + if (meaningfulTokens.length >= 2) { + const first = meaningfulTokens[0]; + const second = meaningfulTokens[1]; + if (first && second) { + return `${capitalize(first)} & ${capitalize(second)}`; + } + } else if (meaningfulTokens.length === 1) { + const first = meaningfulTokens[0]; + if (first) { + return capitalize(first); + } + } + } + + // Last resort: try to find any nouns or proper names + const fullContent = items.map((i) => `${i.title} ${i.description ?? ""}`).join(" "); + const matchedWords = fullContent.toLowerCase().match(/\b[a-z]{4,}\b/g); // Words of 4+ characters + const fallbackTokens: string[] = + matchedWords?.filter((word): word is string => !GENERIC_KEYWORDS.has(word))?.slice(0, 2) ?? + []; + + if (fallbackTokens.length >= 2) { + const first = fallbackTokens[0]; + const second = fallbackTokens[1]; + if (first && second) { + return `${capitalize(first)} & ${capitalize(second)}`; + } + } else if (fallbackTokens.length === 1) { + const first = fallbackTokens[0]; + if (first) { + return capitalize(first); + } + } + + // Ultimate fallback to source-based naming const sources = [...new Set(items.map((i) => i.source))]; return sources.length === 1 ? `${sources[0]} activity` : "Mixed activity"; } - const primary = keywords[0]; - const secondary = keywords[1]; + // For meaningful keywords, prefer longer, more specific ones + const sortedKeywords = meaningfulKeywords + .sort((a, b) => b.length - a.length) // Longer words first + .slice(0, 2); + + const primary = sortedKeywords[0]; + const secondary = sortedKeywords[1]; + if (!primary) { const sources = [...new Set(items.map((i) => i.source))]; return sources.length === 1 ? `${sources[0]} activity` : "Mixed activity"; @@ -347,9 +758,14 @@ export function findCrossClusterConnections( const b = clusters[j]; if (!a || !b) continue; - const sharedKeywords = a.keywords.filter((k) => b.keywords.includes(k)); + // Only consider meaningful shared keywords (filter out generic ones) + const sharedKeywords = a.keywords + .filter((k) => b.keywords.includes(k)) + .filter((k) => !GENERIC_KEYWORDS.has(k.toLowerCase())); - if (sharedKeywords.length > 0) { + // Only create connections if there are meaningful shared keywords + // and the clusters are actually different (not just the same generic keywords) + if (sharedKeywords.length > 0 && a.theme !== b.theme) { connections.push({ from: a.id, to: b.id,