diff --git a/CHANGELOG.md b/CHANGELOG.md index bea08bd56..cc31dbbf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - [EE] Add Ask chat usage metrics to analytics dashboard [#736](https://github.com/sourcebot-dev/sourcebot/pull/736) +### Changed +- Improved initial file tree load times, especially for larger repositories. [#739](https://github.com/sourcebot-dev/sourcebot/pull/739) + ## [4.10.9] - 2026-01-14 ### Changed diff --git a/packages/web/src/app/api/(client)/client.ts b/packages/web/src/app/api/(client)/client.ts index 08a01b5b4..2bf319935 100644 --- a/packages/web/src/app/api/(client)/client.ts +++ b/packages/web/src/app/api/(client)/client.ts @@ -101,4 +101,4 @@ export const getFiles = async (body: GetFilesRequest): Promise response.json()); return result as GetFilesResponse | ServiceError; -} \ No newline at end of file +} diff --git a/packages/web/src/features/fileTree/api.ts b/packages/web/src/features/fileTree/api.ts index ed4c7aede..0e9ea5111 100644 --- a/packages/web/src/features/fileTree/api.ts +++ b/packages/web/src/features/fileTree/api.ts @@ -8,17 +8,20 @@ import { Repo } from '@sourcebot/db'; import { createLogger } from '@sourcebot/shared'; import path from 'path'; import { simpleGit } from 'simple-git'; -import { FileTreeItem, FileTreeNode } from './types'; +import { FileTreeItem } from './types'; +import { buildFileTree, isPathValid, normalizePath } from './utils'; +import { compareFileTreeItems } from './utils'; const logger = createLogger('file-tree'); /** - * Returns the tree of files (blobs) and directories (trees) for a given repository, - * at a given revision. + * Returns a file tree spanning the union of all provided paths for the given + * repo/revision, including intermediate directories needed to connect them + * into a single tree. */ -export const getTree = async (params: { repoName: string, revisionName: string }) => sew(() => +export const getTree = async (params: { repoName: string, revisionName: string, paths: string[] }) => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { - const { repoName, revisionName } = params; + const { repoName, revisionName, paths } = params; const repo = await prisma.repo.findFirst({ where: { name: repoName, @@ -33,21 +36,30 @@ export const getTree = async (params: { repoName: string, revisionName: string } const { path: repoPath } = getRepoPath(repo); const git = simpleGit().cwd(repoPath); + if (!paths.every(path => isPathValid(path))) { + return notFound(); + } - let result: string; + const normalizedPaths = paths.map(path => normalizePath(path)); + + let result: string = ''; try { - result = await git.raw([ + + const command = [ // Disable quoting of non-ASCII characters in paths '-c', 'core.quotePath=false', 'ls-tree', revisionName, - // recursive - '-r', - // include trees when recursing - '-t', // format as output as {type},{path} '--format=%(objecttype),%(path)', - ]); + // include tree nodes + '-t', + '--', + '.', + ...normalizedPaths, + ]; + + result = await git.raw(command); } catch (error) { logger.error('git ls-tree failed.', { error }); return unexpectedError('git ls-tree command failed.'); @@ -90,31 +102,12 @@ export const getFolderContents = async (params: { repoName: string, revisionName } const { path: repoPath } = getRepoPath(repo); + const git = simpleGit().cwd(repoPath); - // @note: we don't allow directory traversal - // or null bytes in the path. - if (path.includes('..') || path.includes('\0')) { + if (!isPathValid(path)) { return notFound(); } - - // Normalize the path by... - let normalizedPath = path; - - // ... adding a trailing slash if it doesn't have one. - // This is important since ls-tree won't return the contents - // of a directory if it doesn't have a trailing slash. - if (!normalizedPath.endsWith('/')) { - normalizedPath = `${normalizedPath}/`; - } - - // ... removing any leading slashes. This is needed since - // the path is relative to the repository's root, so we - // need a relative path. - if (normalizedPath.startsWith('/')) { - normalizedPath = normalizedPath.slice(1); - } - - const git = simpleGit().cwd(repoPath); + const normalizedPath = normalizePath(path); let result: string; try { @@ -145,6 +138,9 @@ export const getFolderContents = async (params: { repoName: string, revisionName } }); + // Sort the contents in place, first by type (trees before blobs), then by name. + contents.sort(compareFileTreeItems); + return contents; })); @@ -199,60 +195,6 @@ export const getFiles = async (params: { repoName: string, revisionName: string })); -const buildFileTree = (flatList: { type: string, path: string }[]): FileTreeNode => { - const root: FileTreeNode = { - name: 'root', - path: '', - type: 'tree', - children: [], - }; - - for (const item of flatList) { - const parts = item.path.split('/'); - let current: FileTreeNode = root; - - for (let i = 0; i < parts.length; i++) { - const part = parts[i]; - const isLeaf = i === parts.length - 1; - const nodeType = isLeaf ? item.type : 'tree'; - let next = current.children.find((child: FileTreeNode) => child.name === part && child.type === nodeType); - - if (!next) { - next = { - name: part, - path: item.path, - type: nodeType, - children: [], - }; - current.children.push(next); - } - current = next; - } - } - - const sortTree = (node: FileTreeNode): FileTreeNode => { - if (node.type === 'blob') { - return node; - } - - const sortedChildren = node.children - .map(sortTree) - .sort((a: FileTreeNode, b: FileTreeNode) => { - if (a.type !== b.type) { - return a.type === 'tree' ? -1 : 1; - } - return a.name.localeCompare(b.name, undefined, { sensitivity: 'base' }); - }); - - return { - ...node, - children: sortedChildren, - }; - }; - - return sortTree(root); -} - // @todo: this is duplicated from the `getRepoPath` function in the // backend's `utils.ts` file. Eventually we should move this to a shared // package. diff --git a/packages/web/src/features/fileTree/components/fileTreePanel.tsx b/packages/web/src/features/fileTree/components/fileTreePanel.tsx index eb751bacf..e96e3dbc4 100644 --- a/packages/web/src/features/fileTree/components/fileTreePanel.tsx +++ b/packages/web/src/features/fileTree/components/fileTreePanel.tsx @@ -9,19 +9,20 @@ import { ResizablePanel } from "@/components/ui/resizable"; import { Separator } from "@/components/ui/separator"; import { Skeleton } from "@/components/ui/skeleton"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { unwrapServiceError } from "@/lib/utils"; +import useCaptureEvent from "@/hooks/useCaptureEvent"; +import { measure, unwrapServiceError } from "@/lib/utils"; import { useQuery } from "@tanstack/react-query"; import { SearchIcon } from "lucide-react"; -import { useRef } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { useHotkeys } from "react-hotkeys-hook"; import { GoSidebarExpand as CollapseIcon, GoSidebarCollapse as ExpandIcon } from "react-icons/go"; import { ImperativePanelHandle } from "react-resizable-panels"; +import { FileTreeNode } from "../types"; import { PureFileTreePanel } from "./pureFileTreePanel"; - interface FileTreePanelProps { order: number; } @@ -30,7 +31,6 @@ const FILE_TREE_PANEL_DEFAULT_SIZE = 20; const FILE_TREE_PANEL_MIN_SIZE = 10; const FILE_TREE_PANEL_MAX_SIZE = 30; - export const FileTreePanel = ({ order }: FileTreePanelProps) => { const { state: { @@ -39,19 +39,79 @@ export const FileTreePanel = ({ order }: FileTreePanelProps) => { updateBrowseState, } = useBrowseState(); - const { repoName, revisionName, path } = useBrowseParams(); + const { repoName, revisionName, path, pathType } = useBrowseParams(); + const [openPaths, setOpenPaths] = useState>(new Set()); + const captureEvent = useCaptureEvent(); const fileTreePanelRef = useRef(null); - const { data, isPending, isError } = useQuery({ - queryKey: ['tree', repoName, revisionName], - queryFn: () => unwrapServiceError( - getTree({ - repoName, - revisionName: revisionName ?? 'HEAD', - }) - ), + + const { data, isError, isPending } = useQuery({ + queryKey: ['tree', repoName, revisionName, ...Array.from(openPaths)], + queryFn: async () => { + const result = await measure(async () => unwrapServiceError( + getTree({ + repoName, + revisionName: revisionName ?? 'HEAD', + paths: Array.from(openPaths), + }) + ), 'getTree'); + + captureEvent('wa_file_tree_loaded', { + durationMs: result.durationMs, + }); + + return result.data; + }, + // The tree changes only when the query key changes (repo/revision/openPaths), + // so we can treat it as perpetually fresh and avoid background refetches. + staleTime: Infinity, + // Reuse the last tree during refetches (openPaths changes) to avoid UI flicker. + placeholderData: (previousData) => previousData, }); - + + // Whenever the repo name or revision name changes, we will need to + // reset the open paths since they no longer reference the same repository/revision. + useEffect(() => { + setOpenPaths(new Set()); + }, [repoName, revisionName]); + + // When the path changes (e.g., the user clicks a reference in the explore panel), + // we want this to be open and visible in the file tree. + useEffect(() => { + let pathParts = path.split('/').filter(Boolean); + + // If the path is a blob, we want to open the parent directory. + if (pathType === 'blob') { + pathParts = pathParts.slice(0, -1); + } + + setOpenPaths(current => { + const next = new Set(current); + for (let i = 0; i < pathParts.length; i++) { + next.add(pathParts.slice(0, i + 1).join('/')); + } + return next; + }); + }, [path, pathType]); + + // When the user clicks a file tree node, we will want to either + // add or remove it from the open paths depending on if it's already open or not. + const onTreeNodeClicked = useCallback((node: FileTreeNode) => { + if (!openPaths.has(node.path)) { + setOpenPaths(current => { + const next = new Set(current); + next.add(node.path); + return next; + }) + } else { + setOpenPaths(current => { + const next = new Set(current); + next.delete(node.path); + return next; + }) + } + }, [openPaths]); + useHotkeys("mod+b", () => { if (isFileTreePanelCollapsed) { fileTreePanelRef.current?.expand(); @@ -132,7 +192,9 @@ export const FileTreePanel = ({ order }: FileTreePanelProps) => { ) : ( )} @@ -323,4 +385,4 @@ const FileTreePanelSkeleton = () => { ) -} \ No newline at end of file +} diff --git a/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx b/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx index 9e8811292..0f7a15c1b 100644 --- a/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx +++ b/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx @@ -1,77 +1,35 @@ 'use client'; -import { FileTreeNode as RawFileTreeNode } from "../types"; +import { FileTreeNode } from "../types"; import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; -import React, { useCallback, useMemo, useState, useEffect, useRef } from "react"; +import React, { useCallback, useMemo, useRef } from "react"; import { FileTreeItemComponent } from "./fileTreeItemComponent"; import { getBrowsePath } from "@/app/[domain]/browse/hooks/utils"; import { useBrowseParams } from "@/app/[domain]/browse/hooks/useBrowseParams"; import { useDomain } from "@/hooks/useDomain"; -export type FileTreeNode = Omit & { - isCollapsed: boolean; - children: FileTreeNode[]; -} - -const buildCollapsibleTree = (tree: RawFileTreeNode): FileTreeNode => { - return { - ...tree, - isCollapsed: true, - children: tree.children.map(buildCollapsibleTree), - } -} - -const transformTree = ( - tree: FileTreeNode, - transform: (node: FileTreeNode) => FileTreeNode -): FileTreeNode => { - const newNode = transform(tree); - const newChildren = tree.children.map(child => transformTree(child, transform)); - return { - ...newNode, - children: newChildren, - } +const renderLoadingSkeleton = (depth: number) => { + return ( +
+
+
+ Loading... +
+ ); } interface PureFileTreePanelProps { - tree: RawFileTreeNode; + tree: FileTreeNode; + openPaths: Set; path: string; + onTreeNodeClicked: (node: FileTreeNode) => void; } -export const PureFileTreePanel = ({ tree: _tree, path }: PureFileTreePanelProps) => { - const [tree, setTree] = useState(buildCollapsibleTree(_tree)); +export const PureFileTreePanel = ({ tree, openPaths, path, onTreeNodeClicked }: PureFileTreePanelProps) => { const scrollAreaRef = useRef(null); const { repoName, revisionName } = useBrowseParams(); const domain = useDomain(); - // @note: When `_tree` changes, it indicates that a new tree has been loaded. - // In that case, we need to rebuild the collapsible tree. - useEffect(() => { - setTree(buildCollapsibleTree(_tree)); - }, [_tree]); - - const setIsCollapsed = useCallback((path: string, isCollapsed: boolean) => { - setTree(currentTree => transformTree(currentTree, (currentNode) => { - if (currentNode.path === path) { - currentNode.isCollapsed = isCollapsed; - } - return currentNode; - })); - }, []); - - // When the path changes, expand all the folders up to the path - useEffect(() => { - const pathParts = path.split('/'); - let currentPath = ''; - for (let i = 0; i < pathParts.length; i++) { - currentPath += pathParts[i]; - setIsCollapsed(currentPath, false); - if (i < pathParts.length - 1) { - currentPath += '/'; - } - } - }, [path, setIsCollapsed]); - const renderTree = useCallback((nodes: FileTreeNode, depth = 0): React.ReactNode => { return ( <> @@ -90,14 +48,14 @@ export const PureFileTreePanel = ({ tree: _tree, path }: PureFileTreePanelProps) node={node} isActive={node.path === path} depth={depth} - isCollapsed={node.isCollapsed} + isCollapsed={!openPaths.has(node.path)} isCollapseChevronVisible={node.type === 'tree'} // Only collapse the tree when a regular click happens. // (i.e., not ctrl/cmd click). onClick={(e) => { const isMetaOrCtrlKey = e.metaKey || e.ctrlKey; if (node.type === 'tree' && !isMetaOrCtrlKey) { - setIsCollapsed(node.path, !node.isCollapsed); + onTreeNodeClicked(node); } }} // @note: onNavigate _won't_ be called when the user ctrl/cmd clicks on a tree node. @@ -110,13 +68,19 @@ export const PureFileTreePanel = ({ tree: _tree, path }: PureFileTreePanelProps) }} parentRef={scrollAreaRef} /> - {node.children.length > 0 && !node.isCollapsed && renderTree(node, depth + 1)} + {node.type === 'tree' && node.children.length > 0 && openPaths.has(node.path) && renderTree(node, depth + 1)} + {/* + @note: a empty tree indicates that the contents are beaing loaded. Render a loading skeleton to indicate that. + This relies on the fact that you cannot have empty tress in git. + @see: https://archive.kernel.org/oldwiki/git.wiki.kernel.org/index.php/GitFaq.html#Can_I_add_empty_directories.3F + */} + {node.type === 'tree' && node.children.length === 0 && openPaths.has(node.path) && renderLoadingSkeleton(depth)} ); })} ); - }, [domain, path, repoName, revisionName, setIsCollapsed]); + }, [domain, onTreeNodeClicked, path, repoName, revisionName, openPaths]); const renderedTree = useMemo(() => renderTree(tree), [tree, renderTree]); diff --git a/packages/web/src/features/fileTree/types.ts b/packages/web/src/features/fileTree/types.ts index 0f0318f45..29aeae121 100644 --- a/packages/web/src/features/fileTree/types.ts +++ b/packages/web/src/features/fileTree/types.ts @@ -3,6 +3,7 @@ import { z } from "zod"; export const getTreeRequestSchema = z.object({ repoName: z.string(), revisionName: z.string(), + paths: z.array(z.string()), }); export type GetTreeRequest = z.infer; @@ -12,6 +13,13 @@ export const getFilesRequestSchema = z.object({ }); export type GetFilesRequest = z.infer; +export const getFolderContentsRequestSchema = z.object({ + repoName: z.string(), + revisionName: z.string(), + path: z.string(), +}); +export type GetFolderContentsRequest = z.infer; + export const fileTreeItemSchema = z.object({ type: z.string(), path: z.string(), @@ -42,3 +50,6 @@ export type GetTreeResponse = z.infer; export const getFilesResponseSchema = z.array(fileTreeItemSchema); export type GetFilesResponse = z.infer; +export const getFolderContentsResponseSchema = z.array(fileTreeItemSchema); +export type GetFolderContentsResponse = z.infer; + diff --git a/packages/web/src/features/fileTree/utils.test.ts b/packages/web/src/features/fileTree/utils.test.ts new file mode 100644 index 000000000..4785689c8 --- /dev/null +++ b/packages/web/src/features/fileTree/utils.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from 'vitest'; +import { buildFileTree, isPathValid, normalizePath } from './utils'; + +test('normalizePath adds a trailing slash and strips leading slashes', () => { + expect(normalizePath('/a/b')).toBe('a/b/'); +}); + +test('normalizePath keeps an existing trailing slash', () => { + expect(normalizePath('a/b/')).toBe('a/b/'); +}); + +test('normalizePath returns empty string for root', () => { + expect(normalizePath('/')).toBe(''); +}); + +test('isPathValid rejects traversal and null bytes', () => { + expect(isPathValid('a/../b')).toBe(false); + expect(isPathValid('a/\0b')).toBe(false); +}); + +test('isPathValid allows normal paths', () => { + expect(isPathValid('a/b')).toBe(true); +}); + +test('isPathValid allows paths with dots', () => { + expect(isPathValid('a/b/c.txt')).toBe(true); + expect(isPathValid('a/b/c...')).toBe(true); + expect(isPathValid('a/b/c.../d')).toBe(true); + expect(isPathValid('a/b/..c')).toBe(true); + expect(isPathValid('a/b/[..path]')).toBe(true); +}); + +test('buildFileTree handles a empty flat list', () => { + const flatList: { type: string, path: string }[] = []; + const tree = buildFileTree(flatList); + expect(tree).toMatchObject({ + name: 'root', + type: 'tree', + path: '', + }); +}); + +test('buildFileTree builds a sorted tree from a flat list', () => { + const flatList: { type: string, path: string }[] = [ + { type: 'blob', path: 'a' }, + { type: 'tree', path: 'b' }, + { type: 'tree', path: 'b/c' }, + { type: 'tree', path: 'd' }, + { type: 'blob', path: 'd/e' } + ]; + + const tree = buildFileTree(flatList); + + expect(tree).toMatchObject({ + name: 'root', + type: 'tree', + path: '', + children: [ + { + name: 'b', + type: 'tree', + path: 'b', + children: [ + { + name: 'c', + type: 'tree', + path: 'b/c', + children: [], + }, + ], + }, + { + name: 'd', + type: 'tree', + path: 'd', + children: [ + { + name: 'e', + type: 'blob', + path: 'd/e', + children: [], + }, + ], + }, + { + name: 'a', + type: 'blob', + path: 'a', + children: [], + } + ], + }); +}); + diff --git a/packages/web/src/features/fileTree/utils.ts b/packages/web/src/features/fileTree/utils.ts new file mode 100644 index 000000000..b654a6a29 --- /dev/null +++ b/packages/web/src/features/fileTree/utils.ts @@ -0,0 +1,89 @@ +import { FileTreeItem, FileTreeNode } from "./types"; + +export const normalizePath = (path: string): string => { + // Normalize the path by... + let normalizedPath = path; + + // ... adding a trailing slash if it doesn't have one. + // This is important since ls-tree won't return the contents + // of a directory if it doesn't have a trailing slash. + if (!normalizedPath.endsWith('/')) { + normalizedPath = `${normalizedPath}/`; + } + + // ... removing any leading slashes. This is needed since + // the path is relative to the repository's root, so we + // need a relative path. + if (normalizedPath.startsWith('/')) { + normalizedPath = normalizedPath.slice(1); + } + + return normalizedPath; +} + +// @note: we don't allow directory traversal +// or null bytes in the path. +export const isPathValid = (path: string) => { + const pathSegments = path.split('/'); + if (pathSegments.some(segment => segment === '..') || path.includes('\0')) { + return false; + } + + return true; +} + +export const buildFileTree = (flatList: { type: string, path: string }[]): FileTreeNode => { + const root: FileTreeNode = { + name: 'root', + path: '', + type: 'tree', + children: [], + }; + + for (const item of flatList) { + const parts = item.path.split('/'); + let current: FileTreeNode = root; + + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const isLeaf = i === parts.length - 1; + const nodeType = isLeaf ? item.type : 'tree'; + let next = current.children.find((child: FileTreeNode) => child.name === part && child.type === nodeType); + + if (!next) { + next = { + name: part, + path: item.path, + type: nodeType, + children: [], + }; + current.children.push(next); + } + current = next; + } + } + + const sortTree = (node: FileTreeNode): FileTreeNode => { + if (node.type === 'blob') { + return node; + } + + const sortedChildren = node.children + .map(sortTree) + .sort(compareFileTreeItems); + + return { + ...node, + children: sortedChildren, + }; + }; + + return sortTree(root); +} + +export const compareFileTreeItems = (a: FileTreeItem, b: FileTreeItem): number => { + if (a.type !== b.type) { + return a.type === 'tree' ? -1 : 1; + } + return a.name.localeCompare(b.name, undefined, { sensitivity: 'base' }); +} diff --git a/packages/web/src/lib/posthogEvents.ts b/packages/web/src/lib/posthogEvents.ts index 82d21eb41..36df22a71 100644 --- a/packages/web/src/lib/posthogEvents.ts +++ b/packages/web/src/lib/posthogEvents.ts @@ -181,6 +181,10 @@ export type PosthogEventMap = { isGlobalSearchEnabled: boolean, }, ////////////////////////////////////////////////////////////////// + wa_file_tree_loaded: { + durationMs: number, + }, + ////////////////////////////////////////////////////////////////// api_code_search_request: { source: string; type: 'streamed' | 'blocking';