From dae59f8e228b6c47dd43a118c2ed1c645eef74f2 Mon Sep 17 00:00:00 2001 From: "Kamat, Trivikram" <16024985+trivikr@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:13:07 -0700 Subject: [PATCH 1/5] perf: parallelize jsDelivr README fallback probes --- server/utils/readme-loaders.ts | 44 +++++++++---- test/unit/server/utils/readme-loaders.spec.ts | 63 +++++++++++++++++++ 2 files changed, 96 insertions(+), 11 deletions(-) diff --git a/server/utils/readme-loaders.ts b/server/utils/readme-loaders.ts index 116e4a1a1f..a9843b5111 100644 --- a/server/utils/readme-loaders.ts +++ b/server/utils/readme-loaders.ts @@ -19,14 +19,23 @@ const standardReadmeFilenames = [ /** Matches standard README filenames (case-insensitive, for checking registry metadata) */ const standardReadmePattern = /^readme(?:\.md|\.markdown)?$/i +const JSDELIVR_README_FETCH_BATCH_SIZE = 3 export function isStandardReadme(filename: string | undefined): boolean { return !!filename && standardReadmePattern.test(filename) } +function buildReadmeFetchCandidates(readmeFilename: string | undefined): string[] { + if (!readmeFilename) { + return standardReadmeFilenames + } + + return [readmeFilename, ...standardReadmeFilenames.filter(name => name !== readmeFilename)] +} + /** * Fetch README from jsdelivr CDN for a specific package version. - * Falls back through common README filenames. + * Falls back through candidate README filenames in small parallel batches. */ export async function fetchReadmeFromJsdelivr( packageName: string, @@ -35,15 +44,27 @@ export async function fetchReadmeFromJsdelivr( ): Promise { const versionSuffix = version ? `@${version}` : '' - for (const filename of readmeFilenames) { - try { - const url = `https://cdn.jsdelivr.net/npm/${packageName}${versionSuffix}/${filename}` - const response = await fetch(url) - if (response.ok) { - return await response.text() - } - } catch { - // Try next filename + for (let index = 0; index < readmeFilenames.length; index += JSDELIVR_README_FETCH_BATCH_SIZE) { + const batch = readmeFilenames.slice(index, index + JSDELIVR_README_FETCH_BATCH_SIZE) + const responses = await Promise.all( + batch.map(async filename => { + try { + const url = `https://cdn.jsdelivr.net/npm/${packageName}${versionSuffix}/${filename}` + const response = await fetch(url) + if (!response.ok) { + return null + } + + return await response.text() + } catch { + return null + } + }), + ) + + const matchedReadme = responses.find((response): response is string => response !== null) + if (matchedReadme) { + return matchedReadme } } @@ -84,10 +105,11 @@ export const resolvePackageReadmeSource = defineCachedFunction( !isStandardReadme(readmeFilename) || readmeContent!.length >= NPM_README_TRUNCATION_THRESHOLD ) { + const readmeCandidates = buildReadmeFetchCandidates(readmeFilename) const resolvedVersion = version ?? packageData['dist-tags']?.latest const jsdelivrReadme = await fetchReadmeFromJsdelivr( packageName, - standardReadmeFilenames, + readmeCandidates, resolvedVersion, ) if (jsdelivrReadme) { diff --git a/test/unit/server/utils/readme-loaders.spec.ts b/test/unit/server/utils/readme-loaders.spec.ts index 2440663088..585f5a46ec 100644 --- a/test/unit/server/utils/readme-loaders.spec.ts +++ b/test/unit/server/utils/readme-loaders.spec.ts @@ -38,6 +38,15 @@ describe('isStandardReadme', () => { }) describe('fetchReadmeFromJsdelivr', () => { + beforeEach(() => { + vi.unstubAllGlobals() + vi.stubGlobal('defineCachedFunction', (fn: Function) => fn) + vi.stubGlobal('$fetch', $fetchMock) + vi.stubGlobal('parsePackageParams', parsePackageParams) + vi.stubGlobal('fetchNpmPackage', fetchNpmPackageMock) + vi.stubGlobal('parseRepositoryInfo', parseRepositoryInfoMock) + }) + it('returns content when first filename succeeds', async () => { const content = '# Package' const fetchMock = vi.fn().mockResolvedValue({ @@ -73,6 +82,59 @@ describe('fetchReadmeFromJsdelivr', () => { expect(result).toBeNull() expect(fetchMock).toHaveBeenCalledTimes(2) }) + + it('starts a small batch of candidate fetches in parallel', async () => { + let resolveReadmeMd!: (value: { ok: false }) => void + let resolveLowercase!: (value: { ok: false }) => void + let resolveReadme!: (value: { ok: true; text: () => Promise }) => void + + const fetchMock = vi.fn((url: string) => { + if (url.endsWith('/README.md')) { + return new Promise(resolve => { + resolveReadmeMd = resolve + }) + } + + if (url.endsWith('/readme.md')) { + return new Promise(resolve => { + resolveLowercase = resolve + }) + } + + if (url.endsWith('/README')) { + return new Promise(resolve => { + resolveReadme = resolve + }) + } + + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + const resultPromise = fetchReadmeFromJsdelivr('pkg', [ + 'README.md', + 'readme.md', + 'README', + 'readme', + ]) + + expect(fetchMock).toHaveBeenCalledTimes(3) + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'https://cdn.jsdelivr.net/npm/pkg/README.md', + 'https://cdn.jsdelivr.net/npm/pkg/readme.md', + 'https://cdn.jsdelivr.net/npm/pkg/README', + ]) + + resolveReadmeMd({ ok: false }) + resolveLowercase({ ok: false }) + resolveReadme({ + ok: true, + text: async () => '# Package', + }) + + await expect(resultPromise).resolves.toBe('# Package') + expect(fetchMock).toHaveBeenCalledTimes(3) + }) }) describe('resolvePackageReadmeSource', () => { @@ -172,6 +234,7 @@ describe('resolvePackageReadmeSource', () => { const result = await resolvePackageReadmeSource('pkg') expect(result).toMatchObject({ markdown: jsdelivrContent }) + expect(fetchMock).toHaveBeenNthCalledWith(1, 'https://cdn.jsdelivr.net/npm/pkg/DOCS.md') }) it('returns undefined markdown when no content and jsdelivr fails', async () => { From fd7d050a1a8633bfc138f497aec0765f963603d3 Mon Sep 17 00:00:00 2001 From: "Kamat, Trivikram" <16024985+trivikr@users.noreply.github.com> Date: Sat, 4 Apr 2026 19:40:22 -0700 Subject: [PATCH 2/5] fix: only read the body of the response which worked --- server/utils/readme-loaders.ts | 6 ++--- test/unit/server/utils/readme-loaders.spec.ts | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/server/utils/readme-loaders.ts b/server/utils/readme-loaders.ts index a9843b5111..e363c858d7 100644 --- a/server/utils/readme-loaders.ts +++ b/server/utils/readme-loaders.ts @@ -55,16 +55,16 @@ export async function fetchReadmeFromJsdelivr( return null } - return await response.text() + return response } catch { return null } }), ) - const matchedReadme = responses.find((response): response is string => response !== null) + const matchedReadme = responses.find((response): response is Response => response !== null) if (matchedReadme) { - return matchedReadme + return await matchedReadme.text() } } diff --git a/test/unit/server/utils/readme-loaders.spec.ts b/test/unit/server/utils/readme-loaders.spec.ts index 585f5a46ec..3bdf984b13 100644 --- a/test/unit/server/utils/readme-loaders.spec.ts +++ b/test/unit/server/utils/readme-loaders.spec.ts @@ -135,6 +135,28 @@ describe('fetchReadmeFromJsdelivr', () => { await expect(resultPromise).resolves.toBe('# Package') expect(fetchMock).toHaveBeenCalledTimes(3) }) + + it('reads only the matched successful response body', async () => { + const firstTextMock = vi.fn().mockResolvedValue('# First') + const secondTextMock = vi.fn().mockResolvedValue('# Second') + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + text: firstTextMock, + }) + .mockResolvedValueOnce({ + ok: true, + text: secondTextMock, + }) + vi.stubGlobal('fetch', fetchMock) + + const result = await fetchReadmeFromJsdelivr('pkg', ['README.md', 'readme.md']) + + expect(result).toBe('# First') + expect(firstTextMock).toHaveBeenCalledTimes(1) + expect(secondTextMock).not.toHaveBeenCalled() + }) }) describe('resolvePackageReadmeSource', () => { From 72bb0b216fcb61b2f8950a33f6e6c96ebd4901e6 Mon Sep 17 00:00:00 2001 From: "Kamat, Trivikram" <16024985+trivikr@users.noreply.github.com> Date: Sat, 4 Apr 2026 19:46:18 -0700 Subject: [PATCH 3/5] chore: try provided readmeFilename before starting the fallback batch --- server/utils/readme-loaders.ts | 27 ++++--- test/unit/server/utils/readme-loaders.spec.ts | 73 +++++++++++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/server/utils/readme-loaders.ts b/server/utils/readme-loaders.ts index e363c858d7..1d84c06090 100644 --- a/server/utils/readme-loaders.ts +++ b/server/utils/readme-loaders.ts @@ -26,11 +26,9 @@ export function isStandardReadme(filename: string | undefined): boolean { } function buildReadmeFetchCandidates(readmeFilename: string | undefined): string[] { - if (!readmeFilename) { - return standardReadmeFilenames - } - - return [readmeFilename, ...standardReadmeFilenames.filter(name => name !== readmeFilename)] + return readmeFilename + ? standardReadmeFilenames.filter(name => name !== readmeFilename) + : standardReadmeFilenames } /** @@ -105,13 +103,20 @@ export const resolvePackageReadmeSource = defineCachedFunction( !isStandardReadme(readmeFilename) || readmeContent!.length >= NPM_README_TRUNCATION_THRESHOLD ) { - const readmeCandidates = buildReadmeFetchCandidates(readmeFilename) const resolvedVersion = version ?? packageData['dist-tags']?.latest - const jsdelivrReadme = await fetchReadmeFromJsdelivr( - packageName, - readmeCandidates, - resolvedVersion, - ) + let jsdelivrReadme = + readmeFilename && + (await fetchReadmeFromJsdelivr(packageName, [readmeFilename], resolvedVersion)) + + if (!jsdelivrReadme) { + const readmeCandidates = buildReadmeFetchCandidates(readmeFilename) + jsdelivrReadme = await fetchReadmeFromJsdelivr( + packageName, + readmeCandidates, + resolvedVersion, + ) + } + if (jsdelivrReadme) { readmeContent = jsdelivrReadme } diff --git a/test/unit/server/utils/readme-loaders.spec.ts b/test/unit/server/utils/readme-loaders.spec.ts index 3bdf984b13..149e399598 100644 --- a/test/unit/server/utils/readme-loaders.spec.ts +++ b/test/unit/server/utils/readme-loaders.spec.ts @@ -259,6 +259,79 @@ describe('resolvePackageReadmeSource', () => { expect(fetchMock).toHaveBeenNthCalledWith(1, 'https://cdn.jsdelivr.net/npm/pkg/DOCS.md') }) + it('tries a provided readmeFilename before starting the fallback batch', async () => { + let resolveDocs!: (value: { ok: false }) => void + let resolveReadmeMd!: (value: { ok: false }) => void + let resolveLowercase!: (value: { ok: false }) => void + let resolveReadmeCase!: (value: { ok: true; text: () => Promise }) => void + + fetchNpmPackageMock.mockResolvedValue({ + readme: undefined, + readmeFilename: 'DOCS.md', + repository: undefined, + versions: {}, + }) + parseRepositoryInfoMock.mockReturnValue(undefined) + + const fetchMock = vi.fn((url: string) => { + if (url.endsWith('/DOCS.md')) { + return new Promise(resolve => { + resolveDocs = resolve + }) + } + + if (url.endsWith('/README.md')) { + return new Promise(resolve => { + resolveReadmeMd = resolve + }) + } + + if (url.endsWith('/readme.md')) { + return new Promise(resolve => { + resolveLowercase = resolve + }) + } + + if (url.endsWith('/Readme.md')) { + return new Promise(resolve => { + resolveReadmeCase = resolve + }) + } + + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + const resultPromise = resolvePackageReadmeSource('pkg') + + await new Promise(resolve => setTimeout(resolve, 0)) + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenNthCalledWith(1, 'https://cdn.jsdelivr.net/npm/pkg/DOCS.md') + + resolveDocs({ ok: false }) + + await new Promise(resolve => setTimeout(resolve, 0)) + expect(fetchMock).toHaveBeenCalledTimes(4) + expect(fetchMock.mock.calls.slice(1).map(([url]) => url)).toEqual([ + 'https://cdn.jsdelivr.net/npm/pkg/README.md', + 'https://cdn.jsdelivr.net/npm/pkg/readme.md', + 'https://cdn.jsdelivr.net/npm/pkg/Readme.md', + ]) + + resolveReadmeMd({ ok: false }) + resolveLowercase({ ok: false }) + resolveReadmeCase({ + ok: true, + text: async () => '# From fallback batch', + }) + + await expect(resultPromise).resolves.toMatchObject({ + packageName: 'pkg', + markdown: '# From fallback batch', + repoInfo: undefined, + }) + }) + it('returns undefined markdown when no content and jsdelivr fails', async () => { fetchNpmPackageMock.mockResolvedValue({ readme: undefined, From 9048f4253e4b456860e4cdf97dc5bff787797ee1 Mon Sep 17 00:00:00 2001 From: "Willow (GHOST)" Date: Sun, 5 Apr 2026 02:55:54 -0400 Subject: [PATCH 4/5] chore: add some comments --- server/utils/readme-loaders.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/utils/readme-loaders.ts b/server/utils/readme-loaders.ts index 1d84c06090..61bbecca71 100644 --- a/server/utils/readme-loaders.ts +++ b/server/utils/readme-loaders.ts @@ -104,10 +104,13 @@ export const resolvePackageReadmeSource = defineCachedFunction( readmeContent!.length >= NPM_README_TRUNCATION_THRESHOLD ) { const resolvedVersion = version ?? packageData['dist-tags']?.latest + + // try fetching the given readme file first let jsdelivrReadme = readmeFilename && (await fetchReadmeFromJsdelivr(packageName, [readmeFilename], resolvedVersion)) + // if it's unsucessful, fetch all known readme filenames if (!jsdelivrReadme) { const readmeCandidates = buildReadmeFetchCandidates(readmeFilename) jsdelivrReadme = await fetchReadmeFromJsdelivr( @@ -117,6 +120,7 @@ export const resolvePackageReadmeSource = defineCachedFunction( ) } + // if we found sometihng, use it if (jsdelivrReadme) { readmeContent = jsdelivrReadme } From 36fced5e71068073ccba3450e2dc0891fafc34b4 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Sun, 5 Apr 2026 08:13:00 -0700 Subject: [PATCH 5/5] chore: check for defined response which is not whitespace-only --- server/utils/readme-loaders.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/utils/readme-loaders.ts b/server/utils/readme-loaders.ts index 61bbecca71..eee3bf8b45 100644 --- a/server/utils/readme-loaders.ts +++ b/server/utils/readme-loaders.ts @@ -60,9 +60,11 @@ export async function fetchReadmeFromJsdelivr( }), ) - const matchedReadme = responses.find((response): response is Response => response !== null) - if (matchedReadme) { - return await matchedReadme.text() + for (const response of responses) { + const text = await response?.text() + if (text?.trim()) { + return text + } } }