diff --git a/server/utils/readme-loaders.ts b/server/utils/readme-loaders.ts index 116e4a1a1..8d1e2e824 100644 --- a/server/utils/readme-loaders.ts +++ b/server/utils/readme-loaders.ts @@ -19,14 +19,28 @@ const standardReadmeFilenames = [ /** Matches standard README filenames (case-insensitive, for checking registry metadata) */ const standardReadmePattern = /^readme(?:\.md|\.markdown)?$/i +const JSDELIVR_README_FETCH_BATCH_SIZE = 3 export function isStandardReadme(filename: string | undefined): boolean { return !!filename && standardReadmePattern.test(filename) } +async function cancelUnreadBatchResponses( + responses: Array, + startIndex: number, +): Promise { + await Promise.allSettled(responses.slice(startIndex).map(response => response?.body?.cancel())) +} + +function buildReadmeFetchCandidates(readmeFilename: string | undefined): string[] { + return readmeFilename + ? standardReadmeFilenames.filter(name => name !== readmeFilename) + : standardReadmeFilenames +} + /** * Fetch README from jsdelivr CDN for a specific package version. - * Falls back through common README filenames. + * Falls back through candidate README filenames in small parallel batches. */ export async function fetchReadmeFromJsdelivr( packageName: string, @@ -35,15 +49,30 @@ export async function fetchReadmeFromJsdelivr( ): Promise { const versionSuffix = version ? `@${version}` : '' - for (const filename of readmeFilenames) { - try { - const url = `https://cdn.jsdelivr.net/npm/${packageName}${versionSuffix}/${filename}` - const response = await fetch(url) - if (response.ok) { - return await response.text() + for (let index = 0; index < readmeFilenames.length; index += JSDELIVR_README_FETCH_BATCH_SIZE) { + const batch = readmeFilenames.slice(index, index + JSDELIVR_README_FETCH_BATCH_SIZE) + const responses = await Promise.all( + batch.map(async filename => { + try { + const url = `https://cdn.jsdelivr.net/npm/${packageName}${versionSuffix}/${filename}` + const response = await fetch(url) + if (!response.ok) { + return null + } + + return response + } catch { + return null + } + }), + ) + + for (const [responseIndex, response] of responses.entries()) { + const text = await response?.text() + if (text?.trim()) { + await cancelUnreadBatchResponses(responses, responseIndex + 1) + return text } - } catch { - // Try next filename } } @@ -85,11 +114,23 @@ export const resolvePackageReadmeSource = defineCachedFunction( readmeContent!.length >= NPM_README_TRUNCATION_THRESHOLD ) { const resolvedVersion = version ?? packageData['dist-tags']?.latest - const jsdelivrReadme = await fetchReadmeFromJsdelivr( - packageName, - standardReadmeFilenames, - resolvedVersion, - ) + + // try fetching the given readme file first + let jsdelivrReadme = + readmeFilename && + (await fetchReadmeFromJsdelivr(packageName, [readmeFilename], resolvedVersion)) + + // if it's unsucessful, fetch all known readme filenames + if (!jsdelivrReadme) { + const readmeCandidates = buildReadmeFetchCandidates(readmeFilename) + jsdelivrReadme = await fetchReadmeFromJsdelivr( + packageName, + readmeCandidates, + resolvedVersion, + ) + } + + // if we found sometihng, use it if (jsdelivrReadme) { readmeContent = jsdelivrReadme } diff --git a/test/unit/server/utils/readme-loaders.spec.ts b/test/unit/server/utils/readme-loaders.spec.ts index 244066308..f3b9d0ca4 100644 --- a/test/unit/server/utils/readme-loaders.spec.ts +++ b/test/unit/server/utils/readme-loaders.spec.ts @@ -38,6 +38,15 @@ describe('isStandardReadme', () => { }) describe('fetchReadmeFromJsdelivr', () => { + beforeEach(() => { + vi.unstubAllGlobals() + vi.stubGlobal('defineCachedFunction', (fn: Function) => fn) + vi.stubGlobal('$fetch', $fetchMock) + vi.stubGlobal('parsePackageParams', parsePackageParams) + vi.stubGlobal('fetchNpmPackage', fetchNpmPackageMock) + vi.stubGlobal('parseRepositoryInfo', parseRepositoryInfoMock) + }) + it('returns content when first filename succeeds', async () => { const content = '# Package' const fetchMock = vi.fn().mockResolvedValue({ @@ -73,6 +82,86 @@ describe('fetchReadmeFromJsdelivr', () => { expect(result).toBeNull() expect(fetchMock).toHaveBeenCalledTimes(2) }) + + it('starts a small batch of candidate fetches in parallel', async () => { + let resolveReadmeMd!: (value: { ok: false }) => void + let resolveLowercase!: (value: { ok: false }) => void + let resolveReadme!: (value: { ok: true; text: () => Promise }) => void + + const fetchMock = vi.fn((url: string) => { + if (url.endsWith('/README.md')) { + return new Promise(resolve => { + resolveReadmeMd = resolve + }) + } + + if (url.endsWith('/readme.md')) { + return new Promise(resolve => { + resolveLowercase = resolve + }) + } + + if (url.endsWith('/README')) { + return new Promise(resolve => { + resolveReadme = resolve + }) + } + + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + const resultPromise = fetchReadmeFromJsdelivr('pkg', [ + 'README.md', + 'readme.md', + 'README', + 'readme', + ]) + + expect(fetchMock).toHaveBeenCalledTimes(3) + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'https://cdn.jsdelivr.net/npm/pkg/README.md', + 'https://cdn.jsdelivr.net/npm/pkg/readme.md', + 'https://cdn.jsdelivr.net/npm/pkg/README', + ]) + + resolveReadmeMd({ ok: false }) + resolveLowercase({ ok: false }) + resolveReadme({ + ok: true, + text: async () => '# Package', + }) + + await expect(resultPromise).resolves.toBe('# Package') + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it('reads only the matched successful response body', async () => { + const firstTextMock = vi.fn().mockResolvedValue('# First') + const secondTextMock = vi.fn().mockResolvedValue('# Second') + const secondCancelMock = vi.fn().mockResolvedValue(undefined) + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + text: firstTextMock, + }) + .mockResolvedValueOnce({ + ok: true, + text: secondTextMock, + body: { + cancel: secondCancelMock, + }, + }) + vi.stubGlobal('fetch', fetchMock) + + const result = await fetchReadmeFromJsdelivr('pkg', ['README.md', 'readme.md']) + + expect(result).toBe('# First') + expect(firstTextMock).toHaveBeenCalledTimes(1) + expect(secondTextMock).not.toHaveBeenCalled() + expect(secondCancelMock).toHaveBeenCalledTimes(1) + }) }) describe('resolvePackageReadmeSource', () => { @@ -172,6 +261,80 @@ describe('resolvePackageReadmeSource', () => { const result = await resolvePackageReadmeSource('pkg') expect(result).toMatchObject({ markdown: jsdelivrContent }) + expect(fetchMock).toHaveBeenNthCalledWith(1, 'https://cdn.jsdelivr.net/npm/pkg/DOCS.md') + }) + + it('tries a provided readmeFilename before starting the fallback batch', async () => { + let resolveDocs!: (value: { ok: false }) => void + let resolveReadmeMd!: (value: { ok: false }) => void + let resolveLowercase!: (value: { ok: false }) => void + let resolveReadmeCase!: (value: { ok: true; text: () => Promise }) => void + + fetchNpmPackageMock.mockResolvedValue({ + readme: undefined, + readmeFilename: 'DOCS.md', + repository: undefined, + versions: {}, + }) + parseRepositoryInfoMock.mockReturnValue(undefined) + + const fetchMock = vi.fn((url: string) => { + if (url.endsWith('/DOCS.md')) { + return new Promise(resolve => { + resolveDocs = resolve + }) + } + + if (url.endsWith('/README.md')) { + return new Promise(resolve => { + resolveReadmeMd = resolve + }) + } + + if (url.endsWith('/readme.md')) { + return new Promise(resolve => { + resolveLowercase = resolve + }) + } + + if (url.endsWith('/Readme.md')) { + return new Promise(resolve => { + resolveReadmeCase = resolve + }) + } + + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + const resultPromise = resolvePackageReadmeSource('pkg') + + await new Promise(resolve => setTimeout(resolve, 0)) + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenNthCalledWith(1, 'https://cdn.jsdelivr.net/npm/pkg/DOCS.md') + + resolveDocs({ ok: false }) + + await new Promise(resolve => setTimeout(resolve, 0)) + expect(fetchMock).toHaveBeenCalledTimes(4) + expect(fetchMock.mock.calls.slice(1).map(([url]) => url)).toEqual([ + 'https://cdn.jsdelivr.net/npm/pkg/README.md', + 'https://cdn.jsdelivr.net/npm/pkg/readme.md', + 'https://cdn.jsdelivr.net/npm/pkg/Readme.md', + ]) + + resolveReadmeMd({ ok: false }) + resolveLowercase({ ok: false }) + resolveReadmeCase({ + ok: true, + text: async () => '# From fallback batch', + }) + + await expect(resultPromise).resolves.toMatchObject({ + packageName: 'pkg', + markdown: '# From fallback batch', + repoInfo: undefined, + }) }) it('returns undefined markdown when no content and jsdelivr fails', async () => {