Skip to content

Commit 4514782

Browse files
rafaelcrrafaelcr
committed
fix: fetch headers
1 parent 0be9114 commit 4514782

File tree

5 files changed

+98
-48
lines changed

5 files changed

+98
-48
lines changed

‎src/env.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,13 +131,19 @@ const schema = Type.Object({
131131
* URLs use the `ipfs:` or `ipns:` protocol schemes. Defaults to `https://cloudflare-ipfs.com`.
132132
*/
133133
PUBLIC_GATEWAY_IPFS: Type.String({ default: 'https://cloudflare-ipfs.com' }),
134+
/**
135+
* Extra header key to add to the request when fetching metadata from the configured IPFS gateway,
136+
* for example if authentication is required. Must be in the form 'Header: Value'.
137+
*/
138+
PUBLIC_GATEWAY_IPFS_EXTRA_HEADER: Type.Optional(Type.String()),
134139
/**
135140
* List of public IPFS gateways that will be replaced with the value of `PUBLIC_GATEWAY_IPFS`
136141
* whenever a metadata URL has these gateways hard coded in `http:` or `https:` URLs.
137142
*/
138143
PUBLIC_GATEWAY_IPFS_REPLACED: Type.String({
139144
default: 'ipfs.io,dweb.link,gateway.pinata.cloud,cloudflare-ipfs.com,infura-ipfs.io',
140145
}),
146+
141147
/**
142148
* Base URL for a public gateway which will provide access to all Arweave resources when metadata
143149
* URLs use the `ar:` protocol scheme. Defaults to

‎src/token-processor/images/image-cache.ts‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { ENV } from '../../env';
2-
import { parseDataUrl, getFetchableDecentralizedStorageUrl } from '../util/metadata-helpers';
2+
import { parseDataUrl, getFetchableMetadataUrl } from '../util/metadata-helpers';
33
import { logger } from '@hirosystems/api-toolkit';
44
import { PgStore } from '../../pg/pg-store';
55
import { Readable } from 'node:stream';
@@ -16,7 +16,6 @@ import {
1616
} from '../util/errors';
1717
import { pipeline } from 'node:stream/promises';
1818
import { Storage } from '@google-cloud/storage';
19-
import { RetryableJobError } from '../queue/errors';
2019

2120
/** Saves an image provided via a `data:` uri string to disk for processing. */
2221
function convertDataImage(uri: string, tmpPath: string): string {
@@ -33,10 +32,15 @@ function convertDataImage(uri: string, tmpPath: string): string {
3332
return filePath;
3433
}
3534

36-
async function downloadImage(imgUrl: string, tmpPath: string): Promise<string> {
35+
async function downloadImage(
36+
imgUrl: string,
37+
tmpPath: string,
38+
headers?: Record<string, string>
39+
): Promise<string> {
3740
return new Promise((resolve, reject) => {
3841
const filePath = `${tmpPath}/image`;
3942
fetch(imgUrl, {
43+
headers,
4044
dispatcher: new Agent({
4145
headersTimeout: ENV.METADATA_FETCH_TIMEOUT_MS,
4246
bodyTimeout: ENV.METADATA_FETCH_TIMEOUT_MS,
@@ -109,22 +113,23 @@ async function transformImage(filePath: string, resize: boolean = false): Promis
109113
* For a list of configuration options, see `env.ts`.
110114
*/
111115
export async function processImageCache(
112-
imgUrl: string,
116+
rawImgUrl: string,
113117
contractPrincipal: string,
114118
tokenNumber: bigint
115119
): Promise<string[]> {
116-
logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${imgUrl}`);
120+
logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${rawImgUrl}`);
117121
try {
118122
const gcs = new Storage();
119123
const gcsBucket = ENV.IMAGE_CACHE_GCS_BUCKET_NAME as string;
120124

121125
const tmpPath = `tmp/${contractPrincipal}_${tokenNumber}`;
122126
fs.mkdirSync(tmpPath, { recursive: true });
123127
let original: string;
124-
if (imgUrl.startsWith('data:')) {
125-
original = convertDataImage(imgUrl, tmpPath);
128+
if (rawImgUrl.startsWith('data:')) {
129+
original = convertDataImage(rawImgUrl, tmpPath);
126130
} else {
127-
original = await downloadImage(imgUrl, tmpPath);
131+
const { url: httpUrl, fetchHeaders } = getFetchableMetadataUrl(rawImgUrl);
132+
original = await downloadImage(httpUrl.toString(), tmpPath, fetchHeaders);
128133
}
129134

130135
const image1 = await transformImage(original);
@@ -152,10 +157,10 @@ export async function processImageCache(
152157
typeError.cause instanceof errors.BodyTimeoutError ||
153158
typeError.cause instanceof errors.ConnectTimeoutError
154159
) {
155-
throw new ImageTimeoutError(new URL(imgUrl));
160+
throw new ImageTimeoutError(new URL(rawImgUrl));
156161
}
157162
if (typeError.cause instanceof errors.ResponseExceededMaxSizeError) {
158-
throw new ImageSizeExceededError(`ImageCache image too large: ${imgUrl}`);
163+
throw new ImageSizeExceededError(`ImageCache image too large: ${rawImgUrl}`);
159164
}
160165
if ((typeError.cause as any).toString().includes('ECONNRESET')) {
161166
throw new ImageHttpError(`ImageCache server connection interrupted`, typeError);
@@ -165,17 +170,6 @@ export async function processImageCache(
165170
}
166171
}
167172

168-
/**
169-
* Converts a raw image URI from metadata into a fetchable URL.
170-
* @param uri - Original image URI
171-
* @returns Normalized URL string
172-
*/
173-
export function normalizeImageUri(uri: string): string {
174-
if (uri.startsWith('data:')) return uri;
175-
const fetchableUrl = getFetchableDecentralizedStorageUrl(uri);
176-
return fetchableUrl.toString();
177-
}
178-
179173
export async function reprocessTokenImageCache(
180174
db: PgStore,
181175
contractPrincipal: string,
@@ -186,7 +180,7 @@ export async function reprocessTokenImageCache(
186180
for (const token of imageUris) {
187181
try {
188182
const [cached, thumbnail] = await processImageCache(
189-
getFetchableDecentralizedStorageUrl(token.image).toString(),
183+
getFetchableMetadataUrl(token.image).toString(),
190184
contractPrincipal,
191185
BigInt(token.token_number)
192186
);

‎src/token-processor/queue/job/process-token-job.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import { StacksNodeRpcClient } from '../../stacks-node/stacks-node-rpc-client';
1212
import { SmartContractClarityError, TooManyRequestsHttpError } from '../../util/errors';
1313
import {
1414
fetchAllMetadataLocalesFromBaseUri,
15-
getFetchableDecentralizedStorageUrl,
15+
getFetchableMetadataUrl,
1616
getTokenSpecificUri,
1717
} from '../../util/metadata-helpers';
1818
import { RetryableJobError } from '../errors';
@@ -194,7 +194,7 @@ export class ProcessTokenJob extends Job {
194194
return;
195195
}
196196
// Before we return the uri, check if its fetchable hostname is not already rate limited.
197-
const fetchable = getFetchableDecentralizedStorageUrl(uri);
197+
const fetchable = getFetchableMetadataUrl(uri).url;
198198
const rateLimitedHost = await this.db.getRateLimitedHost({ hostname: fetchable.hostname });
199199
if (rateLimitedHost) {
200200
const retryAfter = Date.parse(rateLimitedHost.retry_after);

‎src/token-processor/util/metadata-helpers.ts‎

Lines changed: 46 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import {
1919
UndiciCauseTypeError,
2020
} from './errors';
2121
import { RetryableJobError } from '../queue/errors';
22-
import { normalizeImageUri, processImageCache } from '../images/image-cache';
22+
import { processImageCache } from '../images/image-cache';
2323
import {
2424
RawMetadataLocale,
2525
RawMetadataLocalizationCType,
@@ -42,6 +42,12 @@ const METADATA_FETCH_HTTP_AGENT = new Agent({
4242

4343
const PUBLIC_GATEWAY_IPFS_REPLACED = ENV.PUBLIC_GATEWAY_IPFS_REPLACED.split(',');
4444

45+
export type FetchableMetadataUrl = {
46+
url: URL;
47+
gateway: 'ipfs' | 'arweave' | null;
48+
fetchHeaders?: Record<string, string>;
49+
};
50+
4551
/**
4652
* Fetches all the localized metadata JSONs for a token. First, it downloads the default metadata
4753
* JSON and parses it looking for other localizations. If those are found, each of them is then
@@ -174,9 +180,8 @@ async function parseMetadataForInsertion(
174180
let cachedImage: string | undefined;
175181
let cachedThumbnailImage: string | undefined;
176182
if (image && typeof image === 'string' && ENV.IMAGE_CACHE_PROCESSOR_ENABLED) {
177-
const normalizedUrl = normalizeImageUri(image);
178183
[cachedImage, cachedThumbnailImage] = await processImageCache(
179-
normalizedUrl,
184+
image,
180185
contract.principal,
181186
token.token_number
182187
);
@@ -245,14 +250,16 @@ async function parseMetadataForInsertion(
245250
export async function fetchMetadata(
246251
httpUrl: URL,
247252
contract_principal: string,
248-
token_number: bigint
253+
token_number: bigint,
254+
headers?: Record<string, string>
249255
): Promise<string | undefined> {
250256
const url = httpUrl.toString();
251257
try {
252258
logger.info(`MetadataFetch for ${contract_principal}#${token_number} from ${url}`);
253259
const result = await request(url, {
254260
method: 'GET',
255261
throwOnError: true,
262+
headers,
256263
dispatcher:
257264
// Disable during tests so we can inject a global mock agent.
258265
process.env.NODE_ENV === 'test' ? undefined : METADATA_FETCH_HTTP_AGENT,
@@ -306,10 +313,13 @@ export async function getMetadataFromUri(
306313
return parseJsonMetadata(token_uri, content);
307314
}
308315

309-
// Support HTTP/S URLs otherwise
310-
const httpUrl = getFetchableDecentralizedStorageUrl(token_uri);
316+
// Support HTTP/S URLs otherwise.
317+
// Transform the URL to use a public gateway if necessary.
318+
const { url: httpUrl, fetchHeaders } = getFetchableMetadataUrl(token_uri);
311319
const urlStr = httpUrl.toString();
312-
const content = await fetchMetadata(httpUrl, contract_principal, token_number);
320+
321+
// Fetch the metadata.
322+
const content = await fetchMetadata(httpUrl, contract_principal, token_number, fetchHeaders);
313323
return parseJsonMetadata(urlStr, content);
314324
}
315325

@@ -342,30 +352,47 @@ function parseJsonMetadata(url: string, content?: string): RawMetadata {
342352
* @param uri - URL to convert
343353
* @returns Fetchable URL
344354
*/
345-
export function getFetchableDecentralizedStorageUrl(uri: string): URL {
355+
export function getFetchableMetadataUrl(uri: string): FetchableMetadataUrl {
346356
try {
347357
const parsedUri = new URL(uri);
358+
const result: FetchableMetadataUrl = {
359+
url: parsedUri,
360+
gateway: null,
361+
fetchHeaders: {},
362+
};
348363
if (parsedUri.protocol === 'http:' || parsedUri.protocol === 'https:') {
349364
// If this is a known public IPFS gateway, replace it with `ENV.PUBLIC_GATEWAY_IPFS`.
350365
if (PUBLIC_GATEWAY_IPFS_REPLACED.includes(parsedUri.hostname)) {
351-
return new URL(`${ENV.PUBLIC_GATEWAY_IPFS}${parsedUri.pathname}`);
366+
result.url = new URL(`${ENV.PUBLIC_GATEWAY_IPFS}${parsedUri.pathname}`);
367+
result.gateway = 'ipfs';
368+
} else {
369+
result.url = parsedUri;
352370
}
353-
return parsedUri;
354-
}
355-
if (parsedUri.protocol === 'ipfs:') {
371+
} else if (parsedUri.protocol === 'ipfs:') {
356372
const host = parsedUri.host === 'ipfs' ? 'ipfs' : `ipfs/${parsedUri.host}`;
357-
return new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${host}${parsedUri.pathname}`);
358-
}
359-
if (parsedUri.protocol === 'ipns:') {
360-
return new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${parsedUri.host}${parsedUri.pathname}`);
373+
result.url = new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${host}${parsedUri.pathname}`);
374+
result.gateway = 'ipfs';
375+
} else if (parsedUri.protocol === 'ipns:') {
376+
result.url = new URL(`${ENV.PUBLIC_GATEWAY_IPFS}/${parsedUri.host}${parsedUri.pathname}`);
377+
result.gateway = 'ipfs';
378+
} else if (parsedUri.protocol === 'ar:') {
379+
result.url = new URL(`${ENV.PUBLIC_GATEWAY_ARWEAVE}/${parsedUri.host}${parsedUri.pathname}`);
380+
result.gateway = 'arweave';
381+
} else {
382+
throw new MetadataParseError(`Unsupported uri protocol: ${uri}`);
361383
}
362-
if (parsedUri.protocol === 'ar:') {
363-
return new URL(`${ENV.PUBLIC_GATEWAY_ARWEAVE}/${parsedUri.host}${parsedUri.pathname}`);
384+
385+
if (result.gateway === 'ipfs' && ENV.PUBLIC_GATEWAY_IPFS_EXTRA_HEADER) {
386+
const [key, value] = ENV.PUBLIC_GATEWAY_IPFS_EXTRA_HEADER.split(':');
387+
result.fetchHeaders = {
388+
[key.trim()]: value.trim(),
389+
};
364390
}
391+
392+
return result;
365393
} catch (error) {
366394
throw new MetadataParseError(`Invalid uri: ${uri}`);
367395
}
368-
throw new MetadataParseError(`Unsupported uri protocol: ${uri}`);
369396
}
370397

371398
export function parseDataUrl(

‎tests/token-queue/metadata-helpers.test.ts‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { MockAgent, setGlobalDispatcher } from 'undici';
22
import { ENV } from '../../src/env';
33
import { MetadataHttpError, MetadataParseError } from '../../src/token-processor/util/errors';
44
import {
5-
getFetchableDecentralizedStorageUrl,
5+
getFetchableMetadataUrl,
66
getMetadataFromUri,
77
getTokenSpecificUri,
88
fetchMetadata,
@@ -209,23 +209,46 @@ describe('Metadata Helpers', () => {
209209
test('get fetchable URLs', () => {
210210
ENV.PUBLIC_GATEWAY_IPFS = 'https://cloudflare-ipfs.com';
211211
ENV.PUBLIC_GATEWAY_ARWEAVE = 'https://arweave.net';
212+
ENV.PUBLIC_GATEWAY_IPFS_EXTRA_HEADER = 'Authorization: Bearer 1234567890';
213+
212214
const arweave = 'ar://II4z2ziYyqG7-kWDa98lWGfjxRdYOx9Zdld9P_I_kzE/9731.json';
213-
expect(getFetchableDecentralizedStorageUrl(arweave).toString()).toBe(
215+
const fetch1 = getFetchableMetadataUrl(arweave);
216+
expect(fetch1.url.toString()).toBe(
214217
'https://arweave.net/II4z2ziYyqG7-kWDa98lWGfjxRdYOx9Zdld9P_I_kzE/9731.json'
215218
);
219+
expect(fetch1.gateway).toBe('arweave');
220+
expect(fetch1.fetchHeaders).toBeUndefined();
221+
216222
const ipfs =
217223
'ipfs://ipfs/bafybeifwoqwdhs5djtx6vopvuwfcdrqeuecayp5wzpzjylxycejnhtrhgu/vague_art_paintings/vague_art_paintings_6_metadata.json';
218-
expect(getFetchableDecentralizedStorageUrl(ipfs).toString()).toBe(
224+
const fetch2 = getFetchableMetadataUrl(ipfs);
225+
expect(fetch2.url.toString()).toBe(
219226
'https://cloudflare-ipfs.com/ipfs/bafybeifwoqwdhs5djtx6vopvuwfcdrqeuecayp5wzpzjylxycejnhtrhgu/vague_art_paintings/vague_art_paintings_6_metadata.json'
220227
);
228+
expect(fetch2.gateway).toBe('ipfs');
229+
expect(fetch2.fetchHeaders).toEqual({ Authorization: 'Bearer 1234567890' });
230+
221231
const ipfs2 = 'ipfs://QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png';
222-
expect(getFetchableDecentralizedStorageUrl(ipfs2).toString()).toBe(
232+
const fetch3 = getFetchableMetadataUrl(ipfs2);
233+
expect(fetch3.url.toString()).toBe(
223234
'https://cloudflare-ipfs.com/ipfs/QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png'
224235
);
236+
expect(fetch3.gateway).toBe('ipfs');
237+
expect(fetch3.fetchHeaders).toEqual({ Authorization: 'Bearer 1234567890' });
238+
225239
const ipfs3 = 'https://ipfs.io/ipfs/QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png';
226-
expect(getFetchableDecentralizedStorageUrl(ipfs3).toString()).toBe(
240+
const fetch4 = getFetchableMetadataUrl(ipfs3);
241+
expect(fetch4.url.toString()).toBe(
227242
'https://cloudflare-ipfs.com/ipfs/QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png'
228243
);
244+
expect(fetch4.gateway).toBe('ipfs');
245+
expect(fetch4.fetchHeaders).toEqual({ Authorization: 'Bearer 1234567890' });
246+
247+
const http = 'https://test.io/1.json';
248+
const fetch5 = getFetchableMetadataUrl(http);
249+
expect(fetch5.url.toString()).toBe(http);
250+
expect(fetch5.gateway).toBeNull();
251+
expect(fetch5.fetchHeaders).toBeUndefined();
229252
});
230253

231254
test('replace URI string tokens', () => {

0 commit comments

Comments
 (0)