diff --git a/packages/apps/job-launcher/server/tsconfig.json b/packages/apps/job-launcher/server/tsconfig.json index fb6e941176..bbd076c0f3 100644 --- a/packages/apps/job-launcher/server/tsconfig.json +++ b/packages/apps/job-launcher/server/tsconfig.json @@ -7,7 +7,7 @@ "experimentalDecorators": true, "allowSyntheticDefaultImports": true, "allowJs": true, - "target": "ES2020", + "target": "ES2022", "sourceMap": true, "outDir": "./dist", "baseUrl": "./", diff --git a/packages/apps/reputation-oracle/server/docker-compose.yml b/packages/apps/reputation-oracle/server/docker-compose.yml index c6df1a8bb8..10d1a2ad1d 100644 --- a/packages/apps/reputation-oracle/server/docker-compose.yml +++ b/packages/apps/reputation-oracle/server/docker-compose.yml @@ -26,12 +26,9 @@ services: ports: - 9001:9001 - 9000:9000 - environment: - MINIO_ROOT_USER: ${S3_ACCESS_KEY} - MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY} entrypoint: 'sh' command: - -c "mkdir -p /data/reputation && minio server /data --console-address ':9001'" + -c "minio server /data --console-address ':9001'" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 5s @@ -45,7 +42,7 @@ services: condition: service_healthy entrypoint: > /bin/sh -c " - /usr/bin/mc config host add myminio http://minio:9000 ${S3_ACCESS_KEY} ${S3_SECRET_KEY}; + /usr/bin/mc config host add myminio http://minio:9000 minioadmin minioadmin; /usr/bin/mc mb myminio/reputation; /usr/bin/mc anonymous set public myminio/reputation; " diff --git a/packages/apps/reputation-oracle/server/src/common/errors/base.ts b/packages/apps/reputation-oracle/server/src/common/errors/base.ts new file mode 100644 index 0000000000..dfc528a73b --- /dev/null +++ b/packages/apps/reputation-oracle/server/src/common/errors/base.ts @@ -0,0 +1,11 @@ +export class BaseError extends Error { + constructor(message: string, cause?: unknown) { + const errorOptions: ErrorOptions = {}; + if (cause) { + errorOptions.cause = cause; + } + + super(message, errorOptions); + this.name = this.constructor.name; + } +} diff --git a/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.spec.ts b/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.spec.ts index 04899aeceb..62d480f0dc 100644 --- a/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.spec.ts @@ -142,7 +142,9 @@ describe('PayoutService', () => { job_bounty: '10', }; - jest.spyOn(storageService, 'download').mockResolvedValue(manifest); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue(manifest); const escrowAddress = MOCK_ADDRESS; const chainId = ChainId.LOCALHOST; @@ -160,7 +162,9 @@ describe('PayoutService', () => { requestType: JobRequestType.FORTUNE, }; - jest.spyOn(storageService, 'download').mockResolvedValue(manifest); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue(manifest); const results: SaveResultDto = { url: MOCK_FILE_URL, @@ -236,7 +240,9 @@ describe('PayoutService', () => { job_bounty: '10', }; - jest.spyOn(storageService, 'download').mockResolvedValue(manifest); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue(manifest); const escrowAddress = MOCK_ADDRESS; const chainId = ChainId.LOCALHOST; @@ -261,7 +267,9 @@ describe('PayoutService', () => { requestType: JobRequestType.FORTUNE, }; - jest.spyOn(storageService, 'download').mockResolvedValue(manifest); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue(manifest); const results: PayoutsDataDto = { recipients: [MOCK_ADDRESS], @@ -332,7 +340,7 @@ describe('PayoutService', () => { ]; jest - .spyOn(storageService, 'download') + .spyOn(storageService, 'downloadJsonLikeData') .mockResolvedValue(intermediateResults); jest.spyOn(storageService, 'uploadJobSolutions').mockResolvedValue({ @@ -361,7 +369,9 @@ describe('PayoutService', () => { requestType: JobRequestType.FORTUNE, }; - jest.spyOn(storageService, 'download').mockResolvedValue([] as any); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue([] as any); await expect( payoutService.saveResultsFortune(manifest, chainId, escrowAddress), @@ -393,7 +403,7 @@ describe('PayoutService', () => { ]; jest - .spyOn(storageService, 'download') + .spyOn(storageService, 'downloadJsonLikeData') .mockResolvedValue(intermediateResults); await expect( @@ -425,7 +435,7 @@ describe('PayoutService', () => { ]; jest - .spyOn(storageService, 'download') + .spyOn(storageService, 'downloadJsonLikeData') .mockResolvedValue(intermediateResults); const result = await payoutService.calculatePayoutsFortune( @@ -475,7 +485,9 @@ describe('PayoutService', () => { url: MOCK_FILE_URL, hash: MOCK_FILE_HASH, }); - jest.spyOn(storageService, 'download').mockResolvedValue(results); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue(results); const result = await payoutService.saveResultsCvat( chainId, @@ -539,7 +551,9 @@ describe('PayoutService', () => { ], }; - jest.spyOn(storageService, 'download').mockResolvedValue(results); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue(results); const result = await payoutService.calculatePayoutsCvat( manifest as any, @@ -563,7 +577,9 @@ describe('PayoutService', () => { results: [], }; - jest.spyOn(storageService, 'download').mockResolvedValue(results); + jest + .spyOn(storageService, 'downloadJsonLikeData') + .mockResolvedValue(results); await expect( payoutService.calculatePayoutsCvat( diff --git a/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.ts b/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.ts index 78ef73b31c..ab27826c3b 100644 --- a/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/payout/payout.service.ts @@ -60,7 +60,8 @@ export class PayoutService { ); } - const manifest = await this.storageService.download(manifestUrl); + const manifest = + await this.storageService.downloadJsonLikeData(manifestUrl); const requestType = getRequestType(manifest); @@ -102,7 +103,8 @@ export class PayoutService { ); } - const manifest = await this.storageService.download(manifestUrl); + const manifest = + await this.storageService.downloadJsonLikeData(manifestUrl); const requestType = getRequestType(manifest); @@ -209,7 +211,7 @@ export class PayoutService { const intermediateResultsUrl = await escrowClient.getIntermediateResultsUrl(escrowAddress); - const intermediateResults = (await this.storageService.download( + const intermediateResults = (await this.storageService.downloadJsonLikeData( intermediateResultsUrl, )) as FortuneFinalResult[]; @@ -279,7 +281,7 @@ export class PayoutService { manifest: FortuneManifestDto, finalResultsUrl: string, ): Promise { - const finalResults = (await this.storageService.download( + const finalResults = (await this.storageService.downloadJsonLikeData( finalResultsUrl, )) as FortuneFinalResult[]; @@ -315,9 +317,10 @@ export class PayoutService { const intermediateResultsUrl = await escrowClient.getIntermediateResultsUrl(escrowAddress); - const annotations: CvatAnnotationMeta = await this.storageService.download( - `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`, - ); + const annotations: CvatAnnotationMeta = + await this.storageService.downloadJsonLikeData( + `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`, + ); // If annotation meta results does not exist if ( diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts index 102a70c7bc..9923482936 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts @@ -137,7 +137,7 @@ describe('ReputationService', () => { }; jest - .spyOn(storageService, 'download') + .spyOn(storageService, 'downloadJsonLikeData') .mockResolvedValueOnce(manifest) // Mock manifest .mockResolvedValueOnce([]); // Mock final results @@ -165,7 +165,7 @@ describe('ReputationService', () => { ]; jest - .spyOn(storageService, 'download') + .spyOn(storageService, 'downloadJsonLikeData') .mockResolvedValueOnce(manifest) .mockResolvedValueOnce(finalResults); @@ -238,7 +238,7 @@ describe('ReputationService', () => { })); jest - .spyOn(storageService, 'download') + .spyOn(storageService, 'downloadJsonLikeData') .mockResolvedValueOnce(manifest) // Mock manifest .mockResolvedValueOnce([]); // Mock final results @@ -289,7 +289,7 @@ describe('ReputationService', () => { }; jest - .spyOn(storageService, 'download') + .spyOn(storageService, 'downloadJsonLikeData') .mockResolvedValueOnce(manifest) .mockResolvedValueOnce(annotationMeta); diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts index 734ff6bab7..6357830530 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts @@ -65,7 +65,8 @@ export class ReputationService { ); } - const manifest = await this.storageService.download(manifestUrl); + const manifest = + await this.storageService.downloadJsonLikeData(manifestUrl); const requestType = getRequestType(manifest); @@ -160,7 +161,8 @@ export class ReputationService { const escrowClient = await EscrowClient.build(signer); const finalResultsUrl = await escrowClient.getResultsUrl(escrowAddress); - const finalResults = await this.storageService.download(finalResultsUrl); + const finalResults = + await this.storageService.downloadJsonLikeData(finalResultsUrl); if (finalResults.length === 0) { throw new ControlledError( @@ -202,9 +204,10 @@ export class ReputationService { const intermediateResultsUrl = await escrowClient.getIntermediateResultsUrl(escrowAddress); - const annotations: CvatAnnotationMeta = await this.storageService.download( - `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`, - ); + const annotations: CvatAnnotationMeta = + await this.storageService.downloadJsonLikeData( + `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`, + ); // If annotation meta does not exist if (annotations && Array.isArray(annotations) && annotations.length === 0) { diff --git a/packages/apps/reputation-oracle/server/src/modules/storage/storage.errors.ts b/packages/apps/reputation-oracle/server/src/modules/storage/storage.errors.ts new file mode 100644 index 0000000000..b4eecc4ce4 --- /dev/null +++ b/packages/apps/reputation-oracle/server/src/modules/storage/storage.errors.ts @@ -0,0 +1,25 @@ +import { BaseError } from '../../common/errors/base'; + +export class FileDownloadError extends BaseError { + public readonly location: string; + + constructor(location: string, cause?: unknown) { + super('Failed to download file', cause); + + this.location = location; + } +} + +export class InvalidFileUrl extends FileDownloadError { + constructor(url: string) { + super(url); + this.message = 'Invalid file URL'; + } +} + +export class FileNotFoundError extends FileDownloadError { + constructor(location: string) { + super(location); + this.message = 'File not found'; + } +} diff --git a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts index 4a26760af0..06d34a74e9 100644 --- a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts @@ -4,7 +4,6 @@ import { EncryptionUtils, EscrowClient, KVStoreUtils, - StorageClient, } from '@human-protocol/sdk'; import { ConfigService } from '@nestjs/config'; import { Test } from '@nestjs/testing'; @@ -24,9 +23,6 @@ import { HttpStatus } from '@nestjs/common'; jest.mock('@human-protocol/sdk', () => ({ ...jest.requireActual('@human-protocol/sdk'), - StorageClient: { - downloadFileFromUrl: jest.fn(), - }, EscrowClient: { build: jest.fn(), }, @@ -63,6 +59,7 @@ describe('StorageService', () => { let storageService: StorageService; let pgpConfigService: PGPConfigService; let s3ConfigService: S3ConfigService; + let downloadFileFromUrlSpy: jest.SpyInstance; const signerMock = { address: '0x1234567890123456789012345678901234567892', @@ -110,6 +107,10 @@ describe('StorageService', () => { jest.spyOn(pgpConfigService, 'encrypt', 'get').mockReturnValue(true); }); + beforeEach(() => { + downloadFileFromUrlSpy = jest.spyOn(StorageService, 'downloadFileFromUrl'); + }); + describe('uploadJobSolutions', () => { it('should upload the solutions correctly', async () => { const workerAddress = '0x1234567890123456789012345678901234567891'; @@ -302,13 +303,13 @@ describe('StorageService', () => { }); }); - describe('download', () => { - it('should download the file correctly', async () => { + describe('downloadJsonLikeData', () => { + it('should download data correctly', async () => { const exchangeAddress = '0x1234567890123456789012345678901234567892'; const workerAddress = '0x1234567890123456789012345678901234567891'; const solution = 'test'; - const expectedJobFile = { + const expectedJobJson = { exchangeAddress, solutions: [ { @@ -318,19 +319,21 @@ describe('StorageService', () => { ], }; - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce(expectedJobFile); - const solutionsFile = await storageService.download(MOCK_FILE_URL); - expect(solutionsFile).toStrictEqual(expectedJobFile); + downloadFileFromUrlSpy.mockResolvedValueOnce( + Buffer.from(JSON.stringify(expectedJobJson)), + ); + + const solutionsJson = + await storageService.downloadJsonLikeData(MOCK_FILE_URL); + expect(solutionsJson).toEqual(expectedJobJson); }); - it('should download the encrypted file correctly', async () => { + it('should download the encrypted data correctly', async () => { const exchangeAddress = '0x1234567890123456789012345678901234567892'; const workerAddress = '0x1234567890123456789012345678901234567891'; const solution = 'test'; - const expectedJobFile = { + const expectedJobJson = { exchangeAddress, solutions: [ { @@ -340,36 +343,33 @@ describe('StorageService', () => { ], }; - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce('encrypted'); + downloadFileFromUrlSpy.mockResolvedValueOnce(Buffer.from('encrypted')); EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(true); Encryption.build = jest.fn().mockResolvedValue({ - decrypt: jest.fn().mockResolvedValue(JSON.stringify(expectedJobFile)), + decrypt: jest.fn().mockResolvedValue(JSON.stringify(expectedJobJson)), }); - const solutionsFile = await storageService.download(MOCK_FILE_URL); - expect(solutionsFile).toStrictEqual(expectedJobFile); + const solutionsJson = + await storageService.downloadJsonLikeData(MOCK_FILE_URL); + expect(solutionsJson).toEqual(expectedJobJson); }); - it('should return empty array when file cannot be downloaded', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockRejectedValue('Network error'); + it('should return empty array when data cannot be downloaded', async () => { + downloadFileFromUrlSpy.mockRejectedValue('Network error'); - const solutionsFile = await storageService.download(MOCK_FILE_URL); - expect(solutionsFile).toStrictEqual([]); + const solutionsJson = + await storageService.downloadJsonLikeData(MOCK_FILE_URL); + expect(solutionsJson).toEqual([]); }); }); describe('copyFileFromURLToBucket', () => { + const someFileContent = Buffer.from('some-file-content'); const escrowAddress = '0x1234567890123456789012345678901234567890'; const chainId = ChainId.LOCALHOST; it('should copy a file from a valid URL to a bucket', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce('some-file-content'); + downloadFileFromUrlSpy.mockResolvedValueOnce(someFileContent); EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(false); EncryptionUtils.encrypt = jest @@ -395,21 +395,22 @@ describe('StorageService', () => { ), ).toBeTruthy(); expect(uploadedFile.hash).toBeDefined(); - expect(storageService.minioClient.putObject).toBeCalledWith( + expect(storageService.minioClient.putObject).toHaveBeenCalledWith( s3ConfigService.bucket, `s3${crypto .createHash('sha1') .update('encrypted-file-content') .digest('hex')}.zip`, 'encrypted-file-content', - { 'Cache-Control': 'no-store', 'Content-Type': 'application/json' }, + { + 'Cache-Control': 'no-store', + 'Content-Type': 'text/plain', + }, ); }); it('should copy an encrypted file from a valid URL to a bucket', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce('some-file-content'); + downloadFileFromUrlSpy.mockResolvedValueOnce(someFileContent); Encryption.build = jest.fn().mockResolvedValue({ decrypt: jest.fn().mockResolvedValue('decrypted-file-content'), }); @@ -435,21 +436,22 @@ describe('StorageService', () => { ), ).toBeTruthy(); expect(uploadedFile.hash).toBeDefined(); - expect(storageService.minioClient.putObject).toBeCalledWith( + expect(storageService.minioClient.putObject).toHaveBeenCalledWith( s3ConfigService.bucket, `s3${crypto .createHash('sha1') .update('encrypted-file-content') .digest('hex')}.zip`, 'encrypted-file-content', - { 'Cache-Control': 'no-store', 'Content-Type': 'application/json' }, + { + 'Cache-Control': 'no-store', + 'Content-Type': 'text/plain', + }, ); }); it('should return the URL of the file and a hash if it already exists', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce('some-file-content'); + downloadFileFromUrlSpy.mockResolvedValueOnce(someFileContent); Encryption.build = jest.fn().mockResolvedValue({ decrypt: jest.fn().mockResolvedValue('decrypted-file-content'), }); @@ -491,14 +493,9 @@ describe('StorageService', () => { }); it('should copy a file from a valid URL to a bucket', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce('some-file-content'); + downloadFileFromUrlSpy.mockResolvedValueOnce(someFileContent); EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(false); - EncryptionUtils.encrypt = jest - .fn() - .mockResolvedValueOnce('encrypted-file-content'); storageService.minioClient.statObject = jest .fn() @@ -519,26 +516,28 @@ describe('StorageService', () => { ), ).toBeTruthy(); expect(uploadedFile.hash).toBeDefined(); - expect(storageService.minioClient.putObject).toBeCalledWith( + expect(storageService.minioClient.putObject).toHaveBeenCalledWith( s3ConfigService.bucket, `s3${crypto .createHash('sha1') .update('some-file-content') .digest('hex')}.zip`, - 'some-file-content', - { 'Cache-Control': 'no-store', 'Content-Type': 'application/json' }, + someFileContent, + { + 'Cache-Control': 'no-store', + 'Content-Type': 'text/plain', + }, ); }); it('should copy an encrypted file from a valid URL to a bucket', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockResolvedValueOnce('some-file-content'); + downloadFileFromUrlSpy.mockResolvedValueOnce(someFileContent); + + EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(true); Encryption.build = jest.fn().mockResolvedValue({ - decrypt: jest.fn().mockResolvedValue('decrypted-file-content'), + decrypt: jest.fn().mockResolvedValue(someFileContent), }); - EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(true); EncryptionUtils.encrypt = jest .fn() .mockResolvedValueOnce('encrypted-file-content'); @@ -559,28 +558,27 @@ describe('StorageService', () => { ), ).toBeTruthy(); expect(uploadedFile.hash).toBeDefined(); - expect(storageService.minioClient.putObject).toBeCalledWith( + expect(storageService.minioClient.putObject).toHaveBeenCalledWith( s3ConfigService.bucket, `s3${crypto .createHash('sha1') - .update('some-file-content') + .update(someFileContent) .digest('hex')}.zip`, - 'some-file-content', - { 'Cache-Control': 'no-store', 'Content-Type': 'application/json' }, + someFileContent, + { + 'Cache-Control': 'no-store', + 'Content-Type': 'text/plain', + }, ); }); }); it('should handle an invalid URL', async () => { - StorageClient.downloadFileFromUrl = jest - .fn() - .mockRejectedValueOnce('Invalid URL'); - await expect( storageService.copyFileFromURLToBucket( escrowAddress, chainId, - MOCK_FILE_URL, + 'invalid url', ), ).rejects.toThrow( new ControlledError('File not uploaded', HttpStatus.BAD_REQUEST), diff --git a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts index 01f3991886..2eb086e55d 100644 --- a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts @@ -4,9 +4,9 @@ import { EncryptionUtils, EscrowClient, KVStoreUtils, - StorageClient, } from '@human-protocol/sdk'; import { HttpStatus, Injectable } from '@nestjs/common'; +import axios from 'axios'; import * as Minio from 'minio'; import crypto from 'crypto'; import { UploadedFile } from '../../common/interfaces/s3'; @@ -17,6 +17,11 @@ import { S3ConfigService } from '../../common/config/s3-config.service'; import { PGPConfigService } from '../../common/config/pgp-config.service'; import { ControlledError } from '../../common/errors/controlled'; import { isNotFoundError } from '../../common/utils/minio'; +import { + FileDownloadError, + FileNotFoundError, + InvalidFileUrl, +} from './storage.errors'; @Injectable() export class StorageService { @@ -35,6 +40,7 @@ export class StorageService { useSSL: this.s3ConfigService.useSSL, }); } + public getUrl(key: string): string { return `${this.s3ConfigService.useSSL ? 'https' : 'http'}://${ this.s3ConfigService.endpoint @@ -75,34 +81,64 @@ export class StorageService { ]); } - private async decryptFile(fileContent: any): Promise { - if ( - typeof fileContent === 'string' && - EncryptionUtils.isEncrypted(fileContent) - ) { - const encryption = await Encryption.build( - this.pgpConfigService.privateKey!, - this.pgpConfigService.passphrase, - ); - const decryptedData = await encryption.decrypt(fileContent); - const content = Buffer.from(decryptedData).toString(); - try { - return JSON.parse(content); - } catch { - return content; - } - } else { + private async maybeDecryptFile(fileContent: Buffer): Promise { + const contentAsString = fileContent.toString(); + if (!EncryptionUtils.isEncrypted(contentAsString)) { return fileContent; } + + const encryption = await Encryption.build( + this.pgpConfigService.privateKey!, + this.pgpConfigService.passphrase, + ); + + const decryptedData = await encryption.decrypt(contentAsString); + + return Buffer.from(decryptedData); } - public async download(url: string): Promise { + public static isValidUrl(maybeUrl: string): boolean { try { - const fileContent = await StorageClient.downloadFileFromUrl(url); + const url = new URL(maybeUrl); + return ['http:', 'https:'].includes(url.protocol); + } catch (_error) { + return false; + } + } - return await this.decryptFile(fileContent); + public static async downloadFileFromUrl(url: string): Promise { + if (!this.isValidUrl(url)) { + throw new InvalidFileUrl(url); + } + + try { + const { data } = await axios.get(url, { + responseType: 'arraybuffer', + }); + + return Buffer.from(data); } catch (error) { - Logger.error(`Error downloading ${url}:`, error); + if (error.response?.status === HttpStatus.NOT_FOUND) { + throw new FileNotFoundError(url); + } + throw new FileDownloadError(url, error.cause || error.message); + } + } + + public async downloadJsonLikeData(url: string): Promise { + try { + let fileContent = await StorageService.downloadFileFromUrl(url); + + fileContent = await this.maybeDecryptFile(fileContent); + + let jsonLikeData = fileContent.toString(); + try { + jsonLikeData = JSON.parse(jsonLikeData); + } catch (_noop) {} + + return jsonLikeData; + } catch (error) { + Logger.error(`Error downloading json like data ${url}:`, error); return []; } } @@ -169,10 +205,8 @@ export class StorageService { url: string, ): Promise { try { - // Download the content of the file from the bucket - let fileContent = await StorageClient.downloadFileFromUrl(url); - fileContent = await this.decryptFile(fileContent); - + let fileContent = await StorageService.downloadFileFromUrl(url); + fileContent = await this.maybeDecryptFile(fileContent); // Encrypt for job launcher const content = await this.encryptFile( escrowAddress, @@ -204,7 +238,7 @@ export class StorageService { key, content, { - 'Content-Type': 'application/json', + 'Content-Type': 'text/plain', 'Cache-Control': 'no-store', }, ); diff --git a/packages/apps/reputation-oracle/server/test/constants.ts b/packages/apps/reputation-oracle/server/test/constants.ts index 0d230b3907..94b3342667 100644 --- a/packages/apps/reputation-oracle/server/test/constants.ts +++ b/packages/apps/reputation-oracle/server/test/constants.ts @@ -6,7 +6,7 @@ import { IFortuneManifest } from '../src/common/interfaces/manifest'; export const MOCK_REQUESTER_TITLE = 'Mock job title'; export const MOCK_REQUESTER_DESCRIPTION = 'Mock job description'; export const MOCK_ADDRESS = '0xCf88b3f1992458C2f5a229573c768D0E9F70C44e'; -export const MOCK_FILE_URL = 'mockedFileUrl'; +export const MOCK_FILE_URL = 'http://local.test/some-mocked-file-url'; export const MOCK_WEBHOOK_URL = 'mockedWebhookUrl'; export const MOCK_FILE_HASH = 'mockedFileHash'; export const MOCK_FILE_KEY = 'manifest.json'; diff --git a/packages/apps/reputation-oracle/server/tsconfig.json b/packages/apps/reputation-oracle/server/tsconfig.json index fb6e941176..bbd076c0f3 100644 --- a/packages/apps/reputation-oracle/server/tsconfig.json +++ b/packages/apps/reputation-oracle/server/tsconfig.json @@ -7,7 +7,7 @@ "experimentalDecorators": true, "allowSyntheticDefaultImports": true, "allowJs": true, - "target": "ES2020", + "target": "ES2022", "sourceMap": true, "outDir": "./dist", "baseUrl": "./",