feat: get rid of sdk's storage module

Author: dnechay

packages/apps/job-launcher/server/tsconfig.json

@@ -7,7 +7,7 @@
     "experimentalDecorators": true,
     "allowSyntheticDefaultImports": true,
     "allowJs": true,
-    "target": "ES2020",
+    "target": "ES2022",
     "sourceMap": true,
     "outDir": "./dist",
     "baseUrl": "./",

packages/apps/reputation-oracle/server/docker-compose.yml

@@ -26,12 +26,9 @@ services:
     ports:
       - 9001:9001
       - 9000:9000
-    environment:
-      MINIO_ROOT_USER: ${S3_ACCESS_KEY}
-      MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY}
     entrypoint: 'sh'
     command:
-      -c "mkdir -p /data/reputation && minio server /data --console-address ':9001'"
+      -c "minio server /data --console-address ':9001'"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 5s
@@ -45,7 +42,7 @@ services:
         condition: service_healthy
     entrypoint: >
       /bin/sh -c "
-      /usr/bin/mc config host add myminio http://minio:9000 ${S3_ACCESS_KEY} ${S3_SECRET_KEY};
+      /usr/bin/mc config host add myminio http://minio:9000 minioadmin minioadmin;
       /usr/bin/mc mb myminio/reputation;
       /usr/bin/mc anonymous set public myminio/reputation;
       "

packages/apps/reputation-oracle/server/src/common/errors/base.ts

@@ -0,0 +1,11 @@
+export class BaseError extends Error {
+  constructor(message: string, cause?: unknown) {
+    const errorOptions: ErrorOptions = {};
+    if (cause) {
+      errorOptions.cause = cause;
+    }
+
+    super(message, errorOptions);
+    this.name = this.constructor.name;
+  }
+}

packages/apps/reputation-oracle/server/src/modules/payout/payout.service.spec.ts

@@ -114,7 +114,9 @@ describe('PayoutService', () => {
         job_bounty: '10',
       };
 
-      jest.spyOn(storageService, 'download').mockResolvedValue(manifest);
+      jest
+        .spyOn(storageService, 'downloadJsonLikeData')
+        .mockResolvedValue(manifest);
 
       const processing_results = {
         recipients: [MOCK_ADDRESS],
@@ -146,7 +148,9 @@ describe('PayoutService', () => {
         requestType: JobRequestType.FORTUNE,
       };
 
-      jest.spyOn(storageService, 'download').mockResolvedValue(manifest);
+      jest
+        .spyOn(storageService, 'downloadJsonLikeData')
+        .mockResolvedValue(manifest);
 
       const processing_results = {
         recipients: [MOCK_ADDRESS],
@@ -188,7 +192,7 @@ describe('PayoutService', () => {
       ];
 
       jest
-        .spyOn(storageService, 'download')
+        .spyOn(storageService, 'downloadJsonLikeData')
         .mockResolvedValue(intermediateResults);
 
       const escrowAddress = MOCK_ADDRESS;
@@ -220,7 +224,7 @@ describe('PayoutService', () => {
       ];
 
       jest
-        .spyOn(storageService, 'download')
+        .spyOn(storageService, 'downloadJsonLikeData')
         .mockResolvedValue(intermediateResults);
 
       const escrowAddress = MOCK_ADDRESS;
@@ -284,7 +288,9 @@ describe('PayoutService', () => {
         ],
       };
 
-      jest.spyOn(storageService, 'download').mockResolvedValue(results);
+      jest
+        .spyOn(storageService, 'downloadJsonLikeData')
+        .mockResolvedValue(results);
 
       jest
         .spyOn(storageService, 'copyFileFromURLToBucket')

packages/apps/reputation-oracle/server/src/modules/payout/payout.service.ts

@@ -55,7 +55,8 @@ export class PayoutService {
       );
     }
 
-    const manifest = await this.storageService.download(manifestUrl);
+    const manifest =
+      await this.storageService.downloadJsonLikeData(manifestUrl);
 
     const requestType = getRequestType(manifest);
 
@@ -141,7 +142,7 @@ export class PayoutService {
     const intermediateResultsUrl =
       await escrowClient.getIntermediateResultsUrl(escrowAddress);
 
-    const intermediateResults = (await this.storageService.download(
+    const intermediateResults = (await this.storageService.downloadJsonLikeData(
       intermediateResultsUrl,
     )) as FortuneFinalResult[];
 
@@ -206,9 +207,10 @@ export class PayoutService {
       chainId,
       `${intermediateResultsUrl}/${CVAT_RESULTS_ANNOTATIONS_FILENAME}`,
     );
-    const annotations: CvatAnnotationMeta = await this.storageService.download(
-      `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`,
-    );
+    const annotations: CvatAnnotationMeta =
+      await this.storageService.downloadJsonLikeData(
+        `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`,
+      );
 
     // If annotation meta does not exist
     if (annotations && Array.isArray(annotations) && annotations.length === 0) {

packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts

@@ -142,7 +142,7 @@ describe('ReputationService', () => {
         };
 
         jest
-          .spyOn(storageService, 'download')
+          .spyOn(storageService, 'downloadJsonLikeData')
           .mockResolvedValueOnce(manifest) // Mock manifest
           .mockResolvedValueOnce([]); // Mock final results
 
@@ -170,7 +170,7 @@ describe('ReputationService', () => {
         ];
 
         jest
-          .spyOn(storageService, 'download')
+          .spyOn(storageService, 'downloadJsonLikeData')
           .mockResolvedValueOnce(manifest)
           .mockResolvedValueOnce(finalResults);
 
@@ -243,7 +243,7 @@ describe('ReputationService', () => {
         }));
 
         jest
-          .spyOn(storageService, 'download')
+          .spyOn(storageService, 'downloadJsonLikeData')
           .mockResolvedValueOnce(manifest) // Mock manifest
           .mockResolvedValueOnce([]); // Mock final results
 
@@ -294,7 +294,7 @@ describe('ReputationService', () => {
         };
 
         jest
-          .spyOn(storageService, 'download')
+          .spyOn(storageService, 'downloadJsonLikeData')
           .mockResolvedValueOnce(manifest)
           .mockResolvedValueOnce(annotationMeta);
 

packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts

@@ -65,7 +65,8 @@ export class ReputationService {
       );
     }
 
-    const manifest = await this.storageService.download(manifestUrl);
+    const manifest =
+      await this.storageService.downloadJsonLikeData(manifestUrl);
 
     const requestType = getRequestType(manifest);
 
@@ -160,7 +161,8 @@ export class ReputationService {
     const escrowClient = await EscrowClient.build(signer);
 
     const finalResultsUrl = await escrowClient.getResultsUrl(escrowAddress);
-    const finalResults = await this.storageService.download(finalResultsUrl);
+    const finalResults =
+      await this.storageService.downloadJsonLikeData(finalResultsUrl);
 
     if (finalResults.length === 0) {
       throw new ControlledError(
@@ -202,9 +204,10 @@ export class ReputationService {
     const intermediateResultsUrl =
       await escrowClient.getIntermediateResultsUrl(escrowAddress);
 
-    const annotations: CvatAnnotationMeta = await this.storageService.download(
-      `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`,
-    );
+    const annotations: CvatAnnotationMeta =
+      await this.storageService.downloadJsonLikeData(
+        `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`,
+      );
 
     // If annotation meta does not exist
     if (annotations && Array.isArray(annotations) && annotations.length === 0) {

packages/apps/reputation-oracle/server/src/modules/storage/storage.errors.ts

@@ -0,0 +1,25 @@
+import { BaseError } from '../../common/errors/base';
+
+export class FileDownloadError extends BaseError {
+  public readonly location: string;
+
+  constructor(location: string, cause?: unknown) {
+    super('Failed to download file', cause);
+
+    this.location = location;
+  }
+}
+
+export class InvalidFileUrl extends FileDownloadError {
+  constructor(url: string) {
+    super(url);
+    this.message = 'Invalid file URL';
+  }
+}
+
+export class FileNotFoundError extends FileDownloadError {
+  constructor(location: string) {
+    super(location);
+    this.message = 'File not found';
+  }
+}