chore(deps): bump minio from 7.1.3 to 8.0.6

.changeset/gold-doodles-ask.md

@@ -0,0 +1,6 @@
+---
+"@human-protocol/sdk": major
+"@human-protocol/python-sdk": major
+---
+
+Remove deprecated storage utilities and tests

packages/apps/fortune/exchange-oracle/server/package.json

@@ -50,7 +50,7 @@
     "ethers": "~6.15.0",
     "joi": "^17.13.3",
     "jsonwebtoken": "^9.0.2",
-    "minio": "7.1.3",
+    "minio": "8.0.6",
     "passport": "^0.7.0",
     "passport-jwt": "^4.0.1",
     "pg": "8.13.1",

packages/apps/fortune/exchange-oracle/server/scripts/setup-kv-store.ts

@@ -73,7 +73,7 @@ async function setupPublicKeyFile(
     throw new Error('Bucket does not exists');
   }
 
-  await minioClient.putObject(s3Bucket, keyName, publicKey, {
+  await minioClient.putObject(s3Bucket, keyName, publicKey, undefined, {
     'Content-Type': 'text/plain',
     'Cache-Control': 'no-store',
   });

packages/apps/fortune/exchange-oracle/server/src/common/guards/strategy/jwt.http.ts

@@ -3,14 +3,15 @@ import { PassportStrategy } from '@nestjs/passport';
 import { ExtractJwt, Strategy } from 'passport-jwt';
 
 import { KVStore__factory } from '@human-protocol/core/typechain-types';
-import { ChainId, NETWORKS, StorageClient } from '@human-protocol/sdk';
+import { ChainId, NETWORKS } from '@human-protocol/sdk';
 import { ethers } from 'ethers';
 import * as jwt from 'jsonwebtoken';
 import { JWT_KVSTORE_KEY, KYC_APPROVED } from '../../../common/constant';
 import { Role } from '../../../common/enums/role';
 import { JwtUser } from '../../../common/types/jwt';
 import { Web3Service } from '../../../modules/web3/web3.service';
 import { AuthError, ValidationError } from '../../errors';
+import { downloadFileFromUrl } from '../../utils/storage';
 
 @Injectable()
 export class JwtHttpStrategy extends PassportStrategy(Strategy, 'jwt-http') {
@@ -47,9 +48,7 @@ export class JwtHttpStrategy extends PassportStrategy(Strategy, 'jwt-http') {
               address,
               JWT_KVSTORE_KEY,
             );
-            publicKey = (await StorageClient.downloadFileFromUrl(
-              url,
-            )) as string;
+            publicKey = (await downloadFileFromUrl(url)) as string;
 
             this.publicKeyCache.set(cacheKey, {
               value: publicKey,

packages/apps/fortune/exchange-oracle/server/src/common/utils/storage.ts

@@ -0,0 +1,33 @@
+import axios from 'axios';
+import { HttpStatus } from '@nestjs/common';
+
+export const isValidUrl = (maybeUrl: string): boolean => {
+  try {
+    const { protocol } = new URL(maybeUrl);
+    return protocol === 'http:' || protocol === 'https:';
+  } catch {
+    return false;
+  }
+};
+
+export async function downloadFileFromUrl(url: string): Promise<any> {
+  if (!isValidUrl(url)) {
+    throw new Error('Invalid URL string');
+  }
+
+  try {
+    const { data, status } = await axios.get(url, {
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    if (status !== HttpStatus.OK) {
+      throw new Error('Storage file not found');
+    }
+
+    return data;
+  } catch {
+    throw new Error('Storage file not found');
+  }
+}

packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts

@@ -1,13 +1,7 @@
 import { createMock } from '@golevelup/ts-jest';
 import { HMToken__factory } from '@human-protocol/core/typechain-types';
-import {
-  Encryption,
-  EscrowClient,
-  OperatorUtils,
-  StorageClient,
-} from '@human-protocol/sdk';
+import { Encryption, EscrowClient, OperatorUtils } from '@human-protocol/sdk';
 import { HttpService } from '@nestjs/axios';
-import { BadRequestException, NotFoundException } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Test } from '@nestjs/testing';
 import { of } from 'rxjs';
@@ -31,6 +25,7 @@ import {
   ServerError,
   ValidationError,
 } from '../../common/errors';
+import { downloadFileFromUrl } from '../../common/utils/storage';
 import { AssignmentEntity } from '../assignment/assignment.entity';
 import { AssignmentRepository } from '../assignment/assignment.repository';
 import { StorageService } from '../storage/storage.service';
@@ -50,13 +45,14 @@ jest.mock('@human-protocol/sdk', () => ({
   OperatorUtils: {
     getOperator: jest.fn(),
   },
-  StorageClient: {
-    downloadFileFromUrl: jest.fn(),
-  },
   Encryption: {
     build: jest.fn(),
   },
 }));
+jest.mock('../../common/utils/storage', () => ({
+  ...jest.requireActual('../../common/utils/storage'),
+  downloadFileFromUrl: jest.fn(),
+}));
 jest.mock('minio', () => {
   class Client {
     putObject = jest.fn();
@@ -447,6 +443,7 @@ describe('JobService', () => {
   });
 
   describe('solveJob', () => {
+    const downloadFileFromUrlMock = jest.mocked(downloadFileFromUrl);
     const assignment = {
       id: 1,
       jobId: 1,
@@ -485,9 +482,7 @@ describe('JobService', () => {
 
       storageService.downloadJobSolutions = jest.fn().mockResolvedValueOnce([]);
 
-      StorageClient.downloadFileFromUrl = jest
-        .fn()
-        .mockResolvedValueOnce(manifest);
+      downloadFileFromUrlMock.mockResolvedValueOnce(manifest);
 
       const solutionsUrl =
         'http://localhost:9000/solution/0x1234567890123456789012345678901234567890-1.json';
@@ -552,9 +547,7 @@ describe('JobService', () => {
         },
       ]);
 
-      StorageClient.downloadFileFromUrl = jest
-        .fn()
-        .mockResolvedValueOnce(manifest);
+      downloadFileFromUrlMock.mockResolvedValueOnce(manifest);
 
       (Encryption.build as any).mockImplementation(() => ({
         decrypt: jest.fn().mockResolvedValue(JSON.stringify(manifest)),
@@ -587,9 +580,7 @@ describe('JobService', () => {
         },
       ]);
 
-      StorageClient.downloadFileFromUrl = jest
-        .fn()
-        .mockResolvedValueOnce(manifest);
+      downloadFileFromUrlMock.mockResolvedValueOnce(manifest);
 
       (Encryption.build as any).mockImplementation(() => ({
         decrypt: jest.fn().mockResolvedValue(JSON.stringify(manifest)),

packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts

@@ -7,10 +7,10 @@ import {
   Encryption,
   EncryptionUtils,
   EscrowClient,
-  StorageClient,
 } from '@human-protocol/sdk';
 import { Inject, Injectable } from '@nestjs/common';
 
+import { downloadFileFromUrl } from '../../common/utils/storage';
 import { PGPConfigService } from '../../common/config/pgp-config.service';
 import { ErrorAssignment, ErrorJob } from '../../common/constant/errors';
 import { SortDirection } from '../../common/enums/collection';
@@ -348,8 +348,7 @@ export class JobService {
     let manifest: ManifestDto | null = null;
 
     try {
-      const manifestEncrypted =
-        await StorageClient.downloadFileFromUrl(manifestUrl);
+      const manifestEncrypted = await downloadFileFromUrl(manifestUrl);
 
       if (
         typeof manifestEncrypted === 'string' &&

packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.spec.ts

@@ -2,7 +2,6 @@ import {
   ChainId,
   Encryption,
   EncryptionUtils,
-  StorageClient,
   EscrowClient,
   KVStoreUtils,
 } from '@human-protocol/sdk';
@@ -13,12 +12,10 @@ import { Web3Service } from '../web3/web3.service';
 import { ConfigService } from '@nestjs/config';
 import { S3ConfigService } from '../../common/config/s3-config.service';
 import { PGPConfigService } from '../../common/config/pgp-config.service';
+import { downloadFileFromUrl } from '../../common/utils/storage';
 
 jest.mock('@human-protocol/sdk', () => ({
   ...jest.requireActual('@human-protocol/sdk'),
-  StorageClient: {
-    downloadFileFromUrl: jest.fn(),
-  },
   Encryption: {
     build: jest.fn(),
   },
@@ -33,6 +30,11 @@ jest.mock('@human-protocol/sdk', () => ({
   },
 }));
 
+jest.mock('../../common/utils/storage', () => ({
+  ...jest.requireActual('../../common/utils/storage'),
+  downloadFileFromUrl: jest.fn(),
+}));
+
 jest.mock('minio', () => {
   class Client {
     putObject = jest.fn();
@@ -125,6 +127,7 @@ describe('StorageService', () => {
         s3ConfigService.bucket,
         `${escrowAddress}-${chainId}.json`,
         'encrypted',
+        undefined,
         {
           'Content-Type': 'application/json',
           'Cache-Control': 'no-store',
@@ -159,6 +162,7 @@ describe('StorageService', () => {
         s3ConfigService.bucket,
         `${escrowAddress}-${chainId}.json`,
         'encrypted',
+        undefined,
         {
           'Content-Type': 'application/json',
           'Cache-Control': 'no-store',
@@ -239,6 +243,8 @@ describe('StorageService', () => {
   });
 
   describe('downloadJobSolutions', () => {
+    const downloadFileFromUrlMock = jest.mocked(downloadFileFromUrl);
+
     it('should download the encrypted file correctly', async () => {
       const workerAddress = '0x1234567890123456789012345678901234567891';
       const escrowAddress = '0x1234567890123456789012345678901234567890';
@@ -252,9 +258,7 @@ describe('StorageService', () => {
         },
       ];
 
-      StorageClient.downloadFileFromUrl = jest
-        .fn()
-        .mockResolvedValue('encrypted-content');
+      downloadFileFromUrlMock.mockResolvedValue('encrypted-content');
 
       EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(true);
 
@@ -282,9 +286,7 @@ describe('StorageService', () => {
         },
       ];
 
-      StorageClient.downloadFileFromUrl = jest
-        .fn()
-        .mockResolvedValue(expectedJobFile);
+      downloadFileFromUrlMock.mockResolvedValue(expectedJobFile);
 
       EncryptionUtils.isEncrypted = jest.fn().mockReturnValue(false);
 
@@ -299,9 +301,7 @@ describe('StorageService', () => {
       const escrowAddress = '0x1234567890123456789012345678901234567890';
       const chainId = ChainId.LOCALHOST;
 
-      StorageClient.downloadFileFromUrl = jest
-        .fn()
-        .mockRejectedValue('Network error');
+      downloadFileFromUrlMock.mockRejectedValue('Network error');
 
       const solutionsFile = await storageService.downloadJobSolutions(
         escrowAddress,

packages/apps/fortune/exchange-oracle/server/src/modules/storage/storage.service.ts

@@ -4,11 +4,11 @@ import {
   EncryptionUtils,
   EscrowClient,
   KVStoreUtils,
-  StorageClient,
 } from '@human-protocol/sdk';
 import { Inject, Injectable } from '@nestjs/common';
 import * as Minio from 'minio';
 
+import { downloadFileFromUrl } from '../../common/utils/storage';
 import logger from '../../logger';
 import { PGPConfigService } from '../../common/config/pgp-config.service';
 import { S3ConfigService } from '../../common/config/s3-config.service';
@@ -49,7 +49,7 @@ export class StorageService {
   ): Promise<ISolution[]> {
     const url = this.getJobUrl(escrowAddress, chainId);
     try {
-      const fileContent = await StorageClient.downloadFileFromUrl(url);
+      const fileContent = await downloadFileFromUrl(url);
       if (EncryptionUtils.isEncrypted(fileContent)) {
         const encryption = await Encryption.build(
           this.pgpConfigService.privateKey!,
@@ -120,6 +120,7 @@ export class StorageService {
         this.s3ConfigService.bucket,
         `${escrowAddress}-${chainId}.json`,
         fileToUpload,
+        undefined,
         {
           'Content-Type': 'application/json',
           'Cache-Control': 'no-store',

packages/apps/fortune/recording-oracle/package.json

@@ -38,7 +38,7 @@
     "dotenv": "^17.2.2",
     "helmet": "^7.1.0",
     "joi": "^17.13.3",
-    "minio": "7.1.3",
+    "minio": "8.0.6",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.2.0"
   },

packages/apps/fortune/recording-oracle/scripts/setup-kv-store.ts

@@ -74,7 +74,7 @@ async function setupPublicKeyFile(
     throw new Error('Bucket does not exists');
   }
 
-  await minioClient.putObject(s3Bucket, keyName, publicKey, {
+  await minioClient.putObject(s3Bucket, keyName, publicKey, undefined, {
     'Content-Type': 'text/plain',
     'Cache-Control': 'no-store',
   });

packages/apps/fortune/recording-oracle/src/common/utils/storage.ts

@@ -0,0 +1,33 @@
+import axios from 'axios';
+import { HttpStatus } from '@nestjs/common';
+
+export const isValidUrl = (maybeUrl: string): boolean => {
+  try {
+    const { protocol } = new URL(maybeUrl);
+    return protocol === 'http:' || protocol === 'https:';
+  } catch {
+    return false;
+  }
+};
+
+export async function downloadFileFromUrl(url: string): Promise<any> {
+  if (!isValidUrl(url)) {
+    throw new Error('Invalid URL string');
+  }
+
+  try {
+    const { data, status } = await axios.get(url, {
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    if (status !== HttpStatus.OK) {
+      throw new Error('Storage file not found');
+    }
+
+    return data;
+  } catch {
+    throw new Error('Storage file not found');
+  }
+}