test: payouts calculators

Author: dnechay

packages/apps/reputation-oracle/server/jest.config.ts

@@ -16,4 +16,5 @@ module.exports = {
     '^uuid$': require.resolve('uuid'),
     '^typeorm$': require.resolve('typeorm'),
   },
+  clearMocks: true,
 };

packages/apps/reputation-oracle/server/src/modules/escrow-completion/fixtures/cvat.ts

@@ -0,0 +1,16 @@
+import { faker } from '@faker-js/faker';
+
+import { CvatJobType } from '../../../common/enums';
+import { CvatManifest } from '../../../common/types';
+
+export function generateCvatManifest(): CvatManifest {
+  return {
+    annotation: {
+      type: faker.helpers.arrayElement(Object.values(CvatJobType)),
+    },
+    validation: {
+      min_quality: faker.number.float({ min: 0.1, max: 0.9 }),
+    },
+    job_bounty: faker.finance.amount({ max: 42 }),
+  };
+}

packages/apps/reputation-oracle/server/src/modules/escrow-completion/fixtures/fortune.ts

@@ -0,0 +1,20 @@
+import { faker } from '@faker-js/faker';
+
+import { FortuneJobType } from '../../../common/enums';
+import { FortuneFinalResult, FortuneManifest } from '../../../common/types';
+
+export function generateFortuneManifest(): FortuneManifest {
+  return {
+    requestType: FortuneJobType.FORTUNE,
+    fundAmount: Number(faker.finance.amount()),
+    submissionsRequired: faker.number.int({ min: 2, max: 5 }),
+  };
+}
+
+export function generateFortuneSolution(error?: string): FortuneFinalResult {
+  return {
+    workerAddress: faker.finance.ethereumAddress(),
+    solution: faker.string.sample(),
+    error: error as FortuneFinalResult['error'],
+  };
+}

packages/apps/reputation-oracle/server/src/modules/escrow-completion/fixtures/index.ts

@@ -0,0 +1,2 @@
+export * from './fortune';
+export * from './cvat';

packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts

@@ -0,0 +1,152 @@
+jest.mock('@human-protocol/sdk');
+
+import { createMock } from '@golevelup/ts-jest';
+import { faker } from '@faker-js/faker';
+import { EscrowClient } from '@human-protocol/sdk';
+import { Test } from '@nestjs/testing';
+import { ethers } from 'ethers';
+import _ from 'lodash';
+
+import { CvatAnnotationMeta } from '../../../common/types';
+
+import { StorageService } from '../../storage/storage.service';
+import { generateTestnetChainId } from '../../web3/fixtures';
+import { Web3Service } from '../../web3/web3.service';
+
+import { CvatPayoutsCalculator } from './cvat-payouts-calculator';
+import { generateCvatManifest } from '../fixtures';
+
+const mockedStorageService = createMock<StorageService>();
+const mockedWeb3Service = createMock<Web3Service>();
+
+const mockedEscrowClient = jest.mocked(EscrowClient);
+
+describe('CvatPayoutsCalculator', () => {
+  let calculator: CvatPayoutsCalculator;
+
+  beforeAll(async () => {
+    const moduleRef = await Test.createTestingModule({
+      providers: [
+        CvatPayoutsCalculator,
+        {
+          provide: StorageService,
+          useValue: mockedStorageService,
+        },
+        {
+          provide: Web3Service,
+          useValue: mockedWeb3Service,
+        },
+      ],
+    }).compile();
+
+    calculator = moduleRef.get<CvatPayoutsCalculator>(CvatPayoutsCalculator);
+  });
+
+  describe('calculate', () => {
+    const chainId = generateTestnetChainId();
+    const escrowAddress = faker.finance.ethereumAddress();
+
+    const mockedGetIntermediateResultsUrl = jest
+      .fn()
+      .mockImplementation(async () => faker.internet.url());
+
+    beforeAll(() => {
+      mockedEscrowClient.build.mockResolvedValue({
+        getIntermediateResultsUrl: mockedGetIntermediateResultsUrl,
+      } as unknown as EscrowClient);
+    });
+
+    it('throws when invalid annotation meta downloaded from valid url', async () => {
+      const intermediateResultsUrl = faker.internet.url();
+      mockedGetIntermediateResultsUrl.mockResolvedValueOnce(
+        intermediateResultsUrl,
+      );
+
+      mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce({
+        jobs: [],
+        results: [],
+      } as CvatAnnotationMeta);
+
+      await expect(
+        calculator.calculate({
+          finalResultsUrl: faker.internet.url(),
+          chainId,
+          escrowAddress,
+          manifest: generateCvatManifest(),
+        }),
+      ).rejects.toThrow('Invalid annotation meta');
+
+      expect(mockedStorageService.downloadJsonLikeData).toHaveBeenCalledWith(
+        `${intermediateResultsUrl}/validation_meta.json`,
+      );
+    });
+
+    it('should properly calculate workers bounties', async () => {
+      const annotators = [
+        faker.finance.ethereumAddress(),
+        faker.finance.ethereumAddress(),
+      ];
+
+      const jobsPerAnnotator = faker.number.int({ min: 1, max: 3 });
+
+      const annotationsMeta: CvatAnnotationMeta = {
+        jobs: Array.from(
+          { length: jobsPerAnnotator * annotators.length },
+          (_v, index: number) => ({
+            job_id: index,
+            final_result_id: faker.number.int(),
+          }),
+        ),
+        results: [],
+      };
+      for (const job of annotationsMeta.jobs) {
+        const annotatorIndex = job.job_id % annotators.length;
+
+        annotationsMeta.results.push({
+          id: job.final_result_id,
+          job_id: job.job_id,
+          annotator_wallet_address: annotators[annotatorIndex],
+          annotation_quality: faker.number.float(),
+        });
+      }
+
+      // imitate weird case: job w/o result
+      annotationsMeta.jobs.push({
+        job_id: faker.number.int(),
+        final_result_id: faker.number.int(),
+      });
+      // imitate weird case: result w/o job
+      annotationsMeta.results.push({
+        id: faker.number.int(),
+        job_id: faker.number.int(),
+        annotator_wallet_address: faker.helpers.arrayElement(annotators),
+        annotation_quality: faker.number.float(),
+      });
+
+      mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce(
+        annotationsMeta,
+      );
+
+      const manifest = generateCvatManifest();
+
+      const payouts = await calculator.calculate({
+        chainId,
+        escrowAddress,
+        manifest,
+        finalResultsUrl: faker.internet.url(),
+      });
+
+      const expectedAmountPerAnnotator =
+        BigInt(jobsPerAnnotator) * ethers.parseUnits(manifest.job_bounty, 18);
+
+      const expectedPayouts = annotators.map((address) => ({
+        address,
+        amount: expectedAmountPerAnnotator,
+      }));
+
+      expect(_.sortBy(payouts, 'address')).toEqual(
+        _.sortBy(expectedPayouts, 'address'),
+      );
+    });
+  });
+});

packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts

@@ -43,7 +43,7 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator {
         `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`,
       );
 
-    if (annotations.jobs.length === 0 || annotations.results.length === 0) {
+    if (!annotations.jobs.length || !annotations.results.length) {
       throw new Error('Invalid annotation meta');
     }
 

packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/fortune-payouts-calculator.spec.ts

@@ -0,0 +1,75 @@
+import { createMock } from '@golevelup/ts-jest';
+import { faker } from '@faker-js/faker';
+import { Test } from '@nestjs/testing';
+import { ethers } from 'ethers';
+import _ from 'lodash';
+
+import { StorageService } from '../../storage/storage.service';
+
+import { generateFortuneManifest, generateFortuneSolution } from '../fixtures';
+import { FortunePayoutsCalculator } from './fortune-payouts-calculator';
+
+const mockedStorageService = createMock<StorageService>();
+
+describe('FortunePayoutsCalculator', () => {
+  let calculator: FortunePayoutsCalculator;
+
+  beforeAll(async () => {
+    const moduleRef = await Test.createTestingModule({
+      providers: [
+        FortunePayoutsCalculator,
+        {
+          provide: StorageService,
+          useValue: mockedStorageService,
+        },
+      ],
+    }).compile();
+
+    calculator = moduleRef.get<FortunePayoutsCalculator>(
+      FortunePayoutsCalculator,
+    );
+  });
+
+  describe('calculate', () => {
+    it('should properly calculate payouts', async () => {
+      const validSolutions = [
+        generateFortuneSolution(),
+        generateFortuneSolution(),
+      ];
+      const results = faker.helpers.shuffle([
+        ...validSolutions,
+        generateFortuneSolution('curse_word'),
+        generateFortuneSolution('duplicated'),
+        generateFortuneSolution(faker.string.sample()),
+      ]);
+      const resultsUrl = faker.internet.url();
+      mockedStorageService.downloadJsonLikeData.mockImplementationOnce(
+        async (url) => {
+          if (url === resultsUrl) {
+            return results;
+          }
+          throw new Error('Results not found');
+        },
+      );
+      const manifest = generateFortuneManifest();
+
+      const payouts = await calculator.calculate({
+        chainId: faker.number.int(),
+        escrowAddress: faker.finance.ethereumAddress(),
+        finalResultsUrl: resultsUrl,
+        manifest,
+      });
+
+      const expectedPayouts = validSolutions.map((s) => ({
+        address: s.workerAddress,
+        amount:
+          BigInt(ethers.parseUnits(manifest.fundAmount.toString(), 'ether')) /
+          BigInt(validSolutions.length),
+      }));
+
+      expect(_.sortBy(payouts, 'address')).toEqual(
+        _.sortBy(expectedPayouts, 'address'),
+      );
+    });
+  });
+});

packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/fortune-results-processor.spec.ts

@@ -2,30 +2,20 @@ import { createMock } from '@golevelup/ts-jest';
 import { faker } from '@faker-js/faker';
 import { Test } from '@nestjs/testing';
 
-import { FortuneJobType } from '../../../common/enums';
-import { FortuneFinalResult, FortuneManifest } from '../../../common/types';
+import { FortuneFinalResult } from '../../../common/types';
 
 import { PgpEncryptionService } from '../../encryption/pgp-encryption.service';
 import { StorageService } from '../../storage/storage.service';
 import { Web3Service } from '../../web3/web3.service';
 
+import { generateFortuneManifest, generateFortuneSolution } from '../fixtures';
 import { BaseEscrowResultsProcessor } from './escrow-results-processor';
 import { FortuneResultsProcessor } from './fortune-results-processor';
 
 const mockedStorageService = createMock<StorageService>();
 const mockedPgpEncryptionService = createMock<PgpEncryptionService>();
 const mockedWeb3Service = createMock<Web3Service>();
 
-function generateFortuneSolution(
-  error?: FortuneFinalResult['error'],
-): FortuneFinalResult {
-  return {
-    workerAddress: faker.finance.ethereumAddress(),
-    solution: faker.string.sample(),
-    error,
-  };
-}
-
 describe('FortuneResultsProcessor', () => {
   let processor: FortuneResultsProcessor;
 
@@ -66,11 +56,7 @@ describe('FortuneResultsProcessor', () => {
   });
 
   describe('assertResultsComplete', () => {
-    const testManifest: FortuneManifest = {
-      requestType: FortuneJobType.FORTUNE,
-      fundAmount: Number(faker.finance.amount()),
-      submissionsRequired: faker.number.int({ min: 2, max: 5 }),
-    };
+    const testManifest = generateFortuneManifest();
 
     it('throws if results is not json', async () => {
       await expect(
@@ -105,11 +91,7 @@ describe('FortuneResultsProcessor', () => {
         () => generateFortuneSolution(),
       );
       solutions.pop();
-      solutions.push(
-        generateFortuneSolution(
-          faker.helpers.arrayElement(['curse_word', 'duplicated']),
-        ),
-      );
+      solutions.push(generateFortuneSolution(faker.string.sample()));
 
       await expect(
         processor['assertResultsComplete'](
@@ -123,7 +105,9 @@ describe('FortuneResultsProcessor', () => {
       const solutions: FortuneFinalResult[] = Array.from(
         { length: testManifest.submissionsRequired * 2 },
         (i: number) =>
-          generateFortuneSolution(i % 2 === 0 ? 'duplicated' : undefined),
+          generateFortuneSolution(
+            i % 2 === 0 ? faker.string.sample() : undefined,
+          ),
       );
 
       await expect(