Skip to content

Commit bcc49cc

Browse files
committed
Merge branch 'develop' into feat/job-launcher-server/abuse
2 parents fc8d680 + 903e0e1 commit bcc49cc

File tree

299 files changed

+10873
-6685
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

299 files changed

+10873
-6685
lines changed

.github/workflows/ci-dependency-review.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@ jobs:
1111
- name: "Checkout Repository"
1212
uses: actions/[email protected]
1313
- name: "Dependency Review"
14-
uses: actions/dependency-review-action@v4.4.0
14+
uses: actions/dependency-review-action@v4.5.0

packages/apps/dashboard/server/.env.example

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ HMT_PRICE_SOURCE_API_KEY=
1111
HMT_PRICE_FROM=
1212
HMT_PRICE_TO=
1313
HCAPTCHA_API_KEY=
14+
NETWORK_USAGE_FILTER_MONTHS=
15+
NETWORKS_AVAILABLE_CACHE_TTL=
1416

1517
# Redis
1618
REDIS_HOST=localhost

packages/apps/dashboard/server/.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,7 @@ lerna-debug.log*
3535
!.vscode/extensions.json
3636

3737
# Redis Data
38-
/redis_data
38+
/redis_data
39+
40+
.env.development
41+
.env.production

packages/apps/dashboard/server/docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: '3.8'
1+
name: 'dashboard-server'
22

33
services:
44
redis:
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
module.exports = {
2+
coverageDirectory: '../coverage',
3+
collectCoverageFrom: ['**/*.(t|j)s'],
4+
moduleFileExtensions: ['js', 'json', 'ts'],
5+
rootDir: 'src',
6+
testEnvironment: 'node',
7+
testRegex: '.*\\.spec\\.ts$',
8+
transform: {
9+
'^.+\\.(t)s$': 'ts-jest',
10+
},
11+
moduleNameMapper: {
12+
'^uuid$': require.resolve('uuid'),
13+
'^typeorm$': require.resolve('typeorm'),
14+
},
15+
};

packages/apps/dashboard/server/package.json

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,26 @@
1111
"start": "nest start",
1212
"start:dev": "nest start --watch",
1313
"start:debug": "nest start --debug --watch",
14-
"start:prod": "node dist/main",
15-
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix"
14+
"start:prod": "node dist/src/main",
15+
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
16+
"test": "jest",
17+
"test:watch": "jest --watch",
18+
"test:cov": "jest --coverage",
19+
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand"
1620
},
1721
"dependencies": {
1822
"@human-protocol/sdk": "*",
19-
"@nestjs/axios": "^3.0.2",
23+
"@nestjs/axios": "^3.1.2",
2024
"@nestjs/cache-manager": "^2.2.2",
2125
"@nestjs/common": "^10.2.7",
2226
"@nestjs/config": "^3.2.3",
2327
"@nestjs/core": "^10.2.8",
2428
"@nestjs/mapped-types": "*",
2529
"@nestjs/platform-express": "^10.3.10",
26-
"cache-manager-redis-store": "^3.0.1",
30+
"cache-manager": "^5.4.0",
31+
"cache-manager-redis-yet": "^5.1.5",
2732
"dayjs": "^1.11.12",
33+
"lodash": "^4.17.21",
2834
"reflect-metadata": "^0.2.2",
2935
"rxjs": "^7.2.0"
3036
},

packages/apps/dashboard/server/src/app.module.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,12 @@ import { StatsModule } from './modules/stats/stats.module';
2121
validationSchema: Joi.object({
2222
HOST: Joi.string().required(),
2323
PORT: Joi.number().port().default(3000),
24+
REDIS_HOST: Joi.string(),
25+
REDIS_PORT: Joi.number(),
2426
SUBGRAPH_API_KEY: Joi.string().required(),
2527
HCAPTCHA_API_KEY: Joi.string().required(),
28+
CACHE_HMT_PRICE_TTL: Joi.number(),
29+
CACHE_HMT_GENERAL_STATS_TTL: Joi.number(),
2630
}),
2731
}),
2832
CacheModule.registerAsync(CacheFactoryConfig),

packages/apps/dashboard/server/src/common/config/cache-factory.config.ts

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,15 @@
11
import { CacheModuleAsyncOptions } from '@nestjs/common/cache';
22
import { ConfigModule } from '@nestjs/config';
3+
import * as _ from 'lodash';
34
import { RedisConfigService } from './redis-config.service';
4-
import { redisStore } from 'cache-manager-redis-store';
5+
import { redisStore } from 'cache-manager-redis-yet';
6+
import { Logger } from '@nestjs/common';
7+
8+
const logger = new Logger('CacheFactoryRedisStore');
9+
10+
const throttledRedisErrorLog = _.throttle((error) => {
11+
logger.error('Redis client network error', error);
12+
}, 1000 * 5);
513

614
export const CacheFactoryConfig: CacheModuleAsyncOptions = {
715
isGlobal: true,
@@ -12,7 +20,11 @@ export const CacheFactoryConfig: CacheModuleAsyncOptions = {
1220
host: configService.redisHost,
1321
port: configService.redisPort,
1422
},
23+
disableOfflineQueue: true,
1524
});
25+
26+
store.client.on('error', throttledRedisErrorLog);
27+
1628
return {
1729
store: () => store,
1830
};

packages/apps/dashboard/server/src/common/config/env-config.service.ts

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ const DEFAULT_HCAPTCHA_STATS_FILE = 'hcaptchaStats.json';
1414
export const HCAPTCHA_STATS_START_DATE = '2022-07-01';
1515
export const HCAPTCHA_STATS_API_START_DATE = '2024-09-14';
1616
export const HMT_STATS_START_DATE = '2021-04-06';
17+
export const MINIMUM_HMT_TRANSFERS = 5;
18+
export const DEFAULT_NETWORK_USAGE_FILTER_MONTHS = 1;
19+
export const DEFAULT_NETWORKS_AVAILABLE_CACHE_TTL = 2 * 60;
20+
export const MINIMUM_ESCROWS_COUNT = 1;
1721

1822
@Injectable()
1923
export class EnvironmentConfigService {
@@ -81,4 +85,18 @@ export class EnvironmentConfigService {
8185
get reputationSource(): string {
8286
return this.configService.getOrThrow<string>('REPUTATION_SOURCE_URL');
8387
}
88+
89+
get networkUsageFilterMonths(): number {
90+
return this.configService.get<number>(
91+
'NETWORK_USAGE_FILTER_MONTHS',
92+
DEFAULT_NETWORK_USAGE_FILTER_MONTHS,
93+
);
94+
}
95+
96+
get networkAvailableCacheTtl(): number {
97+
return this.configService.get<number>(
98+
'NETWORKS_AVAILABLE_CACHE_TTL',
99+
DEFAULT_NETWORKS_AVAILABLE_CACHE_TTL,
100+
);
101+
}
84102
}
Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
import { createMock } from '@golevelup/ts-jest';
2+
import { Test, TestingModule } from '@nestjs/testing';
3+
import { CACHE_MANAGER } from '@nestjs/cache-manager';
4+
import { Cache } from 'cache-manager';
5+
import { Logger } from '@nestjs/common';
6+
import { StatisticsClient } from '@human-protocol/sdk';
7+
import { EnvironmentConfigService } from '../../common/config/env-config.service';
8+
import { ChainId, NETWORKS } from '@human-protocol/sdk';
9+
import { MainnetsId } from '../../common/utils/constants';
10+
import { HttpService } from '@nestjs/axios';
11+
import { NetworkConfigService } from './network-config.service';
12+
import { ConfigService } from '@nestjs/config';
13+
14+
jest.mock('@human-protocol/sdk', () => ({
15+
...jest.requireActual('@human-protocol/sdk'),
16+
StatisticsClient: jest.fn(),
17+
}));
18+
19+
describe('NetworkConfigService', () => {
20+
let networkConfigService: NetworkConfigService;
21+
let cacheManager: Cache;
22+
23+
beforeEach(async () => {
24+
const module: TestingModule = await Test.createTestingModule({
25+
providers: [
26+
NetworkConfigService,
27+
{ provide: HttpService, useValue: createMock<HttpService>() },
28+
{
29+
provide: CACHE_MANAGER,
30+
useValue: {
31+
get: jest.fn(),
32+
set: jest.fn(),
33+
},
34+
},
35+
{
36+
provide: EnvironmentConfigService,
37+
useValue: {
38+
networkUsageFilterMonths: 3,
39+
networkAvailableCacheTtl: 1000,
40+
},
41+
},
42+
ConfigService,
43+
Logger,
44+
],
45+
}).compile();
46+
47+
networkConfigService =
48+
module.get<NetworkConfigService>(NetworkConfigService);
49+
cacheManager = module.get<Cache>(CACHE_MANAGER);
50+
});
51+
52+
it('should regenerate network list when cache TTL expires', async () => {
53+
const mockNetworkList = [
54+
ChainId.MAINNET,
55+
ChainId.BSC_MAINNET,
56+
ChainId.POLYGON,
57+
ChainId.XLAYER,
58+
ChainId.MOONBEAM,
59+
ChainId.CELO,
60+
ChainId.AVALANCHE,
61+
];
62+
63+
// Step 1: Initial request - populate cache
64+
jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
65+
jest.spyOn(cacheManager, 'set').mockResolvedValue(undefined);
66+
67+
const mockStatisticsClient = {
68+
getHMTDailyData: jest
69+
.fn()
70+
.mockResolvedValue([{ totalTransactionCount: 7 }]),
71+
getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
72+
};
73+
(StatisticsClient as jest.Mock).mockImplementation(
74+
() => mockStatisticsClient,
75+
);
76+
77+
// First call should populate cache
78+
const firstCallResult = await networkConfigService.getAvailableNetworks();
79+
80+
expect(firstCallResult).toEqual(mockNetworkList);
81+
expect(cacheManager.set).toHaveBeenCalledWith(
82+
'available-networks',
83+
mockNetworkList,
84+
1000,
85+
);
86+
87+
// Step 2: Simulate TTL expiration by returning null from cache
88+
jest.spyOn(cacheManager, 'get').mockResolvedValueOnce(null);
89+
90+
// Second call after TTL should re-generate the network list
91+
const secondCallResult = await networkConfigService.getAvailableNetworks();
92+
expect(secondCallResult).toEqual(mockNetworkList);
93+
94+
// Ensure the cache is set again with the regenerated network list
95+
expect(cacheManager.set).toHaveBeenCalledWith(
96+
'available-networks',
97+
mockNetworkList,
98+
1000,
99+
);
100+
});
101+
102+
it('should return cached networks if available', async () => {
103+
const cachedNetworks = [ChainId.MAINNET, ChainId.POLYGON];
104+
jest.spyOn(cacheManager, 'get').mockResolvedValue(cachedNetworks);
105+
106+
const result = await networkConfigService.getAvailableNetworks();
107+
expect(result).toEqual(cachedNetworks);
108+
expect(cacheManager.get).toHaveBeenCalledWith('available-networks');
109+
});
110+
111+
it('should fetch and filter available networks correctly', async () => {
112+
jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
113+
const mockStatisticsClient = {
114+
getHMTDailyData: jest
115+
.fn()
116+
.mockResolvedValue([
117+
{ totalTransactionCount: 4 },
118+
{ totalTransactionCount: 3 },
119+
]),
120+
getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
121+
};
122+
(StatisticsClient as jest.Mock).mockImplementation(
123+
() => mockStatisticsClient,
124+
);
125+
126+
const result = await networkConfigService.getAvailableNetworks();
127+
expect(result).toEqual(
128+
expect.arrayContaining([ChainId.MAINNET, ChainId.POLYGON]),
129+
);
130+
131+
expect(cacheManager.set).toHaveBeenCalledWith(
132+
'available-networks',
133+
result,
134+
1000,
135+
);
136+
});
137+
138+
it('should exclude networks without sufficient HMT transfers', async () => {
139+
jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
140+
const mockStatisticsClient = {
141+
getHMTDailyData: jest
142+
.fn()
143+
.mockResolvedValue([{ totalTransactionCount: 2 }]),
144+
getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
145+
};
146+
(StatisticsClient as jest.Mock).mockImplementation(
147+
() => mockStatisticsClient,
148+
);
149+
150+
const result = await networkConfigService.getAvailableNetworks();
151+
expect(result).toEqual([]);
152+
});
153+
154+
it('should handle missing network configuration gracefully', async () => {
155+
jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
156+
157+
const originalNetworkConfig = NETWORKS[MainnetsId.MAINNET];
158+
NETWORKS[MainnetsId.MAINNET] = undefined;
159+
160+
const mockStatisticsClient = {
161+
getHMTDailyData: jest
162+
.fn()
163+
.mockResolvedValue([
164+
{ totalTransactionCount: 3 },
165+
{ totalTransactionCount: 3 },
166+
]),
167+
getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
168+
};
169+
(StatisticsClient as jest.Mock).mockImplementation(
170+
() => mockStatisticsClient,
171+
);
172+
173+
const result = await networkConfigService.getAvailableNetworks();
174+
175+
expect(result).not.toContain(MainnetsId.MAINNET);
176+
expect(result).toEqual(expect.arrayContaining([]));
177+
178+
NETWORKS[MainnetsId.MAINNET] = originalNetworkConfig;
179+
});
180+
181+
it('should handle errors in getHMTDailyData gracefully', async () => {
182+
jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
183+
const mockStatisticsClient = {
184+
getHMTDailyData: jest
185+
.fn()
186+
.mockRejectedValue(new Error('Failed to fetch HMT data')),
187+
getEscrowStatistics: jest.fn().mockResolvedValue({ totalEscrows: 1 }),
188+
};
189+
(StatisticsClient as jest.Mock).mockImplementation(
190+
() => mockStatisticsClient,
191+
);
192+
193+
const result = await networkConfigService.getAvailableNetworks();
194+
expect(result).toEqual([]);
195+
});
196+
197+
it('should handle errors in getEscrowStatistics gracefully', async () => {
198+
jest.spyOn(cacheManager, 'get').mockResolvedValue(null);
199+
const mockStatisticsClient = {
200+
getHMTDailyData: jest
201+
.fn()
202+
.mockResolvedValue([
203+
{ totalTransactionCount: 3 },
204+
{ totalTransactionCount: 2 },
205+
]),
206+
getEscrowStatistics: jest
207+
.fn()
208+
.mockRejectedValue(new Error('Failed to fetch escrow stats')),
209+
};
210+
(StatisticsClient as jest.Mock).mockImplementation(
211+
() => mockStatisticsClient,
212+
);
213+
214+
const result = await networkConfigService.getAvailableNetworks();
215+
expect(result).toEqual([]);
216+
});
217+
});

0 commit comments

Comments
 (0)