Skip to content

Commit 35f3639

Browse files
PavelPashovPavelPashov
authored
test: add E2E tests for Redis Enterprise maintenance timeout handling (#2)
1 parent 42d2a81 commit 35f3639

File tree

4 files changed

+257
-3
lines changed

4 files changed

+257
-3
lines changed

‎packages/client/lib/tests/test-scenario/fault-injector-client.ts

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,45 @@ export class FaultInjectorClient {
108108
throw new Error(`Timeout waiting for action ${actionId}`);
109109
}
110110

111+
/**
112+
* Triggers a migrate and bind action.
113+
* @param bdbId The database ID to target
114+
* @param clusterIndex The cluster index to migrate to
115+
* @returns The action status
116+
*/
117+
public async migrateAndBindAction({
118+
bdbId,
119+
clusterIndex,
120+
}: {
121+
bdbId: string | number;
122+
clusterIndex: string | number;
123+
}) {
124+
const bdbIdStr = String(bdbId);
125+
const clusterIndexStr = String(clusterIndex);
126+
127+
return this.triggerAction<{ action_id: string }>({
128+
type: "sequence_of_actions",
129+
parameters: {
130+
bdb_id: bdbIdStr,
131+
actions: [
132+
{
133+
type: "migrate",
134+
params: {
135+
cluster_index: clusterIndexStr,
136+
},
137+
},
138+
{
139+
type: "bind",
140+
params: {
141+
bdb_id: bdbIdStr,
142+
cluster_index: clusterIndexStr,
143+
},
144+
},
145+
],
146+
},
147+
});
148+
}
149+
111150
async #request<T>(
112151
method: string,
113152
path: string,

‎packages/client/lib/tests/test-scenario/push-notification.e2e.ts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,7 @@ describe("Push Notifications", () => {
4646
username: clientConfig.username,
4747
RESP: 3,
4848
maintPushNotifications: "auto",
49-
maintMovingEndpointType: "external-ip",
50-
maintRelaxedCommandTimeout: 10000,
51-
maintRelaxedSocketTimeout: 10000,
49+
maintMovingEndpointType: "auto",
5250
});
5351

5452
client.on("error", (err: Error) => {

‎packages/client/lib/tests/test-scenario/test-command-runner.ts

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import { randomUUID } from "node:crypto";
2+
import { setTimeout } from "node:timers/promises";
3+
import { createClient } from "../../..";
4+
5+
/**
6+
* Options for the `fireCommandsUntilStopSignal` method
7+
*/
8+
type FireCommandsUntilStopSignalOptions = {
9+
/**
10+
* Number of commands to fire in each batch
11+
*/
12+
batchSize: number;
13+
/**
14+
* Timeout between batches in milliseconds
15+
*/
16+
timeoutMs: number;
17+
/**
18+
* Function that creates the commands to be executed
19+
*/
20+
createCommands: (
21+
client: ReturnType<typeof createClient<any, any, any, any>>
22+
) => Array<() => Promise<unknown>>;
23+
};
24+
25+
export class TestCommandRunner {
26+
constructor(
27+
private client: ReturnType<typeof createClient<any, any, any, any>>
28+
) {}
29+
30+
private defaultOptions: FireCommandsUntilStopSignalOptions = {
31+
batchSize: 60,
32+
timeoutMs: 10,
33+
createCommands: (
34+
client: ReturnType<typeof createClient<any, any, any, any>>
35+
) => [
36+
() => client.set(randomUUID(), Date.now()),
37+
() => client.get(randomUUID()),
38+
],
39+
};
40+
41+
#toSettled<T>(p: Promise<T>) {
42+
return p
43+
.then((value) => ({ status: "fulfilled" as const, value, error: null }))
44+
.catch((reason) => ({
45+
status: "rejected" as const,
46+
value: null,
47+
error: reason,
48+
}));
49+
}
50+
51+
async #racePromises<S, T>({
52+
timeout,
53+
stopper,
54+
}: {
55+
timeout: Promise<S>;
56+
stopper: Promise<T>;
57+
}) {
58+
return Promise.race([
59+
this.#toSettled<S>(timeout).then((result) => ({
60+
...result,
61+
stop: false,
62+
})),
63+
this.#toSettled<T>(stopper).then((result) => ({ ...result, stop: true })),
64+
]);
65+
}
66+
67+
/**
68+
* Fires commands until a stop signal is received.
69+
* @param stopSignalPromise Promise that resolves when the command execution should stop
70+
* @param options Options for the command execution
71+
* @returns Promise that resolves when the stop signal is received
72+
*/
73+
async fireCommandsUntilStopSignal(
74+
stopSignalPromise: Promise<unknown>,
75+
options?: Partial<FireCommandsUntilStopSignalOptions>
76+
) {
77+
const executeOptions = {
78+
...this.defaultOptions,
79+
...options,
80+
};
81+
82+
const commandPromises = [];
83+
84+
while (true) {
85+
for (let i = 0; i < executeOptions.batchSize; i++) {
86+
for (const command of executeOptions.createCommands(this.client)) {
87+
commandPromises.push(this.#toSettled(command()));
88+
}
89+
}
90+
91+
const result = await this.#racePromises({
92+
timeout: setTimeout(executeOptions.timeoutMs),
93+
stopper: stopSignalPromise,
94+
});
95+
96+
if (result.stop) {
97+
return {
98+
commandPromises,
99+
stopResult: result,
100+
};
101+
}
102+
}
103+
}
104+
}

‎packages/client/lib/tests/test-scenario/timeout-during-notifications.e2e.ts

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import { FaultInjectorClient } from "./fault-injector-client";
2+
import {
3+
getDatabaseConfig,
4+
getDatabaseConfigFromEnv,
5+
getEnvConfig,
6+
RedisConnectionConfig,
7+
} from "./test-scenario.util";
8+
import { createClient } from "../../../dist";
9+
import { before } from "mocha";
10+
import { TestCommandRunner } from "./test-command-runner";
11+
import assert from "node:assert";
12+
13+
describe("Timeout Handling During Notifications", () => {
14+
let clientConfig: RedisConnectionConfig;
15+
let client: ReturnType<typeof createClient<any, any, any, 3>>;
16+
let faultInjectorClient: FaultInjectorClient;
17+
let commandRunner: TestCommandRunner;
18+
19+
before(() => {
20+
const envConfig = getEnvConfig();
21+
const redisConfig = getDatabaseConfigFromEnv(
22+
envConfig.redisEndpointsConfigPath
23+
);
24+
25+
faultInjectorClient = new FaultInjectorClient(envConfig.faultInjectorUrl);
26+
clientConfig = getDatabaseConfig(redisConfig);
27+
});
28+
29+
beforeEach(async () => {
30+
client = createClient({
31+
socket: {
32+
host: clientConfig.host,
33+
port: clientConfig.port,
34+
...(clientConfig.tls === true ? { tls: true } : {}),
35+
},
36+
password: clientConfig.password,
37+
username: clientConfig.username,
38+
RESP: 3,
39+
maintPushNotifications: "auto",
40+
maintMovingEndpointType: "auto",
41+
});
42+
43+
client.on("error", (err: Error) => {
44+
throw new Error(`Client error: ${err.message}`);
45+
});
46+
47+
commandRunner = new TestCommandRunner(client);
48+
49+
await client.connect();
50+
});
51+
52+
afterEach(() => {
53+
client.destroy();
54+
});
55+
56+
it("should relax command timeout on MOVING, MIGRATING, and MIGRATED", async () => {
57+
// PART 1
58+
// Set very low timeout to trigger errors
59+
client.options!.maintRelaxedCommandTimeout = 50;
60+
61+
const { action_id: lowTimeoutBindAndMigrateActionId } =
62+
await faultInjectorClient.migrateAndBindAction({
63+
bdbId: clientConfig.bdbId,
64+
clusterIndex: 0,
65+
});
66+
67+
const lowTimeoutWaitPromise = faultInjectorClient.waitForAction(
68+
lowTimeoutBindAndMigrateActionId
69+
);
70+
71+
const lowTimeoutCommandPromises =
72+
await commandRunner.fireCommandsUntilStopSignal(lowTimeoutWaitPromise);
73+
74+
const lowTimeoutRejectedCommands = (
75+
await Promise.all(lowTimeoutCommandPromises.commandPromises)
76+
).filter((result) => result.status === "rejected");
77+
78+
assert.ok(lowTimeoutRejectedCommands.length > 0);
79+
assert.strictEqual(
80+
lowTimeoutRejectedCommands.filter((rejected) => {
81+
return (
82+
// TODO instanceof doesn't work for some reason
83+
rejected.error.constructor.name ===
84+
"CommandTimeoutDuringMaintananceError"
85+
);
86+
}).length,
87+
lowTimeoutRejectedCommands.length
88+
);
89+
90+
// PART 2
91+
// Set high timeout to avoid errors
92+
client.options!.maintRelaxedCommandTimeout = 10000;
93+
94+
const { action_id: highTimeoutBindAndMigrateActionId } =
95+
await faultInjectorClient.migrateAndBindAction({
96+
bdbId: clientConfig.bdbId,
97+
clusterIndex: 0,
98+
});
99+
100+
const highTimeoutWaitPromise = faultInjectorClient.waitForAction(
101+
highTimeoutBindAndMigrateActionId
102+
);
103+
104+
const highTimeoutCommandPromises =
105+
await commandRunner.fireCommandsUntilStopSignal(highTimeoutWaitPromise);
106+
107+
const highTimeoutRejectedCommands = (
108+
await Promise.all(highTimeoutCommandPromises.commandPromises)
109+
).filter((result) => result.status === "rejected");
110+
111+
assert.strictEqual(highTimeoutRejectedCommands.length, 0);
112+
});
113+
});

0 commit comments

Comments
 (0)