Skip to content

concurrent sandbox execution #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kamath merged 1 commit into from
Dec 4, 2025
Merged

concurrent sandbox execution #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions apps/backend/src/problem-actions/src/concurrency.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
export const DEFAULT_CONCURRENCY = 100;

export function getConcurrency(
env: { SANDBOX_CONCURRENCY?: string } | Env,
): number {
const value = (env as { SANDBOX_CONCURRENCY?: string }).SANDBOX_CONCURRENCY;
if (typeof value === "string") {
const parsed = parseInt(value, 10);
if (!Number.isNaN(parsed)) {
return parsed;
}
}
return DEFAULT_CONCURRENCY;
}

export function pLimit(concurrency: number) {
const queue: (() => void)[] = [];
let activeCount = 0;

const next = () => {
activeCount--;
if (queue.length > 0) {
queue.shift()!();
}
};

return <T>(fn: () => Promise<T>): Promise<T> => {
return new Promise((resolve, reject) => {
const run = async () => {
activeCount++;
try {
resolve(await fn());
} catch (e) {
reject(e);
} finally {
next();
}
};

if (activeCount < concurrency) {
run();
} else {
queue.push(run);
}
});
};
}
36 changes: 21 additions & 15 deletions apps/backend/src/problem-actions/src/generate-test-case-inputs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
type Database,
} from "@repo/db";
import { getPostHogClient } from "@/utils/analytics";
import { pLimit, getConcurrency } from "./concurrency";

export async function generateTestCaseInputs(
problemId: string,
Expand All @@ -22,21 +23,26 @@ export async function generateTestCaseInputs(
);
}

const results: unknown[] = [];
for (const testCase of testCases) {
const result = await sandbox.run(
testCase.inputCode +
"; const output = generateTestInput();" +
"require('fs').writeFileSync('output.json', JSON.stringify(output));",
);
if (result.exitCode !== 0) {
throw new Error(
`Failed to generate test case input for test case ${testCase.id}: exitCode ${result.exitCode}`,
);
}
console.log("Result of running sandbox code:", result);
results.push(JSON.parse(await sandbox.readFile("output.json")));
}
const limit = pLimit(getConcurrency(env));
const results = await Promise.all(
testCases.map((testCase, index) =>
limit(async () => {
const outputFile = `output_${index}.json`;
const result = await sandbox.run(
testCase.inputCode +
"; const output = generateTestInput();" +
`require('fs').writeFileSync('${outputFile}', JSON.stringify(output));`,
);
if (result.exitCode !== 0) {
throw new Error(
`Failed to generate test case input for test case ${testCase.id}: exitCode ${result.exitCode}`,
);
}
console.log("Result of running sandbox code:", result);
return JSON.parse(await sandbox.readFile(outputFile));
}),
),
);

await sandbox.kill();

Expand Down
43 changes: 26 additions & 17 deletions apps/backend/src/problem-actions/src/generate-test-case-outputs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
type Database,
} from "@repo/db";
import { getPostHogClient } from "@/utils/analytics";
import { pLimit, getConcurrency } from "./concurrency";

/**
* Runs the reference solution on an arbitrary input and returns the expected output.
Expand All @@ -16,13 +17,15 @@ import { getPostHogClient } from "@/utils/analytics";
* @param input - The input to run the solution on (array of function arguments)
* @param sandbox - The sandbox instance to execute code in
* @param db - The database instance
* @param outputFileName - The output file name to write results to (for parallel execution)
* @returns The expected output value, or null if execution failed
*/
export async function runReferenceSolutionOnInput(
problemId: string,
input: unknown,
sandbox: Sandbox,
db: Database,
outputFileName: string = "output.json",
): Promise<unknown | null> {
try {
const solution = await getSolution(problemId, db);
Expand All @@ -35,9 +38,9 @@ export async function runReferenceSolutionOnInput(
"; const output = runSolution(..." +
JSON.stringify(input) +
");" +
"require('fs').writeFileSync('output.json', JSON.stringify(output));",
`require('fs').writeFileSync('${outputFileName}', JSON.stringify(output));`,
);
const outputContent = await sandbox.readFile("output.json");
const outputContent = await sandbox.readFile(outputFileName);
return JSON.parse(outputContent);
} catch {
// Return null on any error (execution failure, parse failure, etc.)
Expand All @@ -58,21 +61,27 @@ export async function generateTestCaseOutputs(
);
}

const results: unknown[] = [];
for (const testCase of testCases) {
const result = await runReferenceSolutionOnInput(
problemId,
testCase.input,
sandbox,
db,
);
if (result === null) {
throw new Error(
`Failed to generate result for test case ${testCase.id || "unknown"}`,
);
}
results.push(result);
}
const limit = pLimit(getConcurrency(env));
const results = await Promise.all(
testCases.map((testCase, index) =>
limit(async () => {
const outputFile = `output_${index}.json`;
const result = await runReferenceSolutionOnInput(
problemId,
testCase.input,
sandbox,
db,
outputFile,
);
if (result === null) {
throw new Error(
`Failed to generate result for test case ${testCase.id || "unknown"}`,
);
}
return result;
}),
),
);

await sandbox.kill();

Expand Down
Loading