Skip to content

Commit 8f2fb40

Browse files
committed
fix(orchestrator): Preserve post-interrupt recovery state
1 parent 3ec80f3 commit 8f2fb40

10 files changed

Lines changed: 4358 additions & 283 deletions

File tree

apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts

Lines changed: 161 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1122,14 +1122,19 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11221122
uuid: "00000000-0000-4000-8000-000000000101",
11231123
session_id: WAKE_NATIVE_SESSION,
11241124
});
1125-
const makeResultFrame = (input: { readonly uuid: string; readonly result: string }) =>
1125+
const makeResultFrame = (input: {
1126+
readonly uuid: string;
1127+
readonly result: string;
1128+
readonly numTurns?: number;
1129+
readonly origin?: { readonly kind: "task-notification" };
1130+
}) =>
11261131
claudeSdkFrame({
11271132
type: "result",
11281133
subtype: "success",
11291134
duration_ms: 10,
11301135
duration_api_ms: 10,
11311136
is_error: false,
1132-
num_turns: 1,
1137+
num_turns: input.numTurns ?? 1,
11331138
result: input.result,
11341139
stop_reason: "end_turn",
11351140
total_cost_usd: 0,
@@ -1143,6 +1148,7 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11431148
permission_denials: [],
11441149
uuid: input.uuid,
11451150
session_id: WAKE_NATIVE_SESSION,
1151+
...(input.origin === undefined ? {} : { origin: input.origin }),
11461152
});
11471153
const turnOneResult = makeResultFrame({
11481154
uuid: "00000000-0000-4000-8000-000000000102",
@@ -1161,6 +1167,15 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11611167
const wakeResult = makeResultFrame({
11621168
uuid: "00000000-0000-4000-8000-000000000104",
11631169
result: WAKE_RESULT_TEXT,
1170+
origin: { kind: "task-notification" },
1171+
});
1172+
const STALE_TASK_NOTIFICATION_RESULT_TEXT =
1173+
"Stale task-notification origin text that must not appear.";
1174+
const staleTaskNotificationResult = makeResultFrame({
1175+
uuid: "00000000-0000-4000-8000-000000000106",
1176+
result: STALE_TASK_NOTIFICATION_RESULT_TEXT,
1177+
numTurns: 0,
1178+
origin: { kind: "task-notification" },
11641179
});
11651180

11661181
const awaitUntil = (predicate: () => boolean, label: string): Effect.Effect<void> =>
@@ -1441,6 +1456,150 @@ describe("ClaudeAdapterV2 background wake turns", () => {
14411456
),
14421457
);
14431458

1459+
it.effect("ignores a live task-notification origin result during a normal user turn", () =>
1460+
Effect.scoped(
1461+
Effect.gen(function* () {
1462+
const harness = yield* makeWakeHarness;
1463+
const now = yield* DateTime.now;
1464+
const probeAssistantText = "Probe after stale task-notification result.";
1465+
const recoveryAssistantText = "Recovered after the interrupt; continuing.";
1466+
const staleResultText = STALE_TASK_NOTIFICATION_RESULT_TEXT;
1467+
const hasMessageText = (text: string) =>
1468+
harness.events.some(
1469+
(event) => event.type === "message.updated" && event.message.text === text,
1470+
);
1471+
1472+
yield* harness.runtime.startTurn(
1473+
makeClaudeTestTurnInput({
1474+
threadId: harness.threadId,
1475+
providerThread: harness.providerThread,
1476+
now,
1477+
attemptId: RunAttemptId.make("attempt-claude-stale-notif-1"),
1478+
text: "Continue after interrupt.",
1479+
attachments: [],
1480+
}),
1481+
);
1482+
yield* awaitUntil(() => harness.offeredMessages.length === 1, "recovery prompt offered");
1483+
1484+
// Live interleaving seen after interrupt recovery: a stale stopped
1485+
// task_notification and its task-notification-origin result arrive
1486+
// before the real root assistant stream.
1487+
yield* Queue.offer(
1488+
harness.sdkMessages,
1489+
claudeSdkFrame({
1490+
type: "system",
1491+
subtype: "task_notification",
1492+
task_id: "task-stale-stopped",
1493+
status: "stopped",
1494+
output_file: "/tmp/task-stale-stopped.log",
1495+
summary: "",
1496+
uuid: "00000000-0000-4000-8000-000000000107",
1497+
session_id: WAKE_NATIVE_SESSION,
1498+
}),
1499+
);
1500+
yield* Queue.offer(harness.sdkMessages, staleTaskNotificationResult);
1501+
// Queue-ordered probe: once this assistant text is emitted, the stale
1502+
// origin result ahead of it has been consumed.
1503+
yield* Queue.offer(
1504+
harness.sdkMessages,
1505+
claudeSdkFrame({
1506+
type: "assistant",
1507+
message: {
1508+
role: "assistant",
1509+
content: [{ type: "text", text: probeAssistantText }],
1510+
},
1511+
parent_tool_use_id: null,
1512+
uuid: "00000000-0000-4000-8000-00000000010a",
1513+
session_id: WAKE_NATIVE_SESSION,
1514+
}),
1515+
);
1516+
1517+
yield* awaitUntil(
1518+
() => hasMessageText(probeAssistantText),
1519+
"probe assistant after stale task-notification result",
1520+
);
1521+
assert.lengthOf(harness.terminalEvents(), 0);
1522+
assert.isFalse(hasMessageText(staleResultText));
1523+
1524+
yield* Queue.offer(
1525+
harness.sdkMessages,
1526+
claudeSdkFrame({
1527+
type: "assistant",
1528+
message: {
1529+
role: "assistant",
1530+
content: [{ type: "text", text: recoveryAssistantText }],
1531+
},
1532+
parent_tool_use_id: null,
1533+
uuid: "00000000-0000-4000-8000-000000000108",
1534+
session_id: WAKE_NATIVE_SESSION,
1535+
}),
1536+
);
1537+
yield* Queue.offer(
1538+
harness.sdkMessages,
1539+
makeResultFrame({
1540+
uuid: "00000000-0000-4000-8000-000000000109",
1541+
result: recoveryAssistantText,
1542+
}),
1543+
);
1544+
1545+
yield* awaitUntil(() => harness.terminalEvents().length === 1, "user turn terminal");
1546+
assert.equal(harness.terminalEvents()[0]?.status, "completed");
1547+
assert.isTrue(hasMessageText(recoveryAssistantText));
1548+
assert.isFalse(hasMessageText(staleResultText));
1549+
}).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))),
1550+
),
1551+
);
1552+
1553+
it.effect("terminalizes a continuation turn from a task-notification origin wake result", () =>
1554+
Effect.scoped(
1555+
Effect.gen(function* () {
1556+
const harness = yield* makeWakeHarness;
1557+
const now = yield* DateTime.now;
1558+
1559+
yield* harness.runtime.startTurn(
1560+
makeClaudeTestTurnInput({
1561+
threadId: harness.threadId,
1562+
providerThread: harness.providerThread,
1563+
now,
1564+
attemptId: RunAttemptId.make("attempt-claude-notif-origin-2a"),
1565+
text: "Run the build in the background.",
1566+
attachments: [],
1567+
}),
1568+
);
1569+
yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
1570+
yield* Queue.offer(harness.sdkMessages, turnOneResult);
1571+
yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal");
1572+
yield* Queue.offer(harness.sdkMessages, wakeNotification);
1573+
yield* Queue.offer(harness.sdkMessages, wakeResult);
1574+
yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request");
1575+
1576+
yield* harness.runtime.startTurn(
1577+
makeClaudeTestTurnInput({
1578+
threadId: harness.threadId,
1579+
providerThread: harness.providerThread,
1580+
now,
1581+
attemptId: RunAttemptId.make("attempt-claude-notif-origin-2b"),
1582+
text: "Background task completed.",
1583+
attachments: [],
1584+
providerTurnOrdinal: 2,
1585+
messageCreatedBy: "agent",
1586+
messageCreationSource: "provider",
1587+
}),
1588+
);
1589+
1590+
yield* awaitUntil(() => harness.terminalEvents().length === 2, "continuation terminal");
1591+
assert.equal(harness.terminalEvents()[1]?.status, "completed");
1592+
assert.lengthOf(harness.offeredMessages, 1);
1593+
assert.isTrue(
1594+
harness.events.some(
1595+
(event) => event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT,
1596+
),
1597+
);
1598+
assert.isFalse(yield* harness.hasPendingBackgroundWork);
1599+
}).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))),
1600+
),
1601+
);
1602+
14441603
it.effect("clears the pending task when the wake notification carries no summary", () =>
14451604
Effect.scoped(
14461605
Effect.gen(function* () {

apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1765,6 +1765,19 @@ function isClaudeActiveSteeringAbortResult(message: SDKResultMessage): boolean {
17651765
return message.terminal_reason === "aborted_streaming";
17661766
}
17671767

1768+
function isClaudeProviderContinuationTurn(input: ProviderAdapterV2TurnInput): boolean {
1769+
return input.message.createdBy === "agent" && input.message.creationSource === "provider";
1770+
}
1771+
1772+
function isClaudeTaskNotificationOriginResult(message: SDKMessage): message is SDKResultMessage & {
1773+
readonly origin: Extract<
1774+
NonNullable<SDKResultMessage["origin"]>,
1775+
{ readonly kind: "task-notification" }
1776+
>;
1777+
} {
1778+
return message.type === "result" && message.origin?.kind === "task-notification";
1779+
}
1780+
17681781
function providerFailureFromResult(
17691782
message: SDKResultMessage,
17701783
): OrchestrationV2ProviderFailure | null {
@@ -3266,6 +3279,18 @@ export function makeClaudeAdapterV2(
32663279
return;
32673280
}
32683281

3282+
// Task-notification-origin results can interleave during a normal
3283+
// user turn (for example a stale background stop after interrupt
3284+
// recovery). They must not finalize that turn or supply fallback
3285+
// assistant text. Provider continuation turns still consume them
3286+
// when draining buffered wake messages.
3287+
if (
3288+
isClaudeTaskNotificationOriginResult(message) &&
3289+
!isClaudeProviderContinuationTurn(context.input)
3290+
) {
3291+
return;
3292+
}
3293+
32693294
// An is_error result's text is the error message; it belongs on the
32703295
// terminal-failure item, not on a synthetic assistant message.
32713296
const resultText =
@@ -3549,9 +3574,7 @@ export function makeClaudeAdapterV2(
35493574
// produced instead of prompting it again: drain the buffered wake
35503575
// messages into this turn and let any still-streaming messages
35513576
// follow live. The continuation prompt text never reaches the CLI.
3552-
const isContinuationTurn =
3553-
turnInput.message.createdBy === "agent" &&
3554-
turnInput.message.creationSource === "provider";
3577+
const isContinuationTurn = isClaudeProviderContinuationTurn(turnInput);
35553578
const userMessage = isContinuationTurn
35563579
? null
35573580
: yield* makeClaudeUserMessageWithAttachments({

0 commit comments

Comments
 (0)