diff --git a/packages/coding-agent/test/session-manager/build-context.test.ts b/packages/coding-agent/test/session-manager/build-context.test.ts index 4a99b504a..3896992b5 100644 --- a/packages/coding-agent/test/session-manager/build-context.test.ts +++ b/packages/coding-agent/test/session-manager/build-context.test.ts @@ -638,10 +638,12 @@ describe("buildSessionContext", () => { }); it("strips dangling tool_use when the leaf lands on a mid-batch assistant turn", () => { - // Reproduces the rewind/restore loop: leaf = an assistant turn that emitted - // tool calls. Its results are off-path children, so without normalization the - // turn ends on unpaired tool_use and transformMessages fabricates phantom - // "aborted" results, re-injecting the failed batch. + // Reproduces the rewind/restore loop: leaf = a completed assistant turn that + // emitted tool calls. Its results are off-path children, so without + // normalization the turn ends on unpaired tool_use and transformMessages + // fabricates phantom "aborted" results, re-injecting the failed batch. + // (Aborted/error turns are dropped from context wholesale — covered in + // src/session/session-context.test.ts.) const assistantWithCalls: SessionMessageEntry = { type: "message", id: "a1", @@ -668,7 +670,7 @@ describe("buildSessionContext", () => { totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }, - stopReason: "aborted", + stopReason: "toolUse", timestamp: 1, }, }; diff --git a/scripts/ci-test-ts.ts b/scripts/ci-test-ts.ts index 898c1fdca..5fceccf97 100755 --- a/scripts/ci-test-ts.ts +++ b/scripts/ci-test-ts.ts @@ -449,16 +449,25 @@ async function runTestCommand(testCommand: TestCommand): Promise { } const env = buildChildEnv(); - const proc = Bun.spawn(testCommand.command, { - cwd, - env, - stdout: "inherit", - stderr: "inherit", - }); - const killTimer = setTimeout(() => proc.kill("SIGKILL"), chunkTimeoutMs()); - const exitCode = await proc.exited; - clearTimeout(killTimer); - if (exitCode !== 0) { + for (let attempt = 1; ; attempt++) { + const proc = Bun.spawn(testCommand.command, { + cwd, + env, + stdout: "inherit", + stderr: "inherit", + }); + const killTimer = setTimeout(() => proc.kill("SIGKILL"), chunkTimeoutMs()); + const exitCode = await proc.exited; + clearTimeout(killTimer); + if (exitCode === 0) { + return; + } + if (BUN_CRASH_EXITS[exitCode] && attempt < MAX_CHUNK_ATTEMPTS) { + console.log( + `==> ${testCommand.label}: bun crashed (exit ${exitCode}); retrying (attempt ${attempt + 1}/${MAX_CHUNK_ATTEMPTS})`, + ); + continue; + } throw new Error(`${testCommand.label} failed with exit code ${exitCode}: ${renderedCommand}`); } } @@ -478,6 +487,13 @@ async function runTestCommand(testCommand: TestCommand): Promise { // bucket chunk crashed ~25% of runs with `BUN_JSC_forceRAMSize=256MB`, // 0/10 with markers=1, at zero measured wall-time cost. useConcurrentGC=0 // alone did not prevent it — the crash predates this knob. +// +// The knobs are necessary but not sufficient: the same GC bug also fires on +// the *mutator* thread (`Heap::collectInMutatorThread` → +// `MarkingConstraintSolver::runExecutionThread` → `DOMGCOutputConstraint` → +// `JSAbortSignal::visitAdditionalChildrenInGCThread` reading a dead `reason` +// cell), where no marker/concurrency knob applies. That residual crash is +// handled by retrying crashed chunks in a fresh process (MAX_CHUNK_ATTEMPTS). function buildChildEnv(): Record { const env: Record = { ...Bun.env, @@ -505,6 +521,28 @@ function chunkTimeoutMs(): number { return 600_000; } +// Exit codes that mean the bun process itself died to a runtime fault +// (128 + fatal signal) rather than reporting failing tests (which exit 1). +// Bun's panic handler exits via SIGTRAP (133) on macOS; raw SIGILL/SIGBUS/ +// SIGABRT/SIGFPE/SIGSEGV surface as 132/135|138/134/136/139. 137 (SIGKILL) +// is deliberately excluded: that's the watchdog or the OOM killer, and +// retrying an OOM-killed chunk just OOMs again. +const BUN_CRASH_EXITS: Record = { + 132: true, + 133: true, + 134: true, + 135: true, + 136: true, + 138: true, + 139: true, +}; + +// Total attempts a chunk gets when bun itself crashes. Bun 1.3.14's residual +// GC crash (JSAbortSignal `reason` visited on a dead cell during marking) is +// heap-timing dependent — a fresh process nearly always passes — while a +// deterministic crash still fails every attempt and is reported normally. +const MAX_CHUNK_ATTEMPTS = 3; + // The standard `CI` signal is authoritative. In CI each bucket is its own // memory-capped runner job (a single fat invocation gets OOM-killed at 137), so // chunks run sequentially within a job and parallelism happens across jobs. @@ -545,12 +583,14 @@ const style = { // Outcome of one finished chunk. `output` is the chunk's combined stdout+stderr, // buffered so it can be withheld during a quiet run and replayed only on failure. +// `retries` counts extra attempts spent on bun-crash exits (see BUN_CRASH_EXITS). interface ChunkOutcome { label: string; command: string; exitCode: number; seconds: number; output: string; + retries: number; } // Human duration in bun's bracket style: `[264ms]` under a second, `[3.3s]` @@ -567,14 +607,15 @@ function formatDuration(seconds: number): string { // order as each chunk finishes. export function formatProgressLine(outcome: ChunkOutcome): string { const time = style.dim(`[${formatDuration(outcome.seconds)}]`); + const retryNote = outcome.retries > 0 ? ` ${style.dim(`(retried ×${outcome.retries} after bun crash)`)}` : ""; if (outcome.exitCode === 0) { - return `${style.green("✓")} ${outcome.label} ${time}`; + return `${style.green("✓")} ${outcome.label} ${time}${retryNote}`; } const failing = extractFailingTests(outcome.output); const first = failing[0]?.name; const more = failing.length > 1 ? style.dim(` (+${failing.length - 1} more)`) : ""; const detail = first ? ` ${style.dim("—")} ${style.red(first)}${more}` : ""; - return `${style.bold(style.red(`✗ ${outcome.label}`))} ${time}${detail}`; + return `${style.bold(style.red(`✗ ${outcome.label}`))} ${time}${detail}${retryNote}`; } // Closing tally in `bun test` style, but counting test *chunks* (child commands), @@ -739,6 +780,44 @@ export async function runTestCommandsInParallel(commands: TestCommand[], concurr return settled; } + // One spawn of a chunk: pipes drained into buffers, watchdog-killed if it + // wedges, post-exit drain capped so leaked grandchildren holding the pipes + // can't keep the runner's event loop alive. + async function runAttempt( + testCommand: TestCommand, + ): Promise<{ exitCode: number; output: string; timedOut: boolean }> { + const proc = Bun.spawn(testCommand.command, { + cwd: path.join(repoRoot, testCommand.cwd), + env, + stdout: "pipe", + stderr: "pipe", + }); + const stdout = { text: "" }; + const stderr = { text: "" }; + const stdoutDrain = drainInto(proc.stdout as ReadableStream, stdout); + const stderrDrain = drainInto(proc.stderr as ReadableStream, stderr); + const drains = Promise.all([stdoutDrain.done, stderrDrain.done]); + // Watchdog: a wedged child (e.g. bun's panic handler deadlocking + // after a GC crash) would otherwise hang this worker forever. + let timedOut = false; + const killTimer = setTimeout(() => { + timedOut = true; + proc.kill("SIGKILL"); + }, chunkTimeoutMs()); + const exitCode = await proc.exited; + clearTimeout(killTimer); + if (!(await settleWithin(drains, 5000))) { + stdoutDrain.cancel(); + stderrDrain.cancel(); + await drains; + } + return { + exitCode, + timedOut, + output: `${stdout.text}${stderr.text}${timedOut ? `\n[watchdog] chunk exceeded ${Math.round(chunkTimeoutMs() / 1000)}s; killed with SIGKILL (OMP_TEST_CHUNK_TIMEOUT to change)\n` : ""}`, + }; + } + async function worker(): Promise { for (;;) { const testCommand = queue.shift(); @@ -747,55 +826,39 @@ export async function runTestCommandsInParallel(commands: TestCommand[], concurr } const renderedCommand = testCommand.command.map(shellQuote).join(" "); const startedAt = performance.now(); - const proc = Bun.spawn(testCommand.command, { - cwd: path.join(repoRoot, testCommand.cwd), - env, - stdout: "pipe", - stderr: "pipe", - }); - const stdout = { text: "" }; - const stderr = { text: "" }; - const stdoutDrain = drainInto(proc.stdout as ReadableStream, stdout); - const stderrDrain = drainInto(proc.stderr as ReadableStream, stderr); - const drains = Promise.all([stdoutDrain.done, stderrDrain.done]); - // Watchdog: a wedged child (e.g. bun's panic handler deadlocking - // after a GC crash) would otherwise hang this worker forever. - let timedOut = false; - const killTimer = setTimeout(() => { - timedOut = true; - proc.kill("SIGKILL"); - }, chunkTimeoutMs()); - const exitCode = await proc.exited; - clearTimeout(killTimer); - // Cap the post-exit drain: a leaked grandchild that inherited the - // pipes keeps them open indefinitely, and a pending read would keep - // the runner's event loop alive — cancel the readers instead. - if (!(await settleWithin(drains, 5000))) { - stdoutDrain.cancel(); - stderrDrain.cancel(); - await drains; + let attempt = 1; + let result = await runAttempt(testCommand); + // Bun-crash retry: a 128+signal exit means the runtime died (GC + // segfault/abort), not that tests failed — rerun in a fresh heap. + while (!result.timedOut && BUN_CRASH_EXITS[result.exitCode] && attempt < MAX_CHUNK_ATTEMPTS) { + attempt += 1; + process.stdout.write( + `${style.dim(`↻ ${testCommand.label} — bun crashed (exit ${result.exitCode}); retrying (attempt ${attempt}/${MAX_CHUNK_ATTEMPTS})`)}\n`, + ); + result = await runAttempt(testCommand); } completed += 1; const outcome: ChunkOutcome = { label: testCommand.label, command: renderedCommand, - exitCode, + exitCode: result.exitCode, seconds: (performance.now() - startedAt) / 1000, - output: `${stdout.text}${stderr.text}${timedOut ? `\n[watchdog] chunk exceeded ${Math.round(chunkTimeoutMs() / 1000)}s; killed with SIGKILL (OMP_TEST_CHUNK_TIMEOUT to change)\n` : ""}`, + output: result.output, + retries: attempt - 1, }; if (quiet) { let msg = `${formatProgressLine(outcome)}\n`; - if (exitCode !== 0 || timedOut) { + if (result.exitCode !== 0 || result.timedOut) { msg += `${formatChunkFailure(outcome, true)}\n`; } process.stdout.write(msg); } else { - const status = exitCode === 0 ? "ok" : `FAILED exit ${exitCode}`; + const status = result.exitCode === 0 ? "ok" : `FAILED exit ${result.exitCode}`; process.stdout.write( `\n==> [${completed}/${commands.length}] ${testCommand.label} (${status}, ${outcome.seconds.toFixed(1)}s)\n$ ${renderedCommand}\n${outcome.output}`, ); } - if (exitCode !== 0 || timedOut) { + if (result.exitCode !== 0 || result.timedOut) { failures.push(outcome); } }