From 077b6af2fb81f7ab56945eaca8d973cde508d6fa Mon Sep 17 00:00:00 2001 From: Muhammad Mustaqeem <265153888+Mustaqeem66@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:19:59 +0500 Subject: [PATCH 1/3] test(ci): pin how a chunk's SIGKILL cause is reported Drives real subprocesses so the 137 under test is a genuine SIGKILL rather than a hand-written constant: `kill -9 $$` for the OOM-killer shape, and a watchdog-killed `sleep` for the timeout shape. --- scripts/ci-test-ts.test.ts | 73 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 scripts/ci-test-ts.test.ts diff --git a/scripts/ci-test-ts.test.ts b/scripts/ci-test-ts.test.ts new file mode 100644 index 000000000..2b73cb41a --- /dev/null +++ b/scripts/ci-test-ts.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "bun:test"; +import { describeChunkFailure } from "./ci-test-ts.ts"; + +// The two ways a chunk reaches SIGKILL are indistinguishable by exit code, so +// these drive real subprocesses to produce a genuine 137 rather than asserting +// against a hand-written constant. +async function spawnExitCode(script: string): Promise { + const proc = Bun.spawn(["sh", "-c", script], { stdout: "ignore", stderr: "ignore" }); + return await proc.exited; +} + +// Re-hosts the sequential runner's failure tail: spawn, watchdog, attribute. +// `runTestCommand` itself is not injectable (it builds argv from the repo +// layout), so the decision under test is driven directly. +async function runWithWatchdog(script: string, timeoutMs: number): Promise { + const proc = Bun.spawn(["sh", "-c", script], { stdout: "ignore", stderr: "ignore" }); + let timedOut = false; + const killTimer = setTimeout(() => { + timedOut = true; + proc.kill("SIGKILL"); + }, timeoutMs); + const exitCode = await proc.exited; + clearTimeout(killTimer); + return describeChunkFailure(exitCode, timedOut); +} + +describe("describeChunkFailure", () => { + test("a real SIGKILL that the watchdog did not cause is attributed to the OOM killer", async () => { + const exitCode = await spawnExitCode("kill -9 $$"); + expect(exitCode).toBe(137); + + const message = describeChunkFailure(exitCode, false); + expect(message).toContain("OOM killer"); + expect(message).toContain("chunkSize"); + // The old wording carried no cause at all; it must not come back. + expect(message).not.toBe("failed with exit code 137"); + }); + + test("a watchdog kill is attributed to the watchdog, not to memory", async () => { + const message = await runWithWatchdog("sleep 30", 150); + expect(message).toContain("chunk watchdog"); + expect(message).toContain("OMP_TEST_CHUNK_TIMEOUT"); + expect(message).not.toContain("OOM killer"); + }); + + test("the two SIGKILL causes produce different messages from the same exit code", async () => { + const oomKilled = describeChunkFailure(137, false); + const watchdogKilled = describeChunkFailure(137, true); + expect(oomKilled).not.toBe(watchdogKilled); + }); + + test("an ordinary test failure keeps the plain wording", async () => { + const exitCode = await spawnExitCode("exit 1"); + expect(exitCode).toBe(1); + expect(describeChunkFailure(exitCode, false)).toBe("failed with exit code 1"); + }); + + test("a bun crash exit keeps the plain wording so the retry log still reads naturally", () => { + expect(describeChunkFailure(134, false)).toBe("failed with exit code 134"); + expect(describeChunkFailure(139, false)).toBe("failed with exit code 139"); + }); + + test("the watchdog message reports the configured timeout", () => { + const previous = Bun.env.OMP_TEST_CHUNK_TIMEOUT; + Bun.env.OMP_TEST_CHUNK_TIMEOUT = "42"; + try { + expect(describeChunkFailure(137, true)).toContain("42s"); + } finally { + if (previous === undefined) delete Bun.env.OMP_TEST_CHUNK_TIMEOUT; + else Bun.env.OMP_TEST_CHUNK_TIMEOUT = previous; + } + }); +}); From 6b78c4e11253979b8d5022516478f4e9eb2f3f61 Mon Sep 17 00:00:00 2001 From: Muhammad Mustaqeem <265153888+Mustaqeem66@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:22:53 +0500 Subject: [PATCH 2/3] fix(ci): chunk the singleton bucket and name the cause of a SIGKILLed chunk The singleton/global-state bucket was the only bucket left unchunked, and it is the one that got OOM-killed: 79 files in a single `bun test` process exited 137. Chunking exists precisely to hold peak RSS under the runner's ceiling, so the bucket that opts out is the bucket that hits it. The stated reason for leaving it whole does not hold. That bucket is selected because its suites mutate process-wide state, and what keeps them from colliding is sequencing, which `parallel: 1` already guarantees. Splitting them across processes strictly increases isolation rather than reducing it. Use 10 files per chunk, the width the 650-file native bucket already sustains on the same runner. Separately, the sequential path could not say why a chunk was SIGKILLed. It arms a watchdog but never records when the watchdog fires, so an OOM kill and a watchdog kill both surfaced as a bare "failed with exit code 137" - the exact message this failure produced, which points at neither remedy. The parallel path already tracks this; apply the same idea here and name the cause. --- scripts/ci-test-ts.ts | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/scripts/ci-test-ts.ts b/scripts/ci-test-ts.ts index 61dfb9d79..f0f4c6563 100755 --- a/scripts/ci-test-ts.ts +++ b/scripts/ci-test-ts.ts @@ -65,8 +65,14 @@ const validModes: Record = { // separate `bun test` child process. A fresh process per chunk resets Bun's // heap and reaps any dangling spawned children between groups, keeping peak RSS // under the CI runner's OOM ceiling (a single 170–370-file invocation gets -// SIGKILLed at 137). The singleton/global-state bucket is left whole: its suites -// co-locate in one process to exercise process-wide state, so they must not split. +// SIGKILLed at 137). Every bucket is chunked, including singleton/global-state: +// that bucket is selected precisely because its suites mutate process-wide state +// (env vars, fake timers, Settings/agent-dir singletons), and it is sequencing, +// not co-location, that keeps them from colliding. `parallel: 1` already +// guarantees the sequencing, so splitting into separate processes strictly +// increases isolation. Left whole the bucket grew to 79 files and hit the very +// 137 that chunking exists to prevent; 10 is the width the 650-file native +// bucket already sustains on this runner. // // The UI/TUI bucket uses a smaller chunk (5) than the others: its suites build up // native ghostty-vt cells, and bun 1.3.14's GC aborts (SIGTRAP/SIGABRT, exit @@ -76,7 +82,7 @@ const validModes: Record = { // 10-file chunk aborts ~50% of runs while either 5-file half is 0/20; halving the // chunk keeps each process under the threshold. const codingAgentBucketPlans: Record = { - singleton: { label: "singleton/global-state bucket", parallel: 1 }, + singleton: { label: "singleton/global-state bucket", parallel: 1, chunkSize: 10 }, ui: { label: "UI/TUI bucket", parallel: 1, chunkSize: 5 }, runtime: { label: "runtime/session bucket", parallel: 1, chunkSize: 10 }, native: { label: "native/tooling/browser/unit bucket", parallel: 1, chunkSize: 10 }, @@ -417,19 +423,25 @@ async function runTestCommand(testCommand: TestCommand): Promise { stdout: "inherit", stderr: "inherit", }); - const killTimer = setTimeout(() => proc.kill("SIGKILL"), chunkTimeoutMs()); + // Watchdog, mirroring the parallel path: record that *we* killed the child, + // otherwise the resulting 137 is indistinguishable from an OOM kill. + let timedOut = false; + const killTimer = setTimeout(() => { + timedOut = true; + proc.kill("SIGKILL"); + }, chunkTimeoutMs()); const exitCode = await proc.exited; clearTimeout(killTimer); if (exitCode === 0) { return; } - if (BUN_CRASH_EXITS[exitCode] && attempt < MAX_CHUNK_ATTEMPTS) { + if (!timedOut && BUN_CRASH_EXITS[exitCode] && attempt < MAX_CHUNK_ATTEMPTS) { console.log( `==> ${testCommand.label}: bun crashed (exit ${exitCode}); retrying (attempt ${attempt + 1}/${MAX_CHUNK_ATTEMPTS})`, ); continue; } - throw new Error(`${testCommand.label} failed with exit code ${exitCode}: ${renderedCommand}`); + throw new Error(`${testCommand.label} ${describeChunkFailure(exitCode, timedOut)}: ${renderedCommand}`); } } @@ -505,6 +517,21 @@ const BUN_CRASH_EXITS: Record = { // deterministic crash still fails every attempt and is reported normally. const MAX_CHUNK_ATTEMPTS = 3; +// Why a chunk failed, in words. Exit 137 is SIGKILL, which this runner reaches +// two very different ways -- the per-chunk watchdog firing, or the kernel OOM +// killer reaping a chunk that outgrew the runner -- and the bare exit code +// cannot tell them apart. Which one it was is the difference between "raise +// OMP_TEST_CHUNK_TIMEOUT" and "lower this bucket's chunkSize", so say it. +export function describeChunkFailure(exitCode: number, timedOut: boolean): string { + if (timedOut) { + return `exceeded the ${Math.round(chunkTimeoutMs() / 1000)}s chunk watchdog and was killed (exit ${exitCode}; OMP_TEST_CHUNK_TIMEOUT to change)`; + } + if (exitCode === 137) { + return "was SIGKILLed (exit 137) without reaching the chunk watchdog, which on a CI runner means the OOM killer; lower this bucket's chunkSize"; + } + return `failed with exit code ${exitCode}`; +} + // The standard `CI` signal is authoritative. In CI each bucket is its own // memory-capped runner job (a single fat invocation gets OOM-killed at 137), so // chunks run sequentially within a job and parallelism happens across jobs. From 21a4d0b2fa667a159c8612cae581960cf07cf17b Mon Sep 17 00:00:00 2001 From: Muhammad Mustaqeem <265153888+Mustaqeem66@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:42:37 +0500 Subject: [PATCH 3/3] revert(ci): keep the singleton bucket unchunked CI evidence overturns the chunking half of this branch. Splitting the singleton bucket into 8 chunks made the job run 11m27s and go red, where the unchunked bucket passes in ~1m31s on a concurrent PR. The duration matches the 600s chunk watchdog, i.e. a chunk wedged rather than merely running slowly -- the "these suites co-locate in one process" comment was describing a real coupling, so restore it and the original plan entry. The failure-attribution change is kept and is independently justified: this run is precisely the case where a bare "exit code 137" cannot distinguish the watchdog from the OOM killer. --- scripts/ci-test-ts.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/ci-test-ts.ts b/scripts/ci-test-ts.ts index f0f4c6563..51e492c96 100755 --- a/scripts/ci-test-ts.ts +++ b/scripts/ci-test-ts.ts @@ -65,14 +65,8 @@ const validModes: Record = { // separate `bun test` child process. A fresh process per chunk resets Bun's // heap and reaps any dangling spawned children between groups, keeping peak RSS // under the CI runner's OOM ceiling (a single 170–370-file invocation gets -// SIGKILLed at 137). Every bucket is chunked, including singleton/global-state: -// that bucket is selected precisely because its suites mutate process-wide state -// (env vars, fake timers, Settings/agent-dir singletons), and it is sequencing, -// not co-location, that keeps them from colliding. `parallel: 1` already -// guarantees the sequencing, so splitting into separate processes strictly -// increases isolation. Left whole the bucket grew to 79 files and hit the very -// 137 that chunking exists to prevent; 10 is the width the 650-file native -// bucket already sustains on this runner. +// SIGKILLed at 137). The singleton/global-state bucket is left whole: its suites +// co-locate in one process to exercise process-wide state, so they must not split. // // The UI/TUI bucket uses a smaller chunk (5) than the others: its suites build up // native ghostty-vt cells, and bun 1.3.14's GC aborts (SIGTRAP/SIGABRT, exit @@ -82,7 +76,7 @@ const validModes: Record = { // 10-file chunk aborts ~50% of runs while either 5-file half is 0/20; halving the // chunk keeps each process under the threshold. const codingAgentBucketPlans: Record = { - singleton: { label: "singleton/global-state bucket", parallel: 1, chunkSize: 10 }, + singleton: { label: "singleton/global-state bucket", parallel: 1 }, ui: { label: "UI/TUI bucket", parallel: 1, chunkSize: 5 }, runtime: { label: "runtime/session bucket", parallel: 1, chunkSize: 10 }, native: { label: "native/tooling/browser/unit bucket", parallel: 1, chunkSize: 10 },