chore: implemented watchdog and stabilize GC for test execution

- Added a per-chunk watchdog that terminates wedged test processes with SIGKILL after a configurable timeout. - Implemented incremental stream draining to capture test output even when a process is forcibly killed. - Stabilized test execution by forcing single-threaded GC markers (`BUN_JSC_numberOfGCMarkers=1`) to prevent segmentation faults during heap-heavy operations.
2026-07-03 00:21:25 +02:00
parent 80eccf99e4
commit 83176e159f
2 changed files with 97 additions and 9 deletions
@@ -434,7 +434,9 @@ async function runTestCommand(testCommand: TestCommand): Promise<void> {
 		stdout: "inherit",
 		stderr: "inherit",
 	});
+	const killTimer = setTimeout(() => proc.kill("SIGKILL"), chunkTimeoutMs());
 	const exitCode = await proc.exited;
+	clearTimeout(killTimer);
 	if (exitCode !== 0) {
 		throw new Error(`${testCommand.label} failed with exit code ${exitCode}: ${renderedCommand}`);
 	}
@@ -443,8 +445,25 @@ async function runTestCommand(testCommand: TestCommand): Promise<void> {
 // Child env shared by every spawned test process: the parent env with all CI
 // credential / cloud-config variables scrubbed (see SCRUBBED_ENV_* above) and
 // GITHUB_ACTIONS cleared so suites resolve only against their own fixtures.
+//
+// GC knobs (both needed — they gate different JSC mechanisms):
+// - `BUN_JSC_useConcurrentGC=0` stops the collector from marking concurrently
+//   with the mutator (868789972, an earlier GC crash under bun test).
+// - `BUN_JSC_numberOfGCMarkers=1` removes the ParallelHelperPool marker
+//   threads. Bun 1.3.14 segfaults/aborts inside parallel marking
+//   (`DOMGCOutputConstraint::executeImplImpl` → `visitOutputConstraints` on a
+//   dead cell; also "Pure virtual function called!") on heap-heavy
+//   coding-agent chunks (~1.3GB RSS, native ghostty-vt cells). Repro: UI
+//   bucket chunk crashed ~25% of runs with `BUN_JSC_forceRAMSize=256MB`,
+//   0/10 with markers=1, at zero measured wall-time cost. useConcurrentGC=0
+//   alone did not prevent it — the crash predates this knob.
 function buildChildEnv(): Record<string, string | undefined> {
-	const env: Record<string, string | undefined> = { ...Bun.env, GITHUB_ACTIONS: "", BUN_JSC_useConcurrentGC: "0" };
+	const env: Record<string, string | undefined> = {
+		...Bun.env,
+		GITHUB_ACTIONS: "",
+		BUN_JSC_useConcurrentGC: "0",
+		BUN_JSC_numberOfGCMarkers: "1",
+	};
 	for (const key of Object.keys(env)) {
 		if (isScrubbedEnvVar(key)) {
 			delete env[key];
@@ -453,6 +472,18 @@ function buildChildEnv(): Record<string, string | undefined> {
 	return env;
 }

+// Per-chunk watchdog. A bun child that wedges (e.g. the panic handler
+// deadlocking after a GC crash) would otherwise stall the whole run: the
+// parallel path awaits the child's stdout/stderr pipes, which stay open as
+// long as the wedged process — or any grandchild that inherited them — lives.
+// After this many seconds the child is SIGKILLed and reported as a failure.
+// Override with OMP_TEST_CHUNK_TIMEOUT (seconds).
+function chunkTimeoutMs(): number {
+	const raw = Number(Bun.env.OMP_TEST_CHUNK_TIMEOUT?.trim());
+	if (Number.isFinite(raw) && raw >= 1) return raw * 1000;
+	return 600_000;
+}
+
 // The standard `CI` signal is authoritative. In CI each bucket is its own
 // memory-capped runner job (a single fat invocation gets OOM-killed at 137), so
 // chunks run sequentially within a job and parallelism happens across jobs.
@@ -632,7 +663,7 @@ export function formatFailureReport(failures: ChunkOutcome[], total: number, rep
 // at the end; `--full` streams every chunk's output inline as it completes. All
 // failures are collected and reported together instead of failing fast, so one
 // run surfaces every broken chunk and exits non-zero without a runner stack trace.
-async function runTestCommandsInParallel(commands: TestCommand[], concurrency: number): Promise<void> {
+export async function runTestCommandsInParallel(commands: TestCommand[], concurrency: number): Promise<void> {
 	const env = buildChildEnv();
 	const queue = [...commands];
 	const failures: ChunkOutcome[] = [];
@@ -642,6 +673,45 @@ async function runTestCommandsInParallel(commands: TestCommand[], concurrency: n
 			`(OMP_TEST_CONCURRENCY=<n>|all to change).`,
 	);

+	// Incremental, cancellable drain into a mutable sink, so a watchdog-killed
+	// chunk still reports whatever the child managed to print before it wedged.
+	function drainInto(
+		stream: ReadableStream<Uint8Array>,
+		sink: { text: string },
+	): { done: Promise<void>; cancel: () => void } {
+		const decoder = new TextDecoder();
+		const reader = stream.getReader();
+		const done = (async () => {
+			try {
+				for (;;) {
+					const { done: ended, value } = await reader.read();
+					if (ended) break;
+					sink.text += decoder.decode(value, { stream: true });
+				}
+			} catch {
+				// cancelled or broken pipe — keep what was captured
+			}
+			sink.text += decoder.decode();
+		})();
+		return { done, cancel: () => void reader.cancel().catch(() => {}) };
+	}
+
+	// Wait for `promise` at most `ms`; resolves `true` when it settled in time.
+	// Never rejects.
+	async function settleWithin(promise: Promise<unknown>, ms: number): Promise<boolean> {
+		const { promise: expired, resolve } = Promise.withResolvers<boolean>();
+		const timer = setTimeout(() => resolve(false), ms);
+		const settled = await Promise.race([
+			promise.then(
+				() => true,
+				() => true,
+			),
+			expired,
+		]);
+		clearTimeout(timer);
+		return settled;
+	}
+
 	async function worker(): Promise<void> {
 		for (;;) {
 			const testCommand = queue.shift();
@@ -656,18 +726,35 @@ async function runTestCommandsInParallel(commands: TestCommand[], concurrency: n
 				stdout: "pipe",
 				stderr: "pipe",
 			});
-			const [stdout, stderr, exitCode] = await Promise.all([
-				new Response(proc.stdout as ReadableStream<Uint8Array>).text(),
-				new Response(proc.stderr as ReadableStream<Uint8Array>).text(),
-				proc.exited,
-			]);
+			const stdout = { text: "" };
+			const stderr = { text: "" };
+			const stdoutDrain = drainInto(proc.stdout as ReadableStream<Uint8Array>, stdout);
+			const stderrDrain = drainInto(proc.stderr as ReadableStream<Uint8Array>, stderr);
+			const drains = Promise.all([stdoutDrain.done, stderrDrain.done]);
+			// Watchdog: a wedged child (e.g. bun's panic handler deadlocking
+			// after a GC crash) would otherwise hang this worker forever.
+			let timedOut = false;
+			const killTimer = setTimeout(() => {
+				timedOut = true;
+				proc.kill("SIGKILL");
+			}, chunkTimeoutMs());
+			const exitCode = await proc.exited;
+			clearTimeout(killTimer);
+			// Cap the post-exit drain: a leaked grandchild that inherited the
+			// pipes keeps them open indefinitely, and a pending read would keep
+			// the runner's event loop alive — cancel the readers instead.
+			if (!(await settleWithin(drains, 5000))) {
+				stdoutDrain.cancel();
+				stderrDrain.cancel();
+				await drains;
+			}
 			completed += 1;
 			const outcome: ChunkOutcome = {
 				label: testCommand.label,
 				command: renderedCommand,
 				exitCode,
 				seconds: (performance.now() - startedAt) / 1000,
-				output: `${stdout}${stderr}`,
+				output: `${stdout.text}${stderr.text}${timedOut ? `\n[watchdog] chunk exceeded ${Math.round(chunkTimeoutMs() / 1000)}s; killed with SIGKILL (OMP_TEST_CHUNK_TIMEOUT to change)\n` : ""}`,
 			};
 			if (quiet) {
 				process.stdout.write(`${formatProgressLine(outcome)}\n`);
@@ -677,7 +764,7 @@ async function runTestCommandsInParallel(commands: TestCommand[], concurrency: n
 					`\n==> [${completed}/${commands.length}] ${testCommand.label} (${status}, ${outcome.seconds.toFixed(1)}s)\n$ ${renderedCommand}\n${outcome.output}`,
 				);
 			}
-			if (exitCode !== 0) {
+			if (exitCode !== 0 || timedOut) {
 				failures.push(outcome);
 			}
 		}