feat(eval): added per-cell inactivity timeout budgets in eval executors

- Changed eval timeout behavior from hard wall-clock deadlines to per-cell inactivity budgets in all executors.
- Added IdleTimeout watchdog support, including bumps on status/tool activity and timer cleanup after execution.
- Updated executor option plumbing to replace deadlineMs with idleTimeoutMs and emit inactivity timeout annotations.
- Added IdleTimeout and shared-executor tests and updated prompt/repl docs for the new timeout contract.
This commit is contained in:
can1357
2026-05-31 10:11:59 +02:00
parent d7a5fe1ce5
commit 2003d7382e
12 changed files with 283 additions and 47 deletions
+5 -3
View File
@@ -27,7 +27,7 @@ Tool params:
language: "py" | "js";
code: string;
title?: string;
timeout?: number; // seconds, clamped to 1..600, default 30
timeout?: number; // seconds, clamped to 1..600, default 30. Inactivity budget — see "Cell timeout".
reset?: boolean; // reset this cell's selected runtime before execution
}>;
}
@@ -166,7 +166,9 @@ Python prelude helpers include `agent(prompt, *, agent_type="task", model=None,
### Cell timeout
Each eval cell timeout is in seconds, defaults to 30, and is clamped to `1..600`. The tool combines caller abort signal, session abort signal, and the current cell timeout with `AbortSignal.any(...)`.
Each eval cell `timeout` is in seconds, defaults to 30, and is clamped to `1..600`. It is an **inactivity (idle) budget, not a hard wall-clock cap**: the watchdog (`IdleTimeout`, `src/eval/idle-timeout.ts`) only fires once the cell goes the full window with **no progress signal**. Every status event re-arms it — `agent()` progress snapshots, `log()`/`phase()`, and tool-bridge activity all count — so a long-running fanout that keeps reporting progress runs to completion instead of being killed mid-stream.
Raw `stdout`/`stderr` does **not** re-arm the watchdog, so a pure-compute runaway loop with no progress reporting is still bounded by `timeout`. The tool combines the caller abort signal, the session abort signal, and the idle watchdog's signal with `AbortSignal.any(...)`; no wall-clock deadline is passed to the backend, so neither runtime arms a competing fixed timer.
### Kernel execution cancellation
@@ -174,7 +176,7 @@ On abort/timeout:
- The host sends `kill("SIGINT")` to the runner subprocess.
- The runner's exec-time signal handler raises `KeyboardInterrupt` inside the user code.
- Result includes `cancelled=true`; timeout path annotates output as `Command timed out after <n> seconds`.
- Result includes `cancelled=true`; the timeout path annotates output as `Command timed out after <n> seconds of inactivity`.
- Between requests the runner installs `SIG_IGN` for SIGINT so a stray cancel does not tear down the kernel.
If a second cancel is required (runner stuck in C code), the host escalates to `SIGTERM` and the session restarts on the next call.
+1
View File
@@ -20,6 +20,7 @@
### Changed
- Changed the eval cell `timeout` from a hard wall-clock deadline to an inactivity (idle) budget: a cell is now interrupted only after going the full window with no progress signal, and every status event — `agent()` progress snapshots, `log()`/`phase()`, and tool-bridge activity — re-arms the watchdog. Long `agent()`/`parallel()` fanouts that keep reporting progress no longer time out mid-run (previously the kernel was killed at the fixed deadline even while subagents were actively progressing). Raw `print`/stdout does not reset the watchdog, so pure-compute runaway loops stay bounded; the timeout is driven entirely by the abort signal, so neither runtime arms a competing fixed timer.
- Fixed turn-budget parsing to match `+Nk` directives only at token boundaries, preventing values like `version 1.2.3`, `c++`, and `+500kfoo` from triggering a budget rule
- Changed overflowing provider, hook-option, branch-message, agent, extension, and session-tree pickers to support fuzzy type-to-filter search.
- Changed Shift+Ctrl+P to cycle role models backward instead of cycling forward without persisting.
@@ -0,0 +1,66 @@
import { describe, expect, it } from "bun:test";
import { IdleTimeout } from "../idle-timeout";
/** Resolve true if `signal` aborts within `ms`, false if the window elapses first. */
function abortedWithin(signal: AbortSignal, ms: number): Promise<boolean> {
if (signal.aborted) return Promise.resolve(true);
const { promise, resolve } = Promise.withResolvers<boolean>();
const timer = setTimeout(() => resolve(false), ms);
signal.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve(true);
},
{ once: true },
);
return promise;
}
describe("IdleTimeout", () => {
it("aborts with a TimeoutError reason once the idle window elapses with no activity", async () => {
using idle = new IdleTimeout(40);
expect(idle.signal.aborted).toBe(false);
const fired = await abortedWithin(idle.signal, 500);
expect(fired).toBe(true);
expect(idle.signal.aborted).toBe(true);
// The reason must be a TimeoutError so downstream timeout detection
// (kernel `isTimeoutReason`, executor `isTimedOutCancellation`) classifies
// the cancellation as a timeout rather than a plain abort.
expect(idle.signal.reason).toBeInstanceOf(DOMException);
expect((idle.signal.reason as DOMException).name).toBe("TimeoutError");
});
it("re-arms on every bump and only fires after activity stops", async () => {
using idle = new IdleTimeout(150);
// Bump well past a single window; each bump must push the deadline forward
// so the watchdog never trips while activity continues.
for (let i = 0; i < 6; i++) {
await Bun.sleep(40);
idle.bump();
}
expect(idle.signal.aborted).toBe(false);
// Activity stopped — the watchdog should now fire within roughly one window.
const fired = await abortedWithin(idle.signal, 800);
expect(fired).toBe(true);
});
it("never fires after dispose()", async () => {
const idle = new IdleTimeout(30);
idle.dispose();
const fired = await abortedWithin(idle.signal, 150);
expect(fired).toBe(false);
expect(idle.signal.aborted).toBe(false);
});
it("ignores bump() after the watchdog has already fired", async () => {
using idle = new IdleTimeout(30);
await abortedWithin(idle.signal, 500);
expect(idle.signal.aborted).toBe(true);
// Late activity must not un-abort or rearm a settled watchdog.
idle.bump();
expect(idle.signal.aborted).toBe(true);
});
});
@@ -154,6 +154,27 @@ describe("shared eval executors", () => {
expect(result.output.trim()).toBe("42");
});
it("treats idleTimeoutMs as an inactivity budget, not a fixed timer", async () => {
using tempDir = TempDir.createSync("@omp-eval-js-idle-budget-");
const sessionFile = path.join(tempDir.path(), "session.jsonl");
const sessionId = `js-idle-budget:${crypto.randomUUID()}`;
const session = createToolSession(tempDir.path(), sessionFile);
// With no wall-clock deadlineMs/timeoutMs and no aborting signal, a cell that
// runs well past idleTimeoutMs must still complete: the backend must never
// derive a competing fixed timer from the inactivity budget.
const result = await executeJs("await Bun.sleep(120); return 'done';", {
sessionId,
session,
sessionFile,
idleTimeoutMs: 30,
});
expect(result.cancelled).toBe(false);
expect(result.exitCode).toBe(0);
expect(result.output.trim()).toBe("done");
});
it("shares Python state across executePython calls with one session id", async () => {
using tempDir = TempDir.createSync("@omp-eval-py-shared-");
const sessionFile = path.join(tempDir.path(), "session.jsonl");
+9 -1
View File
@@ -9,7 +9,15 @@ export interface ExecutorBackendExecOptions {
kernelOwnerId: string | undefined;
signal?: AbortSignal;
session: ToolSession;
deadlineMs: number;
/**
* Inactivity budget in milliseconds (the cell's `timeout`). Cancellation is
* driven entirely by `signal`, which the eval tool arms as an idle watchdog
* that fires a `TimeoutError` reason after this much time with no progress
* (status) events. Backends use this value only for timeout-annotation text
* and as cold-start headroom; they MUST NOT derive a competing wall-clock
* timer from it.
*/
idleTimeoutMs: number;
reset: boolean;
artifactPath: string | undefined;
artifactId: string | undefined;
@@ -0,0 +1,80 @@
/**
* Inactivity watchdog for eval cells.
*
* A cell's `timeout` is treated as an *idle* budget rather than a hard
* wall-clock deadline: the watchdog aborts {@link signal} (with a
* `TimeoutError` reason, matching `AbortSignal.timeout`) only once `idleMs`
* elapses with no {@link bump}. Every progress signal re-arms it, so a
* long-running fanout that keeps reporting progress (e.g. `agent()` status
* updates, `log()`/`phase()`) never trips the timeout, while a genuinely
* stalled cell still gets interrupted.
*
* The timer self-reschedules instead of being torn down and recreated on every
* bump, so a high-frequency stream of bumps (sub-second agent progress) costs
* one timestamp write per event rather than churning a timer each time.
*/
export class IdleTimeout {
readonly #controller = new AbortController();
readonly #idleMs: number;
/** Absolute time (epoch ms) at which inactivity is considered to have expired. */
#deadlineMs: number;
#timer: NodeJS.Timeout | undefined;
#settled = false;
constructor(idleMs: number) {
this.#idleMs = Math.max(1, Math.floor(idleMs));
this.#deadlineMs = Date.now() + this.#idleMs;
this.#arm(this.#idleMs);
}
/** Aborts with a `TimeoutError` reason once the inactivity budget is exhausted. */
get signal(): AbortSignal {
return this.#controller.signal;
}
/** Configured inactivity budget in milliseconds. */
get idleMs(): number {
return this.#idleMs;
}
/** Record activity, pushing the inactivity deadline forward by `idleMs`. */
bump(): void {
if (this.#settled) return;
this.#deadlineMs = Date.now() + this.#idleMs;
}
/** Stop the watchdog. Safe to call multiple times. */
dispose(): void {
if (this.#settled) return;
this.#settled = true;
if (this.#timer) {
clearTimeout(this.#timer);
this.#timer = undefined;
}
}
[Symbol.dispose](): void {
this.dispose();
}
#arm(delayMs: number): void {
const timer = setTimeout(() => this.#onExpire(), Math.max(0, delayMs));
// Never keep the event loop alive for the watchdog itself.
timer.unref?.();
this.#timer = timer;
}
#onExpire(): void {
if (this.#settled) return;
const remainingMs = this.#deadlineMs - Date.now();
if (remainingMs > 0) {
// A bump moved the deadline forward after this timer was armed; wait
// out the remaining window instead of firing early.
this.#arm(remainingMs);
return;
}
this.#settled = true;
this.#timer = undefined;
this.#controller.abort(new DOMException(`Idle for ${Math.round(this.#idleMs / 1000)}s`, "TimeoutError"));
}
}
+32 -7
View File
@@ -8,6 +8,12 @@ export interface JsExecutorOptions {
cwd?: string;
timeoutMs?: number;
deadlineMs?: number;
/**
* Inactivity budget (ms). Used for worker cold-start headroom and
* timeout-annotation text when the caller drives cancellation via an
* idle-aware `signal` instead of `deadlineMs`/`timeoutMs`. Never arms a timer.
*/
idleTimeoutMs?: number;
onChunk?: (chunk: string) => Promise<void> | void;
onStatus?: (event: JsStatusEvent) => void;
signal?: AbortSignal;
@@ -46,6 +52,20 @@ function isAbortError(error: unknown): boolean {
);
}
function isTimeoutReason(reason: unknown): boolean {
return (
(reason instanceof DOMException && reason.name === "TimeoutError") ||
(reason instanceof Error && reason.name === "TimeoutError")
);
}
function formatJsTimeoutAnnotation(timeoutMs: number | undefined, idle: boolean): string {
const suffix = idle ? " of inactivity" : "";
if (timeoutMs === undefined) return "Command timed out";
const secs = Math.max(1, Math.round(timeoutMs / 1000));
return `Command timed out after ${secs} seconds${suffix}`;
}
export async function executeJs(code: string, options: JsExecutorOptions): Promise<JsResult> {
const displayOutputs: JsDisplayOutput[] = [];
const outputSink = new OutputSink({
@@ -56,15 +76,20 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
maxColumns: resolveOutputMaxColumns(options.session.settings),
onChunk: chunk => options.onChunk?.(chunk),
});
const timeoutMs = getExecutionTimeoutMs(options);
const legacyTimeoutMs = getExecutionTimeoutMs(options);
const timeoutSignal =
typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0
? AbortSignal.timeout(timeoutMs)
typeof legacyTimeoutMs === "number" && Number.isFinite(legacyTimeoutMs) && legacyTimeoutMs > 0
? AbortSignal.timeout(legacyTimeoutMs)
: undefined;
const signal =
options.signal && timeoutSignal
? AbortSignal.any([options.signal, timeoutSignal])
: (options.signal ?? timeoutSignal);
// Idle mode: the eval tool drives cancellation via an idle-aware `signal` and
// passes only an inactivity budget. Use it for worker cold-start headroom and
// timeout-annotation text; never derive a competing fixed timer from it.
const idleMode = legacyTimeoutMs === undefined && options.idleTimeoutMs !== undefined;
const acquireBudgetMs = legacyTimeoutMs ?? options.idleTimeoutMs;
try {
await executeInVmContext({
@@ -75,7 +100,7 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
reset: options.reset,
code,
filename: `js-cell-${crypto.randomUUID()}.js`,
timeoutMs,
timeoutMs: acquireBudgetMs,
runState: {
signal,
onText: chunk => outputSink.push(chunk),
@@ -100,9 +125,9 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
};
} catch (error) {
if (signal?.aborted || isAbortError(error)) {
const timeoutReason = timeoutSignal?.aborted ? "Command timed out" : "";
if (timeoutReason) {
outputSink.push(timeoutReason);
const timedOut = Boolean(timeoutSignal?.aborted) || isTimeoutReason(options.signal?.reason);
if (timedOut) {
outputSink.push(formatJsTimeoutAnnotation(legacyTimeoutMs ?? options.idleTimeoutMs, idleMode));
}
const summary = await outputSink.dump();
return {
+1 -1
View File
@@ -20,7 +20,7 @@ export default {
async execute(code: string, opts: ExecutorBackendExecOptions): Promise<ExecutorBackendResult> {
const result = await executeJs(code, {
cwd: opts.cwd,
deadlineMs: opts.deadlineMs,
idleTimeoutMs: opts.idleTimeoutMs,
signal: opts.signal,
sessionId: namespaceSessionId(opts.sessionId),
sessionFile: opts.sessionFile,
+25 -7
View File
@@ -25,6 +25,12 @@ export interface PythonExecutorOptions {
timeoutMs?: number;
/** Absolute wall-clock deadline in milliseconds since epoch */
deadlineMs?: number;
/**
* Inactivity budget (ms). Used only for timeout-annotation text when the
* caller drives cancellation via an idle-aware `signal` instead of a
* wall-clock `deadlineMs`/`timeoutMs`. Does not arm a timer.
*/
idleTimeoutMs?: number;
/** Callback for streaming output chunks (already sanitized) */
onChunk?: (chunk: string) => Promise<void> | void;
/** AbortSignal for cancellation */
@@ -225,18 +231,20 @@ async function waitForPromiseWithCancellation<T>(
// Result formatting
// ---------------------------------------------------------------------------
function formatTimeoutAnnotation(timeoutMs?: number): string | undefined {
function formatTimeoutAnnotation(timeoutMs?: number, idle = false): string | undefined {
const suffix = idle ? " of inactivity" : "";
if (timeoutMs === undefined) return "Command timed out";
const secs = Math.max(1, Math.round(timeoutMs / 1000));
return `Command timed out after ${secs} seconds`;
return `Command timed out after ${secs} seconds${suffix}`;
}
function formatKernelTimeoutAnnotation(timeoutMs: number | undefined, kernelKilled: boolean): string {
function formatKernelTimeoutAnnotation(timeoutMs: number | undefined, kernelKilled: boolean, idle = false): string {
const secs = timeoutMs === undefined ? undefined : Math.max(1, Math.round(timeoutMs / 1000));
const suffix = idle ? " of inactivity" : "";
if (kernelKilled) {
return "eval cell timed out and the kernel was unresponsive to interrupt; the kernel has been killed and will be recreated on the next call.";
return `eval cell timed out${suffix} and the kernel was unresponsive to interrupt; the kernel has been killed and will be recreated on the next call.`;
}
const duration = secs === undefined ? "the configured timeout" : `${secs}s`;
const duration = secs === undefined ? "the configured timeout" : `${secs}s${suffix}`;
return `eval cell timed out after ${duration}; kernel interrupted but remains running. Reset the kernel via { reset: true } if state appears corrupted.`;
}
@@ -480,6 +488,10 @@ async function executeWithKernel(
const displayOutputs: KernelDisplayOutput[] = [];
const deadlineMs = getExecutionDeadlineMs(options);
let executionTimeoutMs: number | undefined;
// Idle mode: the caller (eval tool) drives cancellation via an idle-aware
// signal and passes no wall-clock deadline, so annotate timeouts with the
// configured inactivity budget rather than a remaining-deadline figure.
const idleMode = deadlineMs === undefined && options?.idleTimeoutMs !== undefined;
// Collect every display output and, for status events, stream them live so
// long-running bridge helpers (e.g. `agent()`) surface progress mid-cell.
@@ -512,7 +524,11 @@ async function executeWithKernel(
if (result.cancelled) {
const annotation = result.timedOut
? formatKernelTimeoutAnnotation(executionTimeoutMs, result.kernelKilled ?? false)
? formatKernelTimeoutAnnotation(
executionTimeoutMs ?? options?.idleTimeoutMs,
result.kernelKilled ?? false,
idleMode,
)
: undefined;
return {
exitCode: undefined,
@@ -549,7 +565,9 @@ async function executeWithKernel(
cancelled: true,
displayOutputs,
stdinRequested: false,
...(await sink.dump(timedOut ? formatTimeoutAnnotation(executionTimeoutMs) : undefined)),
...(await sink.dump(
timedOut ? formatTimeoutAnnotation(executionTimeoutMs ?? options?.idleTimeoutMs, idleMode) : undefined,
)),
};
}
const error = err instanceof Error ? err : new Error(String(err));
+1 -1
View File
@@ -28,7 +28,7 @@ export default {
const kernelMode = readSetting<PythonExecutorOptions["kernelMode"]>(opts.session, "python.kernelMode");
const executorOptions: PythonExecutorOptions = {
cwd: opts.cwd,
deadlineMs: opts.deadlineMs,
idleTimeoutMs: opts.idleTimeoutMs,
signal: opts.signal,
sessionId: namespaceSessionId(opts.sessionId),
kernelMode,
@@ -8,7 +8,7 @@ Cell fields:
- `language` — {{#if py}}`"py"` for the IPython kernel{{/if}}{{#ifAll py js}}, {{/ifAll}}{{#if js}}`"js"` for the persistent JavaScript VM{{/if}}.
- `code` — cell body, verbatim. Newlines, quotes, and indentation are JSON-encoded; no fences, no headers.
- `title` (optional) — short label shown in the transcript (e.g. `"imports"`, `"load config"`).
- `timeout` (optional) — per-cell timeout in seconds (1-600). Default 30.
- `timeout` (optional) — per-cell **inactivity** budget in seconds (1-600). Default 30. The cell is interrupted only after this long with no progress, and every status event (`agent()` updates, `log()`/`phase()`, tool activity) resets the clock — so a long `agent()`/`parallel()` fanout that keeps reporting progress is not killed. Raw `print`/stdout does not reset it; raise `timeout` for a cell that runs long without emitting status.
- `reset` (optional) — wipe this cell's language kernel before running.{{#ifAll py js}} Reset is per-language: a `py` cell's reset does not touch the JavaScript VM and vice versa.{{/ifAll}}
**Work incrementally:**
+41 -26
View File
@@ -6,7 +6,8 @@ import { formatNumber, prompt } from "@oh-my-pi/pi-utils";
import * as z from "zod/v4";
import { settings } from "../config/settings";
import { jsBackend, pythonBackend } from "../eval";
import type { ExecutorBackend } from "../eval/backend";
import type { ExecutorBackend, ExecutorBackendResult } from "../eval/backend";
import { IdleTimeout } from "../eval/idle-timeout";
import { defaultEvalSessionId } from "../eval/session-id";
import type { EvalCellResult, EvalDisplayOutput, EvalLanguage, EvalStatusEvent, EvalToolDetails } from "../eval/types";
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
@@ -346,12 +347,20 @@ export class EvalTool implements AgentTool<typeof evalSchema> {
for (let i = 0; i < cells.length; i++) {
const cell = cells[i];
const backend = cell.resolved.backend;
const timeoutSec = timeoutSecondsFromMs(cell.timeoutMs);
const deadlineMs = Date.now() + timeoutSec * 1000;
const timeoutSignal = AbortSignal.timeout(Math.max(0, deadlineMs - Date.now()));
// The per-cell `timeout` is an *inactivity* budget, not a hard
// wall-clock cap: it bounds the gap between progress signals
// (status events — agent() updates, log()/phase(), tool-bridge
// activity), so a long fanout that keeps reporting progress runs to
// completion while a genuinely stalled cell (no progress for the
// whole window) is still interrupted. Raw stdout deliberately does
// NOT re-arm it, so pure-compute runaway loops stay bounded. The
// watchdog drives `combinedSignal`; we pass no wall-clock deadline
// downstream so the backends never arm a competing fixed timer.
const idleTimeoutMs = timeoutSecondsFromMs(cell.timeoutMs) * 1000;
const idle = new IdleTimeout(idleTimeoutMs);
const combinedSignal = signal
? AbortSignal.any([signal, timeoutSignal, sessionAbortController.signal])
: AbortSignal.any([timeoutSignal, sessionAbortController.signal]);
? AbortSignal.any([signal, idle.signal, sessionAbortController.signal])
: AbortSignal.any([idle.signal, sessionAbortController.signal]);
const cellResult = cellResults[i];
cellResult.status = "running";
@@ -362,26 +371,32 @@ export class EvalTool implements AgentTool<typeof evalSchema> {
pushUpdate();
const startTime = Date.now();
const result = await backend.execute(cell.code, {
cwd: session.cwd,
sessionId,
sessionFile: sessionFile ?? undefined,
kernelOwnerId,
signal: combinedSignal,
session,
deadlineMs,
reset: cell.reset,
artifactPath,
artifactId,
onChunk: chunk => {
outputSink!.push(chunk);
},
onStatus: event => {
cellResult.statusEvents ??= [];
upsertStatusEvent(cellResult.statusEvents, event);
pushUpdate();
},
});
let result: ExecutorBackendResult;
try {
result = await backend.execute(cell.code, {
cwd: session.cwd,
sessionId,
sessionFile: sessionFile ?? undefined,
kernelOwnerId,
signal: combinedSignal,
session,
idleTimeoutMs,
reset: cell.reset,
artifactPath,
artifactId,
onChunk: chunk => {
outputSink!.push(chunk);
},
onStatus: event => {
idle.bump();
cellResult.statusEvents ??= [];
upsertStatusEvent(cellResult.statusEvents, event);
pushUpdate();
},
});
} finally {
idle.dispose();
}
const durationMs = Date.now() - startTime;
const cellStatusEvents: EvalStatusEvent[] = [];