feat(eval): added per-cell inactivity timeout budgets in eval executors
- Changed eval timeout behavior from hard wall-clock deadlines to per-cell inactivity budgets in all executors. - Added IdleTimeout watchdog support, including bumps on status/tool activity and timer cleanup after execution. - Updated executor option plumbing to replace deadlineMs with idleTimeoutMs and emit inactivity timeout annotations. - Added IdleTimeout and shared-executor tests and updated prompt/repl docs for the new timeout contract.
This commit is contained in:
+5
-3
@@ -27,7 +27,7 @@ Tool params:
|
||||
language: "py" | "js";
|
||||
code: string;
|
||||
title?: string;
|
||||
timeout?: number; // seconds, clamped to 1..600, default 30
|
||||
timeout?: number; // seconds, clamped to 1..600, default 30. Inactivity budget — see "Cell timeout".
|
||||
reset?: boolean; // reset this cell's selected runtime before execution
|
||||
}>;
|
||||
}
|
||||
@@ -166,7 +166,9 @@ Python prelude helpers include `agent(prompt, *, agent_type="task", model=None,
|
||||
|
||||
### Cell timeout
|
||||
|
||||
Each eval cell timeout is in seconds, defaults to 30, and is clamped to `1..600`. The tool combines caller abort signal, session abort signal, and the current cell timeout with `AbortSignal.any(...)`.
|
||||
Each eval cell `timeout` is in seconds, defaults to 30, and is clamped to `1..600`. It is an **inactivity (idle) budget, not a hard wall-clock cap**: the watchdog (`IdleTimeout`, `src/eval/idle-timeout.ts`) only fires once the cell goes the full window with **no progress signal**. Every status event re-arms it — `agent()` progress snapshots, `log()`/`phase()`, and tool-bridge activity all count — so a long-running fanout that keeps reporting progress runs to completion instead of being killed mid-stream.
|
||||
|
||||
Raw `stdout`/`stderr` does **not** re-arm the watchdog, so a pure-compute runaway loop with no progress reporting is still bounded by `timeout`. The tool combines the caller abort signal, the session abort signal, and the idle watchdog's signal with `AbortSignal.any(...)`; no wall-clock deadline is passed to the backend, so neither runtime arms a competing fixed timer.
|
||||
|
||||
### Kernel execution cancellation
|
||||
|
||||
@@ -174,7 +176,7 @@ On abort/timeout:
|
||||
|
||||
- The host sends `kill("SIGINT")` to the runner subprocess.
|
||||
- The runner's exec-time signal handler raises `KeyboardInterrupt` inside the user code.
|
||||
- Result includes `cancelled=true`; timeout path annotates output as `Command timed out after <n> seconds`.
|
||||
- Result includes `cancelled=true`; the timeout path annotates output as `Command timed out after <n> seconds of inactivity`.
|
||||
- Between requests the runner installs `SIG_IGN` for SIGINT so a stray cancel does not tear down the kernel.
|
||||
|
||||
If a second cancel is required (runner stuck in C code), the host escalates to `SIGTERM` and the session restarts on the next call.
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed the eval cell `timeout` from a hard wall-clock deadline to an inactivity (idle) budget: a cell is now interrupted only after going the full window with no progress signal, and every status event — `agent()` progress snapshots, `log()`/`phase()`, and tool-bridge activity — re-arms the watchdog. Long `agent()`/`parallel()` fanouts that keep reporting progress no longer time out mid-run (previously the kernel was killed at the fixed deadline even while subagents were actively progressing). Raw `print`/stdout does not reset the watchdog, so pure-compute runaway loops stay bounded; the timeout is driven entirely by the abort signal, so neither runtime arms a competing fixed timer.
|
||||
- Fixed turn-budget parsing to match `+Nk` directives only at token boundaries, preventing values like `version 1.2.3`, `c++`, and `+500kfoo` from triggering a budget rule
|
||||
- Changed overflowing provider, hook-option, branch-message, agent, extension, and session-tree pickers to support fuzzy type-to-filter search.
|
||||
- Changed Shift+Ctrl+P to cycle role models backward instead of cycling forward without persisting.
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { IdleTimeout } from "../idle-timeout";
|
||||
|
||||
/** Resolve true if `signal` aborts within `ms`, false if the window elapses first. */
|
||||
function abortedWithin(signal: AbortSignal, ms: number): Promise<boolean> {
|
||||
if (signal.aborted) return Promise.resolve(true);
|
||||
const { promise, resolve } = Promise.withResolvers<boolean>();
|
||||
const timer = setTimeout(() => resolve(false), ms);
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve(true);
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
return promise;
|
||||
}
|
||||
|
||||
describe("IdleTimeout", () => {
|
||||
it("aborts with a TimeoutError reason once the idle window elapses with no activity", async () => {
|
||||
using idle = new IdleTimeout(40);
|
||||
expect(idle.signal.aborted).toBe(false);
|
||||
|
||||
const fired = await abortedWithin(idle.signal, 500);
|
||||
expect(fired).toBe(true);
|
||||
expect(idle.signal.aborted).toBe(true);
|
||||
// The reason must be a TimeoutError so downstream timeout detection
|
||||
// (kernel `isTimeoutReason`, executor `isTimedOutCancellation`) classifies
|
||||
// the cancellation as a timeout rather than a plain abort.
|
||||
expect(idle.signal.reason).toBeInstanceOf(DOMException);
|
||||
expect((idle.signal.reason as DOMException).name).toBe("TimeoutError");
|
||||
});
|
||||
|
||||
it("re-arms on every bump and only fires after activity stops", async () => {
|
||||
using idle = new IdleTimeout(150);
|
||||
// Bump well past a single window; each bump must push the deadline forward
|
||||
// so the watchdog never trips while activity continues.
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await Bun.sleep(40);
|
||||
idle.bump();
|
||||
}
|
||||
expect(idle.signal.aborted).toBe(false);
|
||||
|
||||
// Activity stopped — the watchdog should now fire within roughly one window.
|
||||
const fired = await abortedWithin(idle.signal, 800);
|
||||
expect(fired).toBe(true);
|
||||
});
|
||||
|
||||
it("never fires after dispose()", async () => {
|
||||
const idle = new IdleTimeout(30);
|
||||
idle.dispose();
|
||||
const fired = await abortedWithin(idle.signal, 150);
|
||||
expect(fired).toBe(false);
|
||||
expect(idle.signal.aborted).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores bump() after the watchdog has already fired", async () => {
|
||||
using idle = new IdleTimeout(30);
|
||||
await abortedWithin(idle.signal, 500);
|
||||
expect(idle.signal.aborted).toBe(true);
|
||||
// Late activity must not un-abort or rearm a settled watchdog.
|
||||
idle.bump();
|
||||
expect(idle.signal.aborted).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -154,6 +154,27 @@ describe("shared eval executors", () => {
|
||||
expect(result.output.trim()).toBe("42");
|
||||
});
|
||||
|
||||
it("treats idleTimeoutMs as an inactivity budget, not a fixed timer", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-eval-js-idle-budget-");
|
||||
const sessionFile = path.join(tempDir.path(), "session.jsonl");
|
||||
const sessionId = `js-idle-budget:${crypto.randomUUID()}`;
|
||||
const session = createToolSession(tempDir.path(), sessionFile);
|
||||
|
||||
// With no wall-clock deadlineMs/timeoutMs and no aborting signal, a cell that
|
||||
// runs well past idleTimeoutMs must still complete: the backend must never
|
||||
// derive a competing fixed timer from the inactivity budget.
|
||||
const result = await executeJs("await Bun.sleep(120); return 'done';", {
|
||||
sessionId,
|
||||
session,
|
||||
sessionFile,
|
||||
idleTimeoutMs: 30,
|
||||
});
|
||||
|
||||
expect(result.cancelled).toBe(false);
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.output.trim()).toBe("done");
|
||||
});
|
||||
|
||||
it("shares Python state across executePython calls with one session id", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-eval-py-shared-");
|
||||
const sessionFile = path.join(tempDir.path(), "session.jsonl");
|
||||
|
||||
@@ -9,7 +9,15 @@ export interface ExecutorBackendExecOptions {
|
||||
kernelOwnerId: string | undefined;
|
||||
signal?: AbortSignal;
|
||||
session: ToolSession;
|
||||
deadlineMs: number;
|
||||
/**
|
||||
* Inactivity budget in milliseconds (the cell's `timeout`). Cancellation is
|
||||
* driven entirely by `signal`, which the eval tool arms as an idle watchdog
|
||||
* that fires a `TimeoutError` reason after this much time with no progress
|
||||
* (status) events. Backends use this value only for timeout-annotation text
|
||||
* and as cold-start headroom; they MUST NOT derive a competing wall-clock
|
||||
* timer from it.
|
||||
*/
|
||||
idleTimeoutMs: number;
|
||||
reset: boolean;
|
||||
artifactPath: string | undefined;
|
||||
artifactId: string | undefined;
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* Inactivity watchdog for eval cells.
|
||||
*
|
||||
* A cell's `timeout` is treated as an *idle* budget rather than a hard
|
||||
* wall-clock deadline: the watchdog aborts {@link signal} (with a
|
||||
* `TimeoutError` reason, matching `AbortSignal.timeout`) only once `idleMs`
|
||||
* elapses with no {@link bump}. Every progress signal re-arms it, so a
|
||||
* long-running fanout that keeps reporting progress (e.g. `agent()` status
|
||||
* updates, `log()`/`phase()`) never trips the timeout, while a genuinely
|
||||
* stalled cell still gets interrupted.
|
||||
*
|
||||
* The timer self-reschedules instead of being torn down and recreated on every
|
||||
* bump, so a high-frequency stream of bumps (sub-second agent progress) costs
|
||||
* one timestamp write per event rather than churning a timer each time.
|
||||
*/
|
||||
export class IdleTimeout {
|
||||
readonly #controller = new AbortController();
|
||||
readonly #idleMs: number;
|
||||
/** Absolute time (epoch ms) at which inactivity is considered to have expired. */
|
||||
#deadlineMs: number;
|
||||
#timer: NodeJS.Timeout | undefined;
|
||||
#settled = false;
|
||||
|
||||
constructor(idleMs: number) {
|
||||
this.#idleMs = Math.max(1, Math.floor(idleMs));
|
||||
this.#deadlineMs = Date.now() + this.#idleMs;
|
||||
this.#arm(this.#idleMs);
|
||||
}
|
||||
|
||||
/** Aborts with a `TimeoutError` reason once the inactivity budget is exhausted. */
|
||||
get signal(): AbortSignal {
|
||||
return this.#controller.signal;
|
||||
}
|
||||
|
||||
/** Configured inactivity budget in milliseconds. */
|
||||
get idleMs(): number {
|
||||
return this.#idleMs;
|
||||
}
|
||||
|
||||
/** Record activity, pushing the inactivity deadline forward by `idleMs`. */
|
||||
bump(): void {
|
||||
if (this.#settled) return;
|
||||
this.#deadlineMs = Date.now() + this.#idleMs;
|
||||
}
|
||||
|
||||
/** Stop the watchdog. Safe to call multiple times. */
|
||||
dispose(): void {
|
||||
if (this.#settled) return;
|
||||
this.#settled = true;
|
||||
if (this.#timer) {
|
||||
clearTimeout(this.#timer);
|
||||
this.#timer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
[Symbol.dispose](): void {
|
||||
this.dispose();
|
||||
}
|
||||
|
||||
#arm(delayMs: number): void {
|
||||
const timer = setTimeout(() => this.#onExpire(), Math.max(0, delayMs));
|
||||
// Never keep the event loop alive for the watchdog itself.
|
||||
timer.unref?.();
|
||||
this.#timer = timer;
|
||||
}
|
||||
|
||||
#onExpire(): void {
|
||||
if (this.#settled) return;
|
||||
const remainingMs = this.#deadlineMs - Date.now();
|
||||
if (remainingMs > 0) {
|
||||
// A bump moved the deadline forward after this timer was armed; wait
|
||||
// out the remaining window instead of firing early.
|
||||
this.#arm(remainingMs);
|
||||
return;
|
||||
}
|
||||
this.#settled = true;
|
||||
this.#timer = undefined;
|
||||
this.#controller.abort(new DOMException(`Idle for ${Math.round(this.#idleMs / 1000)}s`, "TimeoutError"));
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,12 @@ export interface JsExecutorOptions {
|
||||
cwd?: string;
|
||||
timeoutMs?: number;
|
||||
deadlineMs?: number;
|
||||
/**
|
||||
* Inactivity budget (ms). Used for worker cold-start headroom and
|
||||
* timeout-annotation text when the caller drives cancellation via an
|
||||
* idle-aware `signal` instead of `deadlineMs`/`timeoutMs`. Never arms a timer.
|
||||
*/
|
||||
idleTimeoutMs?: number;
|
||||
onChunk?: (chunk: string) => Promise<void> | void;
|
||||
onStatus?: (event: JsStatusEvent) => void;
|
||||
signal?: AbortSignal;
|
||||
@@ -46,6 +52,20 @@ function isAbortError(error: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function isTimeoutReason(reason: unknown): boolean {
|
||||
return (
|
||||
(reason instanceof DOMException && reason.name === "TimeoutError") ||
|
||||
(reason instanceof Error && reason.name === "TimeoutError")
|
||||
);
|
||||
}
|
||||
|
||||
function formatJsTimeoutAnnotation(timeoutMs: number | undefined, idle: boolean): string {
|
||||
const suffix = idle ? " of inactivity" : "";
|
||||
if (timeoutMs === undefined) return "Command timed out";
|
||||
const secs = Math.max(1, Math.round(timeoutMs / 1000));
|
||||
return `Command timed out after ${secs} seconds${suffix}`;
|
||||
}
|
||||
|
||||
export async function executeJs(code: string, options: JsExecutorOptions): Promise<JsResult> {
|
||||
const displayOutputs: JsDisplayOutput[] = [];
|
||||
const outputSink = new OutputSink({
|
||||
@@ -56,15 +76,20 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
||||
maxColumns: resolveOutputMaxColumns(options.session.settings),
|
||||
onChunk: chunk => options.onChunk?.(chunk),
|
||||
});
|
||||
const timeoutMs = getExecutionTimeoutMs(options);
|
||||
const legacyTimeoutMs = getExecutionTimeoutMs(options);
|
||||
const timeoutSignal =
|
||||
typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0
|
||||
? AbortSignal.timeout(timeoutMs)
|
||||
typeof legacyTimeoutMs === "number" && Number.isFinite(legacyTimeoutMs) && legacyTimeoutMs > 0
|
||||
? AbortSignal.timeout(legacyTimeoutMs)
|
||||
: undefined;
|
||||
const signal =
|
||||
options.signal && timeoutSignal
|
||||
? AbortSignal.any([options.signal, timeoutSignal])
|
||||
: (options.signal ?? timeoutSignal);
|
||||
// Idle mode: the eval tool drives cancellation via an idle-aware `signal` and
|
||||
// passes only an inactivity budget. Use it for worker cold-start headroom and
|
||||
// timeout-annotation text; never derive a competing fixed timer from it.
|
||||
const idleMode = legacyTimeoutMs === undefined && options.idleTimeoutMs !== undefined;
|
||||
const acquireBudgetMs = legacyTimeoutMs ?? options.idleTimeoutMs;
|
||||
|
||||
try {
|
||||
await executeInVmContext({
|
||||
@@ -75,7 +100,7 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
||||
reset: options.reset,
|
||||
code,
|
||||
filename: `js-cell-${crypto.randomUUID()}.js`,
|
||||
timeoutMs,
|
||||
timeoutMs: acquireBudgetMs,
|
||||
runState: {
|
||||
signal,
|
||||
onText: chunk => outputSink.push(chunk),
|
||||
@@ -100,9 +125,9 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
||||
};
|
||||
} catch (error) {
|
||||
if (signal?.aborted || isAbortError(error)) {
|
||||
const timeoutReason = timeoutSignal?.aborted ? "Command timed out" : "";
|
||||
if (timeoutReason) {
|
||||
outputSink.push(timeoutReason);
|
||||
const timedOut = Boolean(timeoutSignal?.aborted) || isTimeoutReason(options.signal?.reason);
|
||||
if (timedOut) {
|
||||
outputSink.push(formatJsTimeoutAnnotation(legacyTimeoutMs ?? options.idleTimeoutMs, idleMode));
|
||||
}
|
||||
const summary = await outputSink.dump();
|
||||
return {
|
||||
|
||||
@@ -20,7 +20,7 @@ export default {
|
||||
async execute(code: string, opts: ExecutorBackendExecOptions): Promise<ExecutorBackendResult> {
|
||||
const result = await executeJs(code, {
|
||||
cwd: opts.cwd,
|
||||
deadlineMs: opts.deadlineMs,
|
||||
idleTimeoutMs: opts.idleTimeoutMs,
|
||||
signal: opts.signal,
|
||||
sessionId: namespaceSessionId(opts.sessionId),
|
||||
sessionFile: opts.sessionFile,
|
||||
|
||||
@@ -25,6 +25,12 @@ export interface PythonExecutorOptions {
|
||||
timeoutMs?: number;
|
||||
/** Absolute wall-clock deadline in milliseconds since epoch */
|
||||
deadlineMs?: number;
|
||||
/**
|
||||
* Inactivity budget (ms). Used only for timeout-annotation text when the
|
||||
* caller drives cancellation via an idle-aware `signal` instead of a
|
||||
* wall-clock `deadlineMs`/`timeoutMs`. Does not arm a timer.
|
||||
*/
|
||||
idleTimeoutMs?: number;
|
||||
/** Callback for streaming output chunks (already sanitized) */
|
||||
onChunk?: (chunk: string) => Promise<void> | void;
|
||||
/** AbortSignal for cancellation */
|
||||
@@ -225,18 +231,20 @@ async function waitForPromiseWithCancellation<T>(
|
||||
// Result formatting
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function formatTimeoutAnnotation(timeoutMs?: number): string | undefined {
|
||||
function formatTimeoutAnnotation(timeoutMs?: number, idle = false): string | undefined {
|
||||
const suffix = idle ? " of inactivity" : "";
|
||||
if (timeoutMs === undefined) return "Command timed out";
|
||||
const secs = Math.max(1, Math.round(timeoutMs / 1000));
|
||||
return `Command timed out after ${secs} seconds`;
|
||||
return `Command timed out after ${secs} seconds${suffix}`;
|
||||
}
|
||||
|
||||
function formatKernelTimeoutAnnotation(timeoutMs: number | undefined, kernelKilled: boolean): string {
|
||||
function formatKernelTimeoutAnnotation(timeoutMs: number | undefined, kernelKilled: boolean, idle = false): string {
|
||||
const secs = timeoutMs === undefined ? undefined : Math.max(1, Math.round(timeoutMs / 1000));
|
||||
const suffix = idle ? " of inactivity" : "";
|
||||
if (kernelKilled) {
|
||||
return "eval cell timed out and the kernel was unresponsive to interrupt; the kernel has been killed and will be recreated on the next call.";
|
||||
return `eval cell timed out${suffix} and the kernel was unresponsive to interrupt; the kernel has been killed and will be recreated on the next call.`;
|
||||
}
|
||||
const duration = secs === undefined ? "the configured timeout" : `${secs}s`;
|
||||
const duration = secs === undefined ? "the configured timeout" : `${secs}s${suffix}`;
|
||||
return `eval cell timed out after ${duration}; kernel interrupted but remains running. Reset the kernel via { reset: true } if state appears corrupted.`;
|
||||
}
|
||||
|
||||
@@ -480,6 +488,10 @@ async function executeWithKernel(
|
||||
const displayOutputs: KernelDisplayOutput[] = [];
|
||||
const deadlineMs = getExecutionDeadlineMs(options);
|
||||
let executionTimeoutMs: number | undefined;
|
||||
// Idle mode: the caller (eval tool) drives cancellation via an idle-aware
|
||||
// signal and passes no wall-clock deadline, so annotate timeouts with the
|
||||
// configured inactivity budget rather than a remaining-deadline figure.
|
||||
const idleMode = deadlineMs === undefined && options?.idleTimeoutMs !== undefined;
|
||||
|
||||
// Collect every display output and, for status events, stream them live so
|
||||
// long-running bridge helpers (e.g. `agent()`) surface progress mid-cell.
|
||||
@@ -512,7 +524,11 @@ async function executeWithKernel(
|
||||
|
||||
if (result.cancelled) {
|
||||
const annotation = result.timedOut
|
||||
? formatKernelTimeoutAnnotation(executionTimeoutMs, result.kernelKilled ?? false)
|
||||
? formatKernelTimeoutAnnotation(
|
||||
executionTimeoutMs ?? options?.idleTimeoutMs,
|
||||
result.kernelKilled ?? false,
|
||||
idleMode,
|
||||
)
|
||||
: undefined;
|
||||
return {
|
||||
exitCode: undefined,
|
||||
@@ -549,7 +565,9 @@ async function executeWithKernel(
|
||||
cancelled: true,
|
||||
displayOutputs,
|
||||
stdinRequested: false,
|
||||
...(await sink.dump(timedOut ? formatTimeoutAnnotation(executionTimeoutMs) : undefined)),
|
||||
...(await sink.dump(
|
||||
timedOut ? formatTimeoutAnnotation(executionTimeoutMs ?? options?.idleTimeoutMs, idleMode) : undefined,
|
||||
)),
|
||||
};
|
||||
}
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
|
||||
@@ -28,7 +28,7 @@ export default {
|
||||
const kernelMode = readSetting<PythonExecutorOptions["kernelMode"]>(opts.session, "python.kernelMode");
|
||||
const executorOptions: PythonExecutorOptions = {
|
||||
cwd: opts.cwd,
|
||||
deadlineMs: opts.deadlineMs,
|
||||
idleTimeoutMs: opts.idleTimeoutMs,
|
||||
signal: opts.signal,
|
||||
sessionId: namespaceSessionId(opts.sessionId),
|
||||
kernelMode,
|
||||
|
||||
@@ -8,7 +8,7 @@ Cell fields:
|
||||
- `language` — {{#if py}}`"py"` for the IPython kernel{{/if}}{{#ifAll py js}}, {{/ifAll}}{{#if js}}`"js"` for the persistent JavaScript VM{{/if}}.
|
||||
- `code` — cell body, verbatim. Newlines, quotes, and indentation are JSON-encoded; no fences, no headers.
|
||||
- `title` (optional) — short label shown in the transcript (e.g. `"imports"`, `"load config"`).
|
||||
- `timeout` (optional) — per-cell timeout in seconds (1-600). Default 30.
|
||||
- `timeout` (optional) — per-cell **inactivity** budget in seconds (1-600). Default 30. The cell is interrupted only after this long with no progress, and every status event (`agent()` updates, `log()`/`phase()`, tool activity) resets the clock — so a long `agent()`/`parallel()` fanout that keeps reporting progress is not killed. Raw `print`/stdout does not reset it; raise `timeout` for a cell that runs long without emitting status.
|
||||
- `reset` (optional) — wipe this cell's language kernel before running.{{#ifAll py js}} Reset is per-language: a `py` cell's reset does not touch the JavaScript VM and vice versa.{{/ifAll}}
|
||||
|
||||
**Work incrementally:**
|
||||
|
||||
@@ -6,7 +6,8 @@ import { formatNumber, prompt } from "@oh-my-pi/pi-utils";
|
||||
import * as z from "zod/v4";
|
||||
import { settings } from "../config/settings";
|
||||
import { jsBackend, pythonBackend } from "../eval";
|
||||
import type { ExecutorBackend } from "../eval/backend";
|
||||
import type { ExecutorBackend, ExecutorBackendResult } from "../eval/backend";
|
||||
import { IdleTimeout } from "../eval/idle-timeout";
|
||||
import { defaultEvalSessionId } from "../eval/session-id";
|
||||
import type { EvalCellResult, EvalDisplayOutput, EvalLanguage, EvalStatusEvent, EvalToolDetails } from "../eval/types";
|
||||
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
|
||||
@@ -346,12 +347,20 @@ export class EvalTool implements AgentTool<typeof evalSchema> {
|
||||
for (let i = 0; i < cells.length; i++) {
|
||||
const cell = cells[i];
|
||||
const backend = cell.resolved.backend;
|
||||
const timeoutSec = timeoutSecondsFromMs(cell.timeoutMs);
|
||||
const deadlineMs = Date.now() + timeoutSec * 1000;
|
||||
const timeoutSignal = AbortSignal.timeout(Math.max(0, deadlineMs - Date.now()));
|
||||
// The per-cell `timeout` is an *inactivity* budget, not a hard
|
||||
// wall-clock cap: it bounds the gap between progress signals
|
||||
// (status events — agent() updates, log()/phase(), tool-bridge
|
||||
// activity), so a long fanout that keeps reporting progress runs to
|
||||
// completion while a genuinely stalled cell (no progress for the
|
||||
// whole window) is still interrupted. Raw stdout deliberately does
|
||||
// NOT re-arm it, so pure-compute runaway loops stay bounded. The
|
||||
// watchdog drives `combinedSignal`; we pass no wall-clock deadline
|
||||
// downstream so the backends never arm a competing fixed timer.
|
||||
const idleTimeoutMs = timeoutSecondsFromMs(cell.timeoutMs) * 1000;
|
||||
const idle = new IdleTimeout(idleTimeoutMs);
|
||||
const combinedSignal = signal
|
||||
? AbortSignal.any([signal, timeoutSignal, sessionAbortController.signal])
|
||||
: AbortSignal.any([timeoutSignal, sessionAbortController.signal]);
|
||||
? AbortSignal.any([signal, idle.signal, sessionAbortController.signal])
|
||||
: AbortSignal.any([idle.signal, sessionAbortController.signal]);
|
||||
|
||||
const cellResult = cellResults[i];
|
||||
cellResult.status = "running";
|
||||
@@ -362,26 +371,32 @@ export class EvalTool implements AgentTool<typeof evalSchema> {
|
||||
pushUpdate();
|
||||
|
||||
const startTime = Date.now();
|
||||
const result = await backend.execute(cell.code, {
|
||||
cwd: session.cwd,
|
||||
sessionId,
|
||||
sessionFile: sessionFile ?? undefined,
|
||||
kernelOwnerId,
|
||||
signal: combinedSignal,
|
||||
session,
|
||||
deadlineMs,
|
||||
reset: cell.reset,
|
||||
artifactPath,
|
||||
artifactId,
|
||||
onChunk: chunk => {
|
||||
outputSink!.push(chunk);
|
||||
},
|
||||
onStatus: event => {
|
||||
cellResult.statusEvents ??= [];
|
||||
upsertStatusEvent(cellResult.statusEvents, event);
|
||||
pushUpdate();
|
||||
},
|
||||
});
|
||||
let result: ExecutorBackendResult;
|
||||
try {
|
||||
result = await backend.execute(cell.code, {
|
||||
cwd: session.cwd,
|
||||
sessionId,
|
||||
sessionFile: sessionFile ?? undefined,
|
||||
kernelOwnerId,
|
||||
signal: combinedSignal,
|
||||
session,
|
||||
idleTimeoutMs,
|
||||
reset: cell.reset,
|
||||
artifactPath,
|
||||
artifactId,
|
||||
onChunk: chunk => {
|
||||
outputSink!.push(chunk);
|
||||
},
|
||||
onStatus: event => {
|
||||
idle.bump();
|
||||
cellResult.statusEvents ??= [];
|
||||
upsertStatusEvent(cellResult.statusEvents, event);
|
||||
pushUpdate();
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
idle.dispose();
|
||||
}
|
||||
const durationMs = Date.now() - startTime;
|
||||
|
||||
const cellStatusEvents: EvalStatusEvent[] = [];
|
||||
|
||||
Reference in New Issue
Block a user