can1357
bda98c63ef
- Elevated `eval` to an essential tool to ensure availability across all discovery modes. - Updated system and tool prompts to mandate the use of `eval` for non-trivial shell operations like conditionals, loops, heredocs, and complex pipelines. - Restricted `bash` usage to simple binary invocations and single-fact computation to reduce shell-escaping and execution errors.
766 lines
27 KiB
TypeScript
766 lines
27 KiB
TypeScript
import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
|
|
import type { ImageContent, ToolExample } from "@oh-my-pi/pi-ai";
|
|
import { prompt } from "@oh-my-pi/pi-utils";
|
|
import { type } from "arktype";
|
|
import { jsBackend, juliaBackend, pythonBackend, rubyBackend } from "../eval";
|
|
import type { ExecutorBackend, ExecutorBackendResult } from "../eval/backend";
|
|
import { EVAL_TIMEOUT_PAUSE_OP, EVAL_TIMEOUT_RESUME_OP } from "../eval/bridge-timeout";
|
|
import { IdleTimeout } from "../eval/idle-timeout";
|
|
import { defaultEvalSessionId } from "../eval/session-id";
|
|
import type { EvalCellResult, EvalDisplayOutput, EvalLanguage, EvalStatusEvent, EvalToolDetails } from "../eval/types";
|
|
import evalDescription from "../prompts/tools/eval.md" with { type: "text" };
|
|
import { DEFAULT_MAX_BYTES, OutputSink, type OutputSummary, TailBuffer } from "../session/streaming-output";
|
|
import { webpExclusionForModel } from "../utils/image-loading";
|
|
import { formatDimensionNote, resizeImage } from "../utils/image-resize";
|
|
import type { ToolSession } from ".";
|
|
import { truncateForPrompt } from "./approval";
|
|
import { type EvalBackendsAllowance, resolveEvalBackends } from "./eval-backends";
|
|
import { upsertStatusEvent } from "./eval-render";
|
|
import { resolveOutputMaxColumns, resolveOutputSinkHeadBytes } from "./output-meta";
|
|
import { ToolAbortError, ToolError } from "./tool-errors";
|
|
import { toolResult } from "./tool-result";
|
|
import { clampTimeout } from "./tool-timeouts";
|
|
|
|
export { EVAL_DEFAULT_PREVIEW_LINES, evalToolRenderer } from "./eval-render";
|
|
|
|
/** Language tokens the eval tool accepts, in stable display order. */
|
|
export type EvalLanguageToken = "py" | "js" | "rb" | "jl";
|
|
const EVAL_LANGUAGE_ORDER: readonly EvalLanguageToken[] = ["py", "js", "rb", "jl"];
|
|
const EVAL_LANGUAGE_RUNTIME: Record<EvalLanguageToken, string> = {
|
|
py: '"py" for the IPython kernel',
|
|
js: '"js" for the persistent JS VM',
|
|
rb: '"rb" for the persistent Ruby kernel',
|
|
jl: '"jl" for the persistent Julia kernel',
|
|
};
|
|
const EVAL_LANGUAGE_NAME: Record<EvalLanguageToken, string> = {
|
|
py: "Python",
|
|
js: "JavaScript",
|
|
rb: "Ruby",
|
|
jl: "Julia",
|
|
};
|
|
const EVAL_CELLS_DESCRIPTION =
|
|
"cells executed in order. State persists within each language across cells and tool calls.";
|
|
|
|
/** Join names as an English "or" list: ["A"]→"A", ["A","B"]→"A or B", 3+→"A, B, or C". */
|
|
function joinWithOr(items: readonly string[]): string {
|
|
if (items.length <= 1) return items[0] ?? "";
|
|
if (items.length === 2) return `${items[0]} or ${items[1]}`;
|
|
return `${items.slice(0, -1).join(", ")}, or ${items[items.length - 1]}`;
|
|
}
|
|
|
|
function describeLanguageField(langs: readonly EvalLanguageToken[]): string {
|
|
return `runtime: ${langs.map(lang => EVAL_LANGUAGE_RUNTIME[lang]).join(", ")}`;
|
|
}
|
|
|
|
function describeCodeField(langs: readonly EvalLanguageToken[]): string {
|
|
const replLangs = langs.filter(lang => lang === "rb" || lang === "jl");
|
|
// No persistent REPL backends → keep the original py/js phrasing verbatim so the
|
|
// default (rb/jl off) wire schema stays byte-identical to the pre-feature one.
|
|
if (replLangs.length === 0) return "cell body, verbatim. Use top-level await freely.";
|
|
const awaitLangs = langs.filter(lang => lang === "py" || lang === "js");
|
|
const clauses: string[] = [];
|
|
if (awaitLangs.length > 0) clauses.push(`Top-level \`await\` is available in ${awaitLangs.join("/")}`);
|
|
clauses.push(`${replLangs.join("/")} auto-display the last expression like a REPL`);
|
|
return `cell body, verbatim. ${clauses.join("; ")}.`;
|
|
}
|
|
|
|
/** One-line discovery summary listing the runtimes available this session. */
|
|
function summarizeEvalLanguages(langs: readonly EvalLanguageToken[]): string {
|
|
const names = langs.map(lang => EVAL_LANGUAGE_NAME[lang]);
|
|
const list = names.length > 0 ? joinWithOr(names) : "Python or JavaScript";
|
|
// "in-process" matches the historical py/js summary; persistent kernels (rb/jl) switch wording.
|
|
const backend = langs.some(lang => lang === "rb" || lang === "jl") ? "a persistent" : "an in-process";
|
|
return `Execute ${list} code in ${backend} eval backend`;
|
|
}
|
|
|
|
/** Resolved-allowance → enabled language tokens, preserving display order. */
|
|
function enabledEvalLanguages(backends: EvalBackendsAllowance): EvalLanguageToken[] {
|
|
const allowed: Record<EvalLanguageToken, boolean> = {
|
|
py: backends.python,
|
|
js: backends.js,
|
|
rb: backends.ruby,
|
|
jl: backends.julia,
|
|
};
|
|
return EVAL_LANGUAGE_ORDER.filter(lang => allowed[lang]);
|
|
}
|
|
|
|
const evalCellCommonFields = {
|
|
"title?": type("string").describe('short label shown in transcript (e.g. "imports", "load config")'),
|
|
"timeout?": type("number").describe("per-cell timeout in seconds"),
|
|
"reset?": type("boolean").describe(
|
|
"wipe this cell's language kernel before running. Other languages are untouched.",
|
|
),
|
|
};
|
|
|
|
/**
|
|
* Per-cell input. Each cell runs in order; state persists within a language
|
|
* across cells and across tool calls. This static schema carries the full
|
|
* language union for typing; {@link buildEvalSchema} narrows the wire copy per
|
|
* session so disabled backends are never advertised to the model.
|
|
*/
|
|
const evalCellSchema = type({
|
|
language: type("'py' | 'js' | 'rb' | 'jl'").describe(describeLanguageField(EVAL_LANGUAGE_ORDER)),
|
|
code: type("string").describe(describeCodeField(EVAL_LANGUAGE_ORDER)),
|
|
...evalCellCommonFields,
|
|
});
|
|
export type EvalCellInput = typeof evalCellSchema.infer;
|
|
|
|
export const evalSchema = type({
|
|
cells: evalCellSchema.array().atLeastLength(1).describe(EVAL_CELLS_DESCRIPTION),
|
|
});
|
|
export type EvalToolParams = typeof evalSchema.infer;
|
|
|
|
/**
|
|
* Build a session-scoped copy of the eval schema whose `language` enum and field
|
|
* descriptions advertise only the runtimes enabled for this session. Disabled
|
|
* backends never reach the model: the wire schema, BM25 discovery corpus, and
|
|
* tool description stay in lockstep with {@link resolveEvalBackends}. The static
|
|
* {@link evalSchema} (full union) remains the type-level source of truth.
|
|
*/
|
|
function buildEvalSchema(langs: readonly EvalLanguageToken[]): typeof evalSchema {
|
|
const cellSchema = type({
|
|
language: type.enumerated(...langs).describe(describeLanguageField(langs)),
|
|
code: type("string").describe(describeCodeField(langs)),
|
|
...evalCellCommonFields,
|
|
});
|
|
const schema = type({
|
|
cells: cellSchema.array().atLeastLength(1).describe(EVAL_CELLS_DESCRIPTION),
|
|
});
|
|
return schema as unknown as typeof evalSchema;
|
|
}
|
|
|
|
export type EvalToolResult = {
|
|
content: Array<{ type: "text"; text: string }>;
|
|
details: EvalToolDetails | undefined;
|
|
};
|
|
|
|
export type EvalProxyExecutor = (params: EvalToolParams, signal?: AbortSignal) => Promise<EvalToolResult>;
|
|
|
|
/** Cap per `display()` value sent back to the model. */
|
|
const MAX_DISPLAY_TEXT_BYTES = 8000;
|
|
|
|
function formatDisplayJsonForText(value: unknown): string {
|
|
let text: string;
|
|
try {
|
|
text = JSON.stringify(value, null, 2) ?? String(value);
|
|
} catch {
|
|
text = String(value);
|
|
}
|
|
if (text.length > MAX_DISPLAY_TEXT_BYTES) {
|
|
text = `${text.slice(0, MAX_DISPLAY_TEXT_BYTES)}\n[…${text.length - MAX_DISPLAY_TEXT_BYTES}ch elided…]`;
|
|
}
|
|
return text;
|
|
}
|
|
|
|
/**
|
|
* Format display() JSON values into text the model can see. Images are surfaced
|
|
* separately as ImageContent so the model can actually inspect them; this helper
|
|
* intentionally does not touch images.
|
|
*/
|
|
function formatDisplayOutputsForText(outputs: EvalDisplayOutput[]): string {
|
|
const chunks: string[] = [];
|
|
let displayIndex = 0;
|
|
for (const output of outputs) {
|
|
if (output.type !== "json") continue;
|
|
displayIndex++;
|
|
chunks.push(`display[${displayIndex}]:\n${formatDisplayJsonForText(output.data)}`);
|
|
}
|
|
return chunks.join("\n\n");
|
|
}
|
|
|
|
export interface EvalToolDescriptionOptions {
|
|
py?: boolean;
|
|
js?: boolean;
|
|
rb?: boolean;
|
|
jl?: boolean;
|
|
/**
|
|
* Whether `agent()` is allowed in this session. Driven by the parent's
|
|
* spawn policy (`getSessionSpawns`). Defaults to `true` for backward
|
|
* compatibility — when the session forbids spawning, the prelude doc
|
|
* omits the `agent()` entry so the model does not promise itself a
|
|
* helper that will only ever throw "spawns disabled".
|
|
*/
|
|
spawns?: boolean;
|
|
}
|
|
|
|
export function getEvalToolDescription(options: EvalToolDescriptionOptions = {}): string {
|
|
const py = options.py ?? true;
|
|
const js = options.js ?? true;
|
|
const rb = options.rb ?? false;
|
|
const jl = options.jl ?? false;
|
|
const spawns = options.spawns ?? true;
|
|
return prompt.render(evalDescription, { py, js, rb, jl, spawns });
|
|
}
|
|
|
|
export interface EvalToolOptions {
|
|
proxyExecutor?: EvalProxyExecutor;
|
|
}
|
|
|
|
interface ResolvedBackend {
|
|
backend: ExecutorBackend;
|
|
notice?: string;
|
|
}
|
|
|
|
interface ResolvedEvalCell {
|
|
index: number;
|
|
title?: string;
|
|
code: string;
|
|
timeoutMs: number;
|
|
reset: boolean;
|
|
resolved: ResolvedBackend;
|
|
}
|
|
|
|
function uniqueEvalLanguages(cells: ResolvedEvalCell[]): EvalLanguage[] {
|
|
return [...new Set(cells.map(cell => cell.resolved.backend.id))];
|
|
}
|
|
|
|
function detailsNotice(cells: ResolvedEvalCell[]): string | undefined {
|
|
const notices = [
|
|
...new Set(cells.map(cell => cell.resolved.notice).filter((notice): notice is string => Boolean(notice))),
|
|
];
|
|
return notices.length > 0 ? notices.join(" ") : undefined;
|
|
}
|
|
|
|
function timeoutSecondsFromMs(timeoutMs: number): number {
|
|
return clampTimeout("eval", timeoutMs / 1000);
|
|
}
|
|
|
|
async function resolveBackend(session: ToolSession, language: EvalLanguage): Promise<ResolvedBackend> {
|
|
const backends = resolveEvalBackends(session);
|
|
const allowPy = backends.python;
|
|
const allowJs = backends.js;
|
|
const allowRb = backends.ruby;
|
|
const allowJl = backends.julia;
|
|
|
|
if (language === "python") {
|
|
if (!allowPy) throw new ToolError("Python backend is disabled (PI_PY=0 or eval.py = false).");
|
|
if (!(await pythonBackend.isAvailable(session))) {
|
|
const alternatives = [allowJs ? '"js"' : null, allowRb ? '"rb"' : null, allowJl ? '"jl"' : null].filter(
|
|
Boolean,
|
|
);
|
|
throw new ToolError(
|
|
alternatives.length > 0
|
|
? `Python backend is unavailable in this session. Pass language: ${alternatives.join(" or ")} or install the python kernel.`
|
|
: 'Python backend is unavailable in this session. Install the python kernel to use language: "py".',
|
|
);
|
|
}
|
|
return { backend: pythonBackend };
|
|
}
|
|
if (language === "ruby") {
|
|
if (!allowRb) throw new ToolError("Ruby backend is disabled (PI_RB=0 or eval.rb = false).");
|
|
if (!(await rubyBackend.isAvailable(session))) {
|
|
const alternatives = [allowJs ? '"js"' : null, allowPy ? '"py"' : null, allowJl ? '"jl"' : null].filter(
|
|
Boolean,
|
|
);
|
|
throw new ToolError(
|
|
alternatives.length > 0
|
|
? `Ruby backend is unavailable in this session. Pass language: ${alternatives.join(" or ")} or install Ruby.`
|
|
: 'Ruby backend is unavailable in this session. Install Ruby to use language: "rb".',
|
|
);
|
|
}
|
|
return { backend: rubyBackend };
|
|
}
|
|
if (language === "julia") {
|
|
if (!allowJl) throw new ToolError("Julia backend is disabled (PI_JL=0 or eval.jl = false).");
|
|
if (!(await juliaBackend.isAvailable(session))) {
|
|
const alternatives = [allowJs ? '"js"' : null, allowPy ? '"py"' : null, allowRb ? '"rb"' : null].filter(
|
|
Boolean,
|
|
);
|
|
throw new ToolError(
|
|
alternatives.length > 0
|
|
? `Julia backend is unavailable in this session. Pass language: ${alternatives.join(" or ")} or install Julia.`
|
|
: 'Julia backend is unavailable in this session. Install Julia to use language: "jl".',
|
|
);
|
|
}
|
|
return { backend: juliaBackend };
|
|
}
|
|
if (!allowJs) throw new ToolError("JavaScript backend is disabled (PI_JS=0 or eval.js = false).");
|
|
return { backend: jsBackend };
|
|
}
|
|
function formatEvalInputLanguage(value: string): string {
|
|
if (value === "py" || value === "python") return "python";
|
|
if (value === "js" || value === "javascript") return "javascript";
|
|
if (value === "rb" || value === "ruby") return "ruby";
|
|
if (value === "jl" || value === "julia") return "julia";
|
|
return value;
|
|
}
|
|
|
|
export class EvalTool implements AgentTool<typeof evalSchema> {
|
|
readonly name = "eval";
|
|
readonly approval = "exec" as const;
|
|
readonly formatApprovalDetails = (args: unknown): string[] => {
|
|
const params = args as Partial<EvalToolParams>;
|
|
const cells = Array.isArray(params.cells) ? params.cells : [];
|
|
const firstCell = cells[0] as Partial<EvalCellInput> | undefined;
|
|
if (!firstCell) return [];
|
|
const language =
|
|
typeof firstCell.language === "string" ? formatEvalInputLanguage(firstCell.language) : "javascript (default)";
|
|
const code = typeof firstCell.code === "string" ? firstCell.code : "";
|
|
const lines = [`Language: ${language}`, `Code:\n${truncateForPrompt(code)}`];
|
|
if (cells.length > 1) {
|
|
lines.push(`+${cells.length - 1} more cell${cells.length === 2 ? "" : "s"}`);
|
|
}
|
|
return lines;
|
|
};
|
|
get summary(): string {
|
|
return summarizeEvalLanguages(this.#enabledLanguages());
|
|
}
|
|
readonly loadMode = "essential";
|
|
readonly label = "Eval";
|
|
get description(): string {
|
|
if (!this.session) return getEvalToolDescription();
|
|
const backends = resolveEvalBackends(this.session);
|
|
const sessionSpawns = this.session.getSessionSpawns?.() ?? "*";
|
|
const spawnsAllowed = sessionSpawns !== "" && sessionSpawns !== null;
|
|
return getEvalToolDescription({
|
|
py: backends.python,
|
|
js: backends.js,
|
|
rb: backends.ruby,
|
|
jl: backends.julia,
|
|
spawns: spawnsAllowed,
|
|
});
|
|
}
|
|
readonly examples: readonly ToolExample<typeof evalSchema.infer>[] = [
|
|
{
|
|
call: {
|
|
cells: [
|
|
{
|
|
language: "py",
|
|
title: "imports",
|
|
timeout: 10,
|
|
code: "import json\nfrom pathlib import Path",
|
|
},
|
|
{
|
|
language: "py",
|
|
title: "load config",
|
|
code: "data = json.loads(read('package.json'))\ndisplay(data)",
|
|
},
|
|
],
|
|
},
|
|
},
|
|
];
|
|
get parameters(): typeof evalSchema {
|
|
const langs = this.#enabledLanguages();
|
|
if (langs.length === 0 || langs.length === EVAL_LANGUAGE_ORDER.length) return evalSchema;
|
|
const key = langs.join(",");
|
|
if (this.#paramsKey !== key) {
|
|
this.#cachedParams = buildEvalSchema(langs);
|
|
this.#paramsKey = key;
|
|
}
|
|
return this.#cachedParams ?? evalSchema;
|
|
}
|
|
readonly concurrency = "exclusive";
|
|
readonly strict = true;
|
|
readonly intent = (args: Partial<typeof evalSchema.infer>): string | undefined => {
|
|
const cells = Array.isArray(args.cells) ? args.cells : [];
|
|
const first = cells.find(c => c && typeof c === "object");
|
|
if (!first) return "evaluating";
|
|
const title = typeof first.title === "string" ? first.title : undefined;
|
|
const language = typeof first.language === "string" ? formatEvalInputLanguage(first.language) : "javascript";
|
|
const label = title || `running ${language}`;
|
|
return cells.length > 1 ? `${label} (+${cells.length - 1})` : label;
|
|
};
|
|
|
|
readonly #proxyExecutor?: EvalProxyExecutor;
|
|
|
|
#paramsKey?: string;
|
|
#cachedParams?: typeof evalSchema;
|
|
|
|
/**
|
|
* Languages enabled for this session, in display order. Detached tools (no
|
|
* session) fall back to the shipped defaults (py/js; rb/jl are opt-in).
|
|
*/
|
|
#enabledLanguages(): EvalLanguageToken[] {
|
|
return this.session ? enabledEvalLanguages(resolveEvalBackends(this.session)) : ["py", "js"];
|
|
}
|
|
|
|
constructor(
|
|
private readonly session: ToolSession | null,
|
|
options?: EvalToolOptions,
|
|
) {
|
|
this.#proxyExecutor = options?.proxyExecutor;
|
|
}
|
|
|
|
async execute(
|
|
_toolCallId: string,
|
|
params: typeof evalSchema.infer,
|
|
signal?: AbortSignal,
|
|
onUpdate?: AgentToolUpdateCallback,
|
|
_ctx?: AgentToolContext,
|
|
): Promise<AgentToolResult<EvalToolDetails | undefined>> {
|
|
if (this.#proxyExecutor) {
|
|
return this.#proxyExecutor(params, signal);
|
|
}
|
|
|
|
if (!this.session) {
|
|
throw new ToolError("Eval tool requires a session when not using proxy executor");
|
|
}
|
|
const session = this.session;
|
|
const excludeWebP = webpExclusionForModel(session.getActiveModel?.());
|
|
|
|
const cells: ResolvedEvalCell[] = [];
|
|
for (let i = 0; i < params.cells.length; i++) {
|
|
const cell = params.cells[i];
|
|
const language: EvalLanguage =
|
|
cell.language === "py"
|
|
? "python"
|
|
: cell.language === "rb"
|
|
? "ruby"
|
|
: cell.language === "jl"
|
|
? "julia"
|
|
: "js";
|
|
const resolved = await resolveBackend(session, language);
|
|
cells.push({
|
|
index: i,
|
|
title: cell.title,
|
|
code: cell.code,
|
|
timeoutMs: (cell.timeout ?? 30) * 1000,
|
|
reset: cell.reset ?? false,
|
|
resolved,
|
|
});
|
|
}
|
|
const languages = uniqueEvalLanguages(cells);
|
|
const notice = detailsNotice(cells);
|
|
const sessionAbortController = new AbortController();
|
|
let outputSink: OutputSink | undefined;
|
|
let outputSummary: OutputSummary | undefined;
|
|
let outputDumped = false;
|
|
const finalizeOutput = async (): Promise<OutputSummary | undefined> => {
|
|
if (outputDumped || !outputSink) return outputSummary;
|
|
outputSummary = await outputSink.dump();
|
|
outputDumped = true;
|
|
return outputSummary;
|
|
};
|
|
|
|
const execution = (async (): Promise<AgentToolResult<EvalToolDetails | undefined>> => {
|
|
try {
|
|
if (signal?.aborted) {
|
|
throw new ToolAbortError();
|
|
}
|
|
session.assertEvalExecutionAllowed?.();
|
|
|
|
const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES * 2);
|
|
const jsonOutputs: unknown[] = [];
|
|
const images: ImageContent[] = [];
|
|
const statusEvents: EvalStatusEvent[] = [];
|
|
|
|
const cellResults: EvalCellResult[] = cells.map(cell => ({
|
|
index: cell.index,
|
|
title: cell.title,
|
|
code: cell.code,
|
|
language: cell.resolved.backend.id,
|
|
output: "",
|
|
status: "pending",
|
|
}));
|
|
const cellOutputs: string[] = [];
|
|
|
|
const appendTail = (text: string) => {
|
|
tailBuffer.append(text);
|
|
};
|
|
|
|
const buildUpdateDetails = (): EvalToolDetails => {
|
|
const details: EvalToolDetails = {
|
|
language: languages[0],
|
|
languages,
|
|
cells: cellResults.map(cell => ({
|
|
...cell,
|
|
statusEvents: cell.statusEvents ? [...cell.statusEvents] : undefined,
|
|
})),
|
|
};
|
|
if (jsonOutputs.length > 0) {
|
|
details.jsonOutputs = jsonOutputs;
|
|
}
|
|
if (images.length > 0) {
|
|
details.images = images;
|
|
}
|
|
if (statusEvents.length > 0) {
|
|
details.statusEvents = statusEvents;
|
|
}
|
|
if (notice) {
|
|
details.notice = notice;
|
|
}
|
|
return details;
|
|
};
|
|
|
|
const pushUpdate = () => {
|
|
if (!onUpdate) return;
|
|
const tailText = tailBuffer.text();
|
|
onUpdate({
|
|
content: [{ type: "text", text: tailText }],
|
|
details: buildUpdateDetails(),
|
|
});
|
|
};
|
|
|
|
const sessionFile = session.getSessionFile?.() ?? undefined;
|
|
const kernelOwnerId = session.getEvalKernelOwnerId?.() ?? undefined;
|
|
const { path: artifactPath, id: artifactId } = (await session.allocateOutputArtifact?.("eval")) ?? {};
|
|
session.assertEvalExecutionAllowed?.();
|
|
outputSink = new OutputSink({
|
|
artifactPath,
|
|
artifactId,
|
|
headBytes: resolveOutputSinkHeadBytes(session.settings),
|
|
maxColumns: resolveOutputMaxColumns(session.settings),
|
|
onChunk: chunk => {
|
|
appendTail(chunk);
|
|
pushUpdate();
|
|
},
|
|
});
|
|
const sessionId = session.getEvalSessionId?.() ?? defaultEvalSessionId(session);
|
|
|
|
for (let i = 0; i < cells.length; i++) {
|
|
const cell = cells[i];
|
|
const backend = cell.resolved.backend;
|
|
// The per-cell `timeout` is a budget on the cell runtime's *own*
|
|
// work. Host-side `agent()`/`parallel()`/`completion()` bridge calls suspend
|
|
// that budget entirely and restart a fresh timeout window when control
|
|
// returns to the active backend runtime. Compute, stdout, `log()`/`phase()`, and
|
|
// ordinary tool calls all count against the budget. The watchdog drives
|
|
// `combinedSignal`; we pass no wall-clock deadline downstream so the
|
|
// backends never arm a competing fixed timer.
|
|
const idleTimeoutMs = timeoutSecondsFromMs(cell.timeoutMs) * 1000;
|
|
const idle = new IdleTimeout(idleTimeoutMs);
|
|
const combinedSignal = signal
|
|
? AbortSignal.any([signal, idle.signal, sessionAbortController.signal])
|
|
: AbortSignal.any([idle.signal, sessionAbortController.signal]);
|
|
|
|
const cellResult = cellResults[i];
|
|
cellResult.status = "running";
|
|
cellResult.output = "";
|
|
cellResult.statusEvents = undefined;
|
|
cellResult.exitCode = undefined;
|
|
cellResult.durationMs = undefined;
|
|
pushUpdate();
|
|
|
|
const startTime = Date.now();
|
|
let result: ExecutorBackendResult;
|
|
try {
|
|
result = await backend.execute(cell.code, {
|
|
cwd: session.cwd,
|
|
sessionId,
|
|
sessionFile: sessionFile ?? undefined,
|
|
kernelOwnerId,
|
|
signal: combinedSignal,
|
|
session,
|
|
idleTimeoutMs,
|
|
reset: cell.reset,
|
|
onChunk: chunk => {
|
|
outputSink!.push(chunk);
|
|
},
|
|
onStatus: event => {
|
|
if (event.op === EVAL_TIMEOUT_PAUSE_OP) {
|
|
idle.pause();
|
|
return;
|
|
}
|
|
if (event.op === EVAL_TIMEOUT_RESUME_OP) {
|
|
idle.resume();
|
|
return;
|
|
}
|
|
cellResult.statusEvents ??= [];
|
|
upsertStatusEvent(cellResult.statusEvents, event);
|
|
pushUpdate();
|
|
},
|
|
});
|
|
} finally {
|
|
idle.dispose();
|
|
}
|
|
const durationMs = Date.now() - startTime;
|
|
|
|
const cellStatusEvents: EvalStatusEvent[] = [];
|
|
const cellDisplayOutputs: EvalDisplayOutput[] = [];
|
|
const cellImageNotes: string[] = [];
|
|
let cellHasMarkdown = false;
|
|
for (const output of result.displayOutputs) {
|
|
if (output.type === "json") {
|
|
jsonOutputs.push(output.data);
|
|
cellDisplayOutputs.push(output);
|
|
}
|
|
if (output.type === "image") {
|
|
const resized = await resizeImage(
|
|
{
|
|
type: "image",
|
|
data: output.data,
|
|
mimeType: output.mimeType,
|
|
},
|
|
{ excludeWebP },
|
|
);
|
|
const image: ImageContent = {
|
|
type: "image",
|
|
data: resized.data,
|
|
mimeType: resized.mimeType,
|
|
};
|
|
images.push(image);
|
|
cellDisplayOutputs.push({
|
|
type: "image",
|
|
data: image.data,
|
|
mimeType: image.mimeType,
|
|
});
|
|
const dimensionNote = formatDimensionNote(resized);
|
|
if (dimensionNote) {
|
|
cellImageNotes.push(`display image ${cellImageNotes.length + 1}: ${dimensionNote}`);
|
|
}
|
|
}
|
|
if (output.type === "status") {
|
|
upsertStatusEvent(statusEvents, output.event);
|
|
upsertStatusEvent(cellStatusEvents, output.event);
|
|
}
|
|
if (output.type === "markdown") {
|
|
cellHasMarkdown = true;
|
|
}
|
|
}
|
|
|
|
const stdoutTrimmed = result.output.trim();
|
|
const imageText = cellImageNotes.join("\n");
|
|
const displayText = formatDisplayOutputsForText(cellDisplayOutputs);
|
|
const visibleDisplayText =
|
|
displayText && imageText ? `${displayText}\n\n${imageText}` : displayText || imageText;
|
|
const cellOutput =
|
|
stdoutTrimmed && visibleDisplayText
|
|
? `${stdoutTrimmed}\n\n${visibleDisplayText}`
|
|
: stdoutTrimmed || visibleDisplayText;
|
|
cellResult.output = cellOutput;
|
|
cellResult.exitCode = result.exitCode;
|
|
cellResult.durationMs = durationMs;
|
|
cellResult.statusEvents = cellStatusEvents.length > 0 ? cellStatusEvents : undefined;
|
|
cellResult.hasMarkdown = cellHasMarkdown || undefined;
|
|
|
|
let combinedCellOutput = "";
|
|
if (cells.length > 1) {
|
|
const cellHeader = `[${i + 1}/${cells.length}]`;
|
|
const cellTitle = cell.title ? ` ${cell.title}` : "";
|
|
if (cellOutput) {
|
|
combinedCellOutput = `${cellHeader}${cellTitle}\n${cellOutput}`;
|
|
} else {
|
|
combinedCellOutput = `${cellHeader}${cellTitle} (ok)`;
|
|
}
|
|
cellOutputs.push(combinedCellOutput);
|
|
} else if (cellOutput) {
|
|
combinedCellOutput = cellOutput;
|
|
cellOutputs.push(combinedCellOutput);
|
|
}
|
|
|
|
if (combinedCellOutput) {
|
|
const prefix = cellOutputs.length > 1 ? "\n\n" : "";
|
|
appendTail(`${prefix}${combinedCellOutput}`);
|
|
}
|
|
|
|
if (result.cancelled) {
|
|
cellResult.status = "error";
|
|
pushUpdate();
|
|
const errorMsg = result.output || "Command aborted";
|
|
const combinedOutput = cellOutputs.join("\n\n");
|
|
const outputText =
|
|
cells.length > 1
|
|
? `${combinedOutput}\n\nCell ${i + 1} aborted: ${errorMsg}`
|
|
: combinedOutput || errorMsg;
|
|
|
|
const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput);
|
|
const details: EvalToolDetails = {
|
|
language: languages[0],
|
|
languages,
|
|
cells: cellResults,
|
|
jsonOutputs: jsonOutputs.length > 0 ? jsonOutputs : undefined,
|
|
statusEvents: statusEvents.length > 0 ? statusEvents : undefined,
|
|
isError: true,
|
|
};
|
|
if (notice) details.notice = notice;
|
|
|
|
return toolResult(details)
|
|
.content([{ type: "text", text: outputText }, ...images])
|
|
.truncationFromSummary(summaryForMeta, { direction: "tail" })
|
|
.done();
|
|
}
|
|
|
|
if (result.exitCode !== 0 && result.exitCode !== undefined) {
|
|
cellResult.status = "error";
|
|
pushUpdate();
|
|
const combinedOutput = cellOutputs.join("\n\n");
|
|
const outputText =
|
|
cells.length > 1
|
|
? `${combinedOutput}\n\nCell ${i + 1} failed (exit code ${result.exitCode}). Earlier cells succeeded—their state persists. Fix only cell ${i + 1}.`
|
|
: combinedOutput
|
|
? `${combinedOutput}\n\nCommand exited with code ${result.exitCode}`
|
|
: `Command exited with code ${result.exitCode}`;
|
|
|
|
const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput);
|
|
const details: EvalToolDetails = {
|
|
language: languages[0],
|
|
languages,
|
|
cells: cellResults,
|
|
jsonOutputs: jsonOutputs.length > 0 ? jsonOutputs : undefined,
|
|
statusEvents: statusEvents.length > 0 ? statusEvents : undefined,
|
|
isError: true,
|
|
};
|
|
if (notice) details.notice = notice;
|
|
|
|
return toolResult(details)
|
|
.content([{ type: "text", text: outputText }, ...images])
|
|
.truncationFromSummary(summaryForMeta, { direction: "tail" })
|
|
.done();
|
|
}
|
|
|
|
cellResult.status = "complete";
|
|
pushUpdate();
|
|
}
|
|
|
|
const combinedOutput = cellOutputs.join("\n\n");
|
|
const hasImages = images.length > 0;
|
|
const outputText =
|
|
combinedOutput ||
|
|
(hasImages
|
|
? `(displayed ${images.length} image${images.length === 1 ? "" : "s"}; no text output)`
|
|
: "(no output)");
|
|
const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput);
|
|
|
|
const details: EvalToolDetails = {
|
|
language: languages[0],
|
|
languages,
|
|
cells: cellResults,
|
|
jsonOutputs: jsonOutputs.length > 0 ? jsonOutputs : undefined,
|
|
statusEvents: statusEvents.length > 0 ? statusEvents : undefined,
|
|
};
|
|
if (notice) details.notice = notice;
|
|
|
|
return toolResult(details)
|
|
.content([{ type: "text", text: outputText }, ...images])
|
|
.truncationFromSummary(summaryForMeta, { direction: "tail" })
|
|
.done();
|
|
} finally {
|
|
if (!outputDumped) {
|
|
try {
|
|
await finalizeOutput();
|
|
} catch {}
|
|
}
|
|
}
|
|
})();
|
|
|
|
return await (session.trackEvalExecution?.(execution, sessionAbortController) ?? execution);
|
|
}
|
|
}
|
|
|
|
async function summarizeFinal(
|
|
combinedOutput: string,
|
|
finalizeOutput: () => Promise<OutputSummary | undefined>,
|
|
): Promise<OutputSummary> {
|
|
const rawSummary = (await finalizeOutput()) ?? {
|
|
output: "",
|
|
truncated: false,
|
|
totalLines: 0,
|
|
totalBytes: 0,
|
|
outputLines: 0,
|
|
outputBytes: 0,
|
|
};
|
|
const outputLines = combinedOutput.length > 0 ? combinedOutput.split("\n").length : 0;
|
|
const outputBytes = Buffer.byteLength(combinedOutput, "utf-8");
|
|
const missingLines = Math.max(0, rawSummary.totalLines - rawSummary.outputLines);
|
|
const missingBytes = Math.max(0, rawSummary.totalBytes - rawSummary.outputBytes);
|
|
return {
|
|
output: combinedOutput,
|
|
truncated: rawSummary.truncated,
|
|
totalLines: outputLines + missingLines,
|
|
totalBytes: outputBytes + missingBytes,
|
|
outputLines,
|
|
outputBytes,
|
|
artifactId: rawSummary.artifactId,
|
|
};
|
|
}
|