From 84ec8fba499ea92a05482e5767936aee5f48cb33 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 16 May 2026 19:33:18 +0200 Subject: [PATCH] feat(coding-agent/eval): implemented JSON cell-based eval tool inputs - Removed the legacy `parseEvalInput` parser module and `eval.lark`, eliminating `*** Cell` stream parsing. - Replaced eval tool arguments from single `input` strings to ordered `cells` arrays in tool calls and schema. - Updated execution to resolve language explicitly, map `py` to `python`, and apply timeout/reset defaults. - Removed backend sniffing and `ABORT_WARNING` suffix handling, then updated docs and tests to the new JSON cells format. --- packages/coding-agent/src/eval/eval.lark | 36 -- packages/coding-agent/src/eval/index.ts | 2 - packages/coding-agent/src/eval/parse.ts | 407 ------------------ packages/coding-agent/src/eval/sniff.ts | 28 -- .../coding-agent/src/prompts/tools/eval.md | 51 ++- packages/coding-agent/src/tools/eval.ts | 183 ++++---- packages/coding-agent/test/eval/parse.test.ts | 352 --------------- .../test/tools/eval-display-text.test.ts | 12 +- .../test/tools/eval-fallback.test.ts | 12 +- 9 files changed, 115 insertions(+), 968 deletions(-) delete mode 100644 packages/coding-agent/src/eval/eval.lark delete mode 100644 packages/coding-agent/src/eval/parse.ts delete mode 100644 packages/coding-agent/src/eval/sniff.ts delete mode 100644 packages/coding-agent/test/eval/parse.test.ts diff --git a/packages/coding-agent/src/eval/eval.lark b/packages/coding-agent/src/eval/eval.lark deleted file mode 100644 index 5e50115ce..000000000 --- a/packages/coding-agent/src/eval/eval.lark +++ /dev/null @@ -1,36 +0,0 @@ -// Canonical Eval input. Each cell is introduced by a single header line: -// -// *** Cell :"" [t:<duration>] [rst] -// -// Attribute order is fixed: language+title, then optional timeout, then -// optional reset flag. Title may be empty (`py:""`). -// -// Tokens: -// -// py:"..." | js:"..." language plus title (required) -// t:<digits>(ms|s|m)? per-cell timeout (default 30s) -// rst reset this language's kernel before running -// -// Everything between one header line and the next (or the optional trailing -// `*** End`, or end of input) is the cell's code, verbatim. The runtime -// parser additionally accepts content before the first header as an implicit -// default-language cell, but that is lenient fallback and MUST NOT be relied -// on. - -start: cell+ end_marker - -cell: cell_header code_line* - -cell_header: "*** Cell" WS_INLINE LANG_TITLE (WS_INLINE T_ATTR)? (WS_INLINE RST_FLAG)? LF - -end_marker: "*** End" LF? - -code_line: CODE_TEXT LF | LF -CODE_TEXT: /([^*\r\n]|\*\*?[^*\r\n])+\*{0,2}|\*{1,2}/ - -LANG_TITLE: ("py" | "js") ":\"" /[^"\r\n]*/ "\"" -T_ATTR: "t:" /\d+(ms|s|m)?/ -RST_FLAG: "rst" - -%import common.LF -%import common.WS_INLINE diff --git a/packages/coding-agent/src/eval/index.ts b/packages/coding-agent/src/eval/index.ts index 4d0c0097d..5986aa401 100644 --- a/packages/coding-agent/src/eval/index.ts +++ b/packages/coding-agent/src/eval/index.ts @@ -1,6 +1,4 @@ export * from "./backend"; export { default as jsBackend } from "./js"; -export * from "./parse"; export { default as pythonBackend } from "./py"; -export * from "./sniff"; export * from "./types"; diff --git a/packages/coding-agent/src/eval/parse.ts b/packages/coding-agent/src/eval/parse.ts deleted file mode 100644 index 7e566f900..000000000 --- a/packages/coding-agent/src/eval/parse.ts +++ /dev/null @@ -1,407 +0,0 @@ -import { sniffEvalLanguage } from "./sniff"; -import type { EvalLanguage } from "./types"; - -export type EvalLanguageOrigin = "default" | "header"; - -export interface ParsedEvalCell { - index: number; - title?: string; - code: string; - language: EvalLanguage; - languageOrigin: EvalLanguageOrigin; - timeoutMs: number; - reset: boolean; -} - -export interface ParsedEvalInput { - cells: ParsedEvalCell[]; - /** - * True when the parser encountered `*** Abort` (recovery sentinel emitted - * by the agent loop's harmony-leak mitigation; see - * `docs/ERRATA-GPT5-HARMONY.md`). The cell containing the marker, if any, - * is dropped — its body is incomplete and unsafe to execute. - */ - aborted?: boolean; -} - -const DEFAULT_TIMEOUT_MS = 30_000; -const DEFAULT_LANGUAGE: EvalLanguage = "python"; - -/** - * Canonical language tokens plus common long-form aliases. The grammar - * advertises only `PY` / `JS` / `TS`, but unconstrained models reach for - * `Python` / `JavaScript` / `TypeScript` often enough that we accept them. - */ -const LANGUAGE_MAP: Record<string, EvalLanguage> = { - PY: "python", - PYTHON: "python", - IPY: "python", - IPYTHON: "python", - JS: "js", - JAVASCRIPT: "js", - TS: "js", - TYPESCRIPT: "js", -}; - -// Markers are case-insensitive, accept ≥2 leading stars (so `**Cell` and -// `*** Cell` both work), and tolerate any whitespace (including tabs) -// between tokens. Models that can't constrain-sample frequently emit minor -// variations like `**End` or `*** cell py`. -const STARS = String.raw`\*{2,}`; -// Cell header: `*** Cell <attrs...>`. The remainder of the line is captured -// and tokenized separately so we can handle quoted values. -const CELL_RE = new RegExp(`^${STARS}\\s*Cell\\b\\s*(.*)$`, "i"); -// `*** End` is a tolerated cell/file terminator. Documented as required at -// the file level in the lark grammar (the trailing `*** End` quirks GPT- -// trained models naturally produce), but optional at the parser level. -const END_RE = new RegExp(`^${STARS}\\s*End\\b.*$`, "i"); -// `*** Abort` is the harmony-leak recovery sentinel; see ABORT_WARNING. -const ABORT_RE = new RegExp(`^${STARS}\\s*Abort\\s*$`, "i"); - -/** - * Warning text appended to the eval tool result when parsing terminated on - * `*** Abort`. Tells the model that earlier cells (if any) ran normally and - * that any aborted cell needs to be re-issued. - */ -export const ABORT_WARNING = - "Tool stream truncated mid-call due to detected output corruption. Earlier cells (if any) executed normally; their state persists. Re-issue the aborted cell."; - -const DURATION_RE = /^(\d+)(ms|s|m)?$/i; - -function resolveLang(token: string | undefined): EvalLanguage | undefined { - return token ? LANGUAGE_MAP[token.toUpperCase()] : undefined; -} - -function parseDurationMs(raw: string, lineNumber: number): number { - const match = DURATION_RE.exec(raw.trim()); - if (!match) { - throw new Error( - `Eval line ${lineNumber}: invalid duration \`${raw}\`; use a number with optional ms, s, or m units.`, - ); - } - const value = Number.parseInt(match[1], 10); - const unit = (match[2] ?? "s").toLowerCase(); - if (unit === "ms") return value; - if (unit === "s") return value * 1000; - return value * 60_000; -} - -// Markdown fence wrapping a single bare cell, e.g. "```py\n...\n```" or -// "```\n...\n```". Used by models that wrap eval input in code fences. -const FENCE_OPEN_RE = /^```\s*([A-Za-z]\w*)?\s*$/; -const FENCE_CLOSE_RE = /^```\s*$/; - -/** - * Last-resort fallback when the input has no recognizable `*** Cell` header. - * Models that can't constrain-sample sometimes pass bare code or wrap it in - * a markdown fence (```py / ```python / bare ```). Treat the whole input as - * a single implicit cell, sniffing the language from the body. - */ -function parseImplicitCell(lines: string[]): ParsedEvalCell { - let body = lines.slice(); - while (body.length > 0 && body[0].trim() === "") body.shift(); - while (body.length > 0 && body[body.length - 1].trim() === "") body.pop(); - - let fenceLang: string | undefined; - if (body.length >= 2) { - const open = FENCE_OPEN_RE.exec(body[0]); - const closeIdx = body.length - 1; - if (open && FENCE_CLOSE_RE.test(body[closeIdx])) { - fenceLang = open[1]; - body = body.slice(1, closeIdx); - } - } - - const code = body.join("\n"); - const explicitLanguage = resolveLang(fenceLang); - const language = explicitLanguage ?? sniffEvalLanguage(code) ?? DEFAULT_LANGUAGE; - return { - index: 0, - title: undefined, - code, - language, - languageOrigin: explicitLanguage ? "header" : "default", - timeoutMs: DEFAULT_TIMEOUT_MS, - reset: false, - }; -} - -/** - * Tokenize a `*** Cell` header's attribute list while preserving quoted - * segments (`id:"some title"`, `py:"hi"`, single quotes too) as single - * tokens. Outer whitespace separates tokens; the quote characters - * themselves are kept verbatim so attribute parsing can strip them later. - */ -function tokenizeCellAttrs(input: string): string[] { - const tokens: string[] = []; - let i = 0; - while (i < input.length) { - while (i < input.length && /\s/.test(input[i])) i++; - if (i >= input.length) break; - let token = ""; - while (i < input.length && !/\s/.test(input[i])) { - const ch = input[i]; - if (ch === '"' || ch === "'") { - token += ch; - i++; - while (i < input.length && input[i] !== ch) { - token += input[i]; - i++; - } - if (i < input.length) { - token += input[i]; - i++; - } - } else { - token += ch; - i++; - } - } - tokens.push(token); - } - return tokens; -} - -interface CellHeader { - language: EvalLanguage | undefined; - languageOrigin: EvalLanguageOrigin; - title: string | undefined; - timeoutMs: number | undefined; - reset: boolean; -} - -/** - * Map an attribute key (from `key:value` or bare `key`) to one of the three - * canonical roles. Canonical keys: `id`, `t`, `rst`. Fallback aliases — - * accepted but not advertised in the prompt — cover common synonyms LLMs - * reach for instead of the short canonical. - */ -const ID_KEYS = new Set(["id", "title", "name", "cell", "file", "label"]); -const T_KEYS = new Set(["t", "timeout", "duration", "time"]); -const RST_KEYS = new Set(["rst", "reset"]); - -function classifyAttrKey(key: string): "id" | "t" | "rst" | null { - if (ID_KEYS.has(key)) return "id"; - if (T_KEYS.has(key)) return "t"; - if (RST_KEYS.has(key)) return "rst"; - return null; -} - -// `key:value` form. `value` may be `"..."`, `'...'`, or a bare run. -const ATTR_TOKEN_RE = /^([a-zA-Z][\w-]*)(?::(?:"([^"]*)"|'([^']*)'|(.*)))?$/; -// Bare positional duration (lenient — `t:` is canonical). -const DURATION_TOKEN_RE = /^\d+(?:ms|s|m)?$/; - -function parseBooleanFlag(value: string): boolean | undefined { - const v = value.trim().toLowerCase(); - if (v === "true" || v === "1" || v === "yes" || v === "on") return true; - if (v === "false" || v === "0" || v === "no" || v === "off") return false; - return undefined; -} - -/** - * Decode a `*** Cell` header's attribute list into language, title, - * timeout, and reset flag. - * - * Token forms (all optional, any order): - * - `py` / `js` / `ts` bare language - * - `py:"..."` / `js:"..."` / `ts:"..."` language + title shorthand - * - `id:"..."` cell title (canonical) - * - `t:<duration>` per-cell timeout (canonical) - * - `<duration>` (e.g. `30s`) bare positional duration - * - `rst` reset flag (canonical) - * - `rst:true|false|1|0|yes|no|on|off` reset flag with explicit value - * - * Fallback aliases (accepted but not advertised in the prompt): - * - id: title, name, cell, file, label - * - t: timeout, duration, time - * - rst: reset - * - * Quotes may be `"` or `'`. Truly unknown keys are silently dropped. First - * occurrence wins when a key is repeated (canonical or alias). Anything - * that doesn't classify accumulates as a positional title fragment joined - * by spaces. - */ -function parseCellHeader(rest: string, lineNumber: number): CellHeader { - const tokens = tokenizeCellAttrs(rest); - let language: EvalLanguage | undefined; - let titleAttr: string | undefined; - let positionalDurationMs: number | undefined; - let tAttr: string | undefined; - let rstAttr: string | undefined; - let bareReset = false; - const titleParts: string[] = []; - - for (const token of tokens) { - // Bare reset flag (canonical or alias). - if (RST_KEYS.has(token.toLowerCase())) { - bareReset = true; - continue; - } - - const attrMatch = ATTR_TOKEN_RE.exec(token); - if (attrMatch && token.includes(":")) { - const key = attrMatch[1].toLowerCase(); - const value = attrMatch[2] ?? attrMatch[3] ?? attrMatch[4] ?? ""; - - // Language-with-title shorthand: `py:"foo"`, `js:'bar'`, etc. - const langCandidate = resolveLang(key); - if (langCandidate) { - if (language === undefined) language = langCandidate; - if (titleAttr === undefined && value !== "") titleAttr = value; - continue; - } - - const role = classifyAttrKey(key); - if (role === "id" && titleAttr === undefined) titleAttr = value; - else if (role === "t" && tAttr === undefined) tAttr = value; - else if (role === "rst" && rstAttr === undefined) rstAttr = value; - // unknown / repeated keys silently dropped - continue; - } - - // Bare language token (no colon). - const lang = resolveLang(token); - if (lang && language === undefined) { - language = lang; - continue; - } - - // Bare positional duration (lenient — `t:` is canonical). - if (positionalDurationMs === undefined && DURATION_TOKEN_RE.test(token)) { - positionalDurationMs = parseDurationMs(token, lineNumber); - continue; - } - - titleParts.push(token); - } - - const explicitTitle = (titleAttr ?? "").trim(); - const positionalTitle = titleParts.join(" ").trim(); - const title = explicitTitle.length > 0 ? explicitTitle : positionalTitle.length > 0 ? positionalTitle : undefined; - - let timeoutMs: number | undefined; - if (tAttr !== undefined) { - timeoutMs = parseDurationMs(tAttr, lineNumber); - } else if (positionalDurationMs !== undefined) { - timeoutMs = positionalDurationMs; - } - - let reset = false; - if (rstAttr !== undefined) { - const parsed = parseBooleanFlag(rstAttr); - if (parsed === undefined) { - throw new Error(`Eval line ${lineNumber}: invalid rst value \`${rstAttr}\`; use true or false.`); - } - reset = parsed; - } else if (bareReset) { - reset = true; - } - - return { - language, - languageOrigin: language ? "header" : "default", - title, - timeoutMs, - reset, - }; -} - -export function parseEvalInput(input: string): ParsedEvalInput { - const normalized = input.replace(/\r\n?/g, "\n"); - const lines = normalized.split("\n"); - if (lines.length > 0 && lines[lines.length - 1] === "") lines.pop(); - - const cells: ParsedEvalCell[] = []; - let aborted = false; - let i = 0; - - // Skip leading blank lines. - while (i < lines.length && lines[i].trim() === "") i++; - - // Lenient fallback: if the input has no recognizable cell header, treat - // the entire input as one implicit cell — unless that content contains - // `*** Abort`, in which case the body is incomplete/unsafe and we drop it. - if (i < lines.length && !CELL_RE.test(lines[i])) { - const tail = lines.slice(i); - if (tail.some(line => ABORT_RE.test(line))) { - return { cells, aborted: true }; - } - const cell = parseImplicitCell(tail); - if (cell.code.length > 0) cells.push(cell); - return { cells }; - } - - while (i < lines.length) { - const headerLine = lines[i]; - const cellMatch = CELL_RE.exec(headerLine); - if (!cellMatch) { - // Stray content between/after cells (blank lines were already - // consumed). `*** Abort` here terminates parsing; `*** End` is - // the optional file-level terminator (silently consumed). Anything - // else — typically a harmony-leak fragment — is skipped. - if (ABORT_RE.test(headerLine)) { - aborted = true; - break; - } - i++; - continue; - } - const header = parseCellHeader(cellMatch[1] ?? "", i + 1); - i++; - - // Collect cell body. Close on `*** End` (any form), the next - // `*** Cell` header, or `*** Abort` (which drops the in-progress - // cell as its body is partial and unsafe to run). - const codeLines: string[] = []; - let cellAborted = false; - while (i < lines.length) { - const line = lines[i]; - if (ABORT_RE.test(line)) { - cellAborted = true; - aborted = true; - i++; - break; - } - if (END_RE.test(line)) { - i++; - break; - } - if (CELL_RE.test(line)) break; - codeLines.push(line); - i++; - } - - if (cellAborted) break; - - // Strip trailing blank lines so visual spacing between cells doesn't - // leak into the preceding cell's code. - while (codeLines.length > 0 && codeLines[codeLines.length - 1].trim() === "") { - codeLines.pop(); - } - const code = codeLines.join("\n"); - - const language = header.language ?? sniffEvalLanguage(code) ?? DEFAULT_LANGUAGE; - const languageOrigin: EvalLanguageOrigin = header.language ? "header" : "default"; - - cells.push({ - index: cells.length, - title: header.title, - code, - language, - languageOrigin, - timeoutMs: header.timeoutMs ?? DEFAULT_TIMEOUT_MS, - reset: header.reset, - }); - - // Skip blank separator lines between cells; an `*** Abort` here - // terminates parsing while keeping previously-collected cells. - while (i < lines.length && lines[i].trim() === "") i++; - if (i < lines.length && ABORT_RE.test(lines[i])) { - aborted = true; - break; - } - } - - return aborted ? { cells, aborted: true } : { cells }; -} diff --git a/packages/coding-agent/src/eval/sniff.ts b/packages/coding-agent/src/eval/sniff.ts deleted file mode 100644 index 399e42b66..000000000 --- a/packages/coding-agent/src/eval/sniff.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { EvalLanguage } from "./types"; - -/** - * Best-effort language sniff for cells with no explicit `language`. - * - * Order: - * 1. Shebang on first line (`#!/usr/bin/env python`, `#!/usr/bin/env node`, etc.) - * 2. Strong syntactic markers unique to one language. Bias false negatives over - * false positives — anything ambiguous returns `undefined` and the caller - * falls back to the default-backend rules. - */ -export function sniffEvalLanguage(code: string): EvalLanguage | undefined { - const stripped = code.replace(/^\s+/, ""); - if (stripped.startsWith("#!")) { - const firstLine = stripped.split("\n", 1)[0]!.toLowerCase(); - if (/(\bpython\d?\b|\bipython\b)/.test(firstLine)) return "python"; - if (/(\bnode\b|\bbun\b|\bdeno\b|\bjavascript\b|\bjs\b)/.test(firstLine)) return "js"; - } - const jsMarkers = - /(^|\n)\s*(const|let|var|async\s+function|function\s*\*?\s*[\w$]*\s*\(|import\s+[^\n]+\sfrom\s|export\s+(default|const|let|function|class|async)|require\s*\(|console\.\w+\s*\(|=>|;\s*$)/m; - const pyMarkers = - /(^|\n)\s*(def\s+\w+\s*\(|from\s+[\w.]+\s+import|import\s+\w+(\s+as\s+\w+)?\s*$|class\s+\w+\s*[(:]|print\s*\(|elif\s+[^\n]*:|with\s+[^\n]+:\s*$|@[\w.]+\s*$)/m; - const hasJs = jsMarkers.test(code); - const hasPy = pyMarkers.test(code); - if (hasJs && !hasPy) return "js"; - if (hasPy && !hasJs) return "python"; - return undefined; -} diff --git a/packages/coding-agent/src/prompts/tools/eval.md b/packages/coding-agent/src/prompts/tools/eval.md index d20af720f..78dec551d 100644 --- a/packages/coding-agent/src/prompts/tools/eval.md +++ b/packages/coding-agent/src/prompts/tools/eval.md @@ -1,25 +1,22 @@ -Run code in a persistent kernel using codeblock cells. +Run code in a persistent kernel using a list of cells. <instruction> -Each cell starts with a single header line and runs until the next header (or end of input): +Each call submits one or more cells. Cells run in array order. State persists within each language across cells **and across tool calls**. -``` -*** Cell py:"optional title" t:10s rst -print("hi") -``` +Cell fields: -- **Language + title**: `<lang>:"<title>"` — {{#if py}}`py` for Python{{/if}}{{#ifAll py js}}, {{/ifAll}}{{#if js}}`js` for JavaScript{{/if}}. Title may be empty (`py:""`). -- **Attributes** (optional, in this order, after the language+title): - - `t:<duration>` — per-cell timeout. Digits with optional `ms` / `s` / `m` units (e.g. `500ms`, `15s`, `2m`). Default 30s. - - `rst` — wipe this cell's own language kernel before running.{{#ifAll py js}} Other languages are untouched.{{/ifAll}} -- Anything after the header line, up to the next `*** Cell` header, is the cell's code, verbatim. -- Stack multiple cells back-to-back; blank lines between cells are ignored. +- `language` — {{#if py}}`"py"` for the IPython kernel{{/if}}{{#ifAll py js}}, {{/ifAll}}{{#if js}}`"js"` for the persistent JavaScript VM{{/if}}. +- `code` — cell body, verbatim. Newlines, quotes, and indentation are JSON-encoded; no fences, no headers. +- `title` (optional) — short label shown in the transcript (e.g. `"imports"`, `"load config"`). +- `timeout` (optional) — per-cell timeout in seconds (1-600). Default 30. +- `reset` (optional) — wipe this cell's language kernel before running.{{#ifAll py js}} Reset is per-language: a `py` cell's reset does not touch the JavaScript VM and vice versa.{{/ifAll}} **Work incrementally:** + - One logical step per cell (imports, define, test, use). - Pass multiple small cells in one call. - Define small reusable functions for individual debugging. -- Put workflow explanations in the assistant message or cell title — never inside cell code. +- Put workflow explanations in the assistant message or `title` — never inside cell code. {{#if py}}- Python cells run inside an IPython kernel with a live event loop. Use top-level `await` directly (e.g. `await main()`); `asyncio.run(…)` raises "cannot be called from a running event loop".{{/if}} **On failure:** errors identify the failing cell (e.g., "Cell 3 failed"). Resubmit only the fixed cell (or fixed cell + remaining cells). </instruction> @@ -55,22 +52,24 @@ Cells render like a Jupyter notebook. `display(value)` renders non-presentable d </output> <caution> -- In session mode, use `rst` on a cell to wipe its language's kernel before running.{{#ifAll py js}} Reset is per-language: a python cell's `rst` does not touch the JavaScript kernel and vice versa.{{/ifAll}} {{#if js}}- **js**: the VM exposes a selective `process` subset, Web APIs, `Buffer`, `fs/promises`, and the `Bun` global. {{/if}}</caution> <example> -{{#if py}}*** Cell py:"imports" t:10s -import json -from pathlib import Path +{{#if py}}```json +{ + "cells": [ + { "language": "py", "title": "imports", "timeout": 10, "code": "import json\nfrom pathlib import Path" }, + { "language": "py", "title": "load config", "code": "data = json.loads(read('package.json'))\ndisplay(data)" } + ] +} +```{{/if}}{{#ifAll py js}} -*** Cell py:"load config" -data = json.loads(read('package.json')) -display(data) -{{/if}}{{#ifAll py js}} -{{/ifAll}}{{#if js}}*** Cell js:"summary" rst -const data = JSON.parse(await read('package.json')); -display(data); -return data.name; -{{/if}} +{{/ifAll}}{{#if js}}```json +{ + "cells": [ + { "language": "js", "title": "summary", "reset": true, "code": "const data = JSON.parse(await read('package.json'));\ndisplay(data);\nreturn data.name;" } + ] +} +```{{/if}} </example> diff --git a/packages/coding-agent/src/tools/eval.ts b/packages/coding-agent/src/tools/eval.ts index 805091c66..a931c45fb 100644 --- a/packages/coding-agent/src/tools/eval.ts +++ b/packages/coding-agent/src/tools/eval.ts @@ -4,10 +4,8 @@ import type { Component } from "@oh-my-pi/pi-tui"; import { Markdown, Text } from "@oh-my-pi/pi-tui"; import { prompt } from "@oh-my-pi/pi-utils"; import * as z from "zod/v4"; -import { jsBackend, parseEvalInput, pythonBackend, sniffEvalLanguage } from "../eval"; +import { jsBackend, pythonBackend } from "../eval"; import type { ExecutorBackend } from "../eval/backend"; -import evalGrammar from "../eval/eval.lark" with { type: "text" }; -import { ABORT_WARNING, type ParsedEvalCell } from "../eval/parse"; import type { EvalCellResult, EvalDisplayOutput, EvalLanguage, EvalStatusEvent, EvalToolDetails } from "../eval/types"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import { truncateToVisualLines } from "../modes/components/visual-truncate"; @@ -29,8 +27,27 @@ import { clampTimeout } from "./tool-timeouts"; export const EVAL_DEFAULT_PREVIEW_LINES = 10; +/** + * Per-cell input. Each cell runs in order; state persists within a language + * across cells and across tool calls. + */ +const evalCellSchema = z.object({ + language: z.enum(["py", "js"]).describe('runtime: "py" for the IPython kernel, "js" for the persistent JS VM'), + code: z.string().describe("cell body, verbatim. Use top-level await freely."), + title: z.string().optional().describe('short label shown in transcript (e.g. "imports", "load config")'), + timeout: z.number().int().min(1).max(600).optional().describe("per-cell timeout in seconds (1-600, default 30)"), + reset: z + .boolean() + .optional() + .describe("wipe this cell's language kernel before running. Other languages are untouched."), +}); +export type EvalCellInput = z.infer<typeof evalCellSchema>; + export const evalSchema = z.object({ - input: z.string().describe('eval input as a sequence of `*** Cell <lang>:"title"` cell headers followed by code'), + cells: z + .array(evalCellSchema) + .min(1) + .describe("cells executed in order. State persists within each language across cells and tool calls."), }); export type EvalToolParams = z.infer<typeof evalSchema>; @@ -134,7 +151,6 @@ export interface EvalToolOptions { interface ResolvedBackend { backend: ExecutorBackend; - fallback: boolean; notice?: string; } @@ -166,51 +182,21 @@ function timeoutSecondsFromMs(timeoutMs: number): number { return clampTimeout("eval", timeoutMs / 1000); } -async function resolveBackend( - session: ToolSession, - requested: EvalLanguage | undefined, - code: string, -): Promise<ResolvedBackend> { +async function resolveBackend(session: ToolSession, language: EvalLanguage): Promise<ResolvedBackend> { const allowPy = (session.settings.get("eval.py") as boolean | undefined) ?? true; const allowJs = (session.settings.get("eval.js") as boolean | undefined) ?? true; - if (requested === "python") { + if (language === "python") { if (!allowPy) throw new ToolError("Python backend is disabled (eval.py = false)."); if (!(await pythonBackend.isAvailable(session))) { throw new ToolError( 'Python backend is unavailable in this session. Pass language: "js" or install the python kernel.', ); } - return { backend: pythonBackend, fallback: false }; + return { backend: pythonBackend }; } - if (requested === "js") { - if (!allowJs) throw new ToolError("JavaScript backend is disabled (eval.js = false)."); - return { backend: jsBackend, fallback: false }; - } - // Auto-detect. - const sniffed = sniffEvalLanguage(code); - if (sniffed === "python" && allowPy && (await pythonBackend.isAvailable(session))) { - return { backend: pythonBackend, fallback: false }; - } - if (sniffed === "js" && allowJs) { - return { backend: jsBackend, fallback: false }; - } - - // Sniffer returned undefined or the preferred backend was disabled. Prefer - // python when its kernel is up, else fall back to js. - if (allowPy && (await pythonBackend.isAvailable(session))) { - const notice = - sniffed === "js" ? "JavaScript markers detected but eval.js is disabled; using Python." : undefined; - return { backend: pythonBackend, fallback: false, notice }; - } - if (allowJs) { - const notice = - sniffed === "python" - ? "Python markers detected but the python kernel is unavailable; using JavaScript." - : undefined; - return { backend: jsBackend, fallback: true, notice }; - } - throw new ToolError("No eval backend is available; enable eval.py or eval.js."); + if (!allowJs) throw new ToolError("JavaScript backend is disabled (eval.js = false)."); + return { backend: jsBackend }; } export class EvalTool implements AgentTool<typeof evalSchema> { @@ -227,20 +213,15 @@ export class EvalTool implements AgentTool<typeof evalSchema> { readonly concurrency = "exclusive"; readonly strict = true; readonly intent = (args: Partial<z.infer<typeof evalSchema>>): string | undefined => { - const input = args.input; - if (input) { - try { - const cells = parseEvalInput(input).cells; - return cells.map(cell => cell.title || `running ${cell.language}`).join("\n"); - } catch {} - } - return "evaluating"; + const cells = Array.isArray(args.cells) ? args.cells : []; + const first = cells.find(c => c && typeof c === "object"); + if (!first) return "evaluating"; + const title = typeof first.title === "string" ? first.title : undefined; + const language = typeof first.language === "string" ? first.language : "?"; + const label = title || `running ${language}`; + return cells.length > 1 ? `${label} (+${cells.length - 1})` : label; }; - get customFormat(): { syntax: "lark"; definition: string } { - return { syntax: "lark", definition: evalGrammar }; - } - readonly #proxyExecutor?: EvalProxyExecutor; constructor( @@ -266,19 +247,17 @@ export class EvalTool implements AgentTool<typeof evalSchema> { } const session = this.session; - const parsedInput = parseEvalInput(params.input); - let previousRuntimeLanguage: EvalLanguage | undefined; const cells: ResolvedEvalCell[] = []; - for (const cell of parsedInput.cells) { - const requested = cell.languageOrigin === "header" ? cell.language : (previousRuntimeLanguage ?? undefined); - const resolved = await resolveBackend(session, requested, cell.code); - previousRuntimeLanguage = resolved.backend.id; + for (let i = 0; i < params.cells.length; i++) { + const cell = params.cells[i]; + const language: EvalLanguage = cell.language === "py" ? "python" : "js"; + const resolved = await resolveBackend(session, language); cells.push({ - index: cell.index, + index: i, title: cell.title, code: cell.code, - timeoutMs: cell.timeoutMs, - reset: cell.reset, + timeoutMs: (cell.timeout ?? 30) * 1000, + reset: cell.reset ?? false, resolved, }); } @@ -462,11 +441,10 @@ export class EvalTool implements AgentTool<typeof evalSchema> { pushUpdate(); const errorMsg = result.output || "Command aborted"; const combinedOutput = cellOutputs.join("\n\n"); - const abortSuffix = parsedInput.aborted ? `\n\n${ABORT_WARNING}` : ""; const outputText = - (cells.length > 1 + cells.length > 1 ? `${combinedOutput}\n\nCell ${i + 1} aborted: ${errorMsg}` - : combinedOutput || errorMsg) + abortSuffix; + : combinedOutput || errorMsg; const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput); const details: EvalToolDetails = { @@ -489,13 +467,12 @@ export class EvalTool implements AgentTool<typeof evalSchema> { cellResult.status = "error"; pushUpdate(); const combinedOutput = cellOutputs.join("\n\n"); - const abortSuffix = parsedInput.aborted ? `\n\n${ABORT_WARNING}` : ""; const outputText = - (cells.length > 1 + cells.length > 1 ? `${combinedOutput}\n\nCell ${i + 1} failed (exit code ${result.exitCode}). Earlier cells succeeded—their state persists. Fix only cell ${i + 1}.` : combinedOutput ? `${combinedOutput}\n\nCommand exited with code ${result.exitCode}` - : `Command exited with code ${result.exitCode}`) + abortSuffix; + : `Command exited with code ${result.exitCode}`; const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput); const details: EvalToolDetails = { @@ -519,13 +496,12 @@ export class EvalTool implements AgentTool<typeof evalSchema> { } const combinedOutput = cellOutputs.join("\n\n"); - const abortSuffix = parsedInput.aborted ? `\n\n${ABORT_WARNING}` : ""; const hasImages = images.length > 0; const outputText = - (combinedOutput || - (hasImages - ? `(displayed ${images.length} image${images.length === 1 ? "" : "s"}; no text output)` - : "(no output)")) + abortSuffix; + combinedOutput || + (hasImages + ? `(displayed ${images.length} image${images.length === 1 ? "" : "s"}; no text output)` + : "(no output)"); const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput); const details: EvalToolDetails = { @@ -581,8 +557,14 @@ async function summarizeFinal( }; } +interface EvalRenderCellArg { + language?: string; + code?: string; + title?: string; +} + interface EvalRenderArgs { - input?: string; + cells?: EvalRenderCellArg[]; __partialJson?: string; } @@ -593,27 +575,30 @@ interface EvalRenderContext { timeout?: number; } -function decodePartialJsonStringFragment(fragment: string): string { - let text = fragment.replace(/\\u[0-9a-fA-F]{0,3}$/, ""); - const trailingBackslashes = text.match(/\\+$/)?.[0].length ?? 0; - if (trailingBackslashes % 2 === 1) text = text.slice(0, -1); - try { - return JSON.parse(`"${text}"`) as string; - } catch { - return text; +interface EvalRenderCell { + language: EvalLanguage; + code: string; + title?: string; +} + +function normalizeRenderLanguage(value: string | undefined): EvalLanguage { + return value === "js" ? "js" : "python"; +} + +function getRenderCells(args: EvalRenderArgs | undefined): EvalRenderCell[] { + const raw = args?.cells; + if (!Array.isArray(raw)) return []; + const out: EvalRenderCell[] = []; + for (const cell of raw) { + if (!cell || typeof cell !== "object") continue; + const code = typeof cell.code === "string" ? cell.code : ""; + out.push({ + language: normalizeRenderLanguage(typeof cell.language === "string" ? cell.language : undefined), + code, + title: typeof cell.title === "string" ? cell.title : undefined, + }); } -} - -function extractPartialJsonString(partialJson: string | undefined, key: string): string | undefined { - if (!partialJson) return undefined; - const pattern = new RegExp(`"${key}"\\s*:\\s*"((?:\\\\.|[^"\\\\])*)`, "u"); - const match = pattern.exec(partialJson); - if (!match) return undefined; - return decodePartialJsonStringFragment(match[1]); -} - -function getRenderInput(args: EvalRenderArgs | undefined): string | undefined { - return args?.input ?? extractPartialJsonString(args?.__partialJson, "input"); + return out; } /** Format a status event as a single line for display. */ @@ -861,15 +846,7 @@ function formatCellOutputLines( export const evalToolRenderer = { renderCall(args: EvalRenderArgs, _options: RenderResultOptions, uiTheme: Theme): Component { - const input = getRenderInput(args); - let cells: ParsedEvalCell[] = []; - if (input) { - try { - cells = parseEvalInput(input).cells; - } catch { - cells = []; - } - } + const cells = getRenderCells(args); if (cells.length === 0) { const promptSym = uiTheme.fg("accent", ">>>"); @@ -881,7 +858,7 @@ export const evalToolRenderer = { return { render: (width: number): string[] => { - const key = `${input?.length ?? 0}`; + const key = cells.map(c => `${c.language}:${c.title ?? ""}:${c.code.length}`).join("|"); if (cached && cached.key === key && cached.width === width) { return cached.result; } diff --git a/packages/coding-agent/test/eval/parse.test.ts b/packages/coding-agent/test/eval/parse.test.ts deleted file mode 100644 index 99d56e86f..000000000 --- a/packages/coding-agent/test/eval/parse.test.ts +++ /dev/null @@ -1,352 +0,0 @@ -import { describe, expect, it } from "bun:test"; -import { parseEvalInput } from "../../src/eval/parse"; - -describe("parseEvalInput", () => { - it("parses a single cell with title and timeout", () => { - const result = parseEvalInput(`*** Cell py:"setup" t:10s -print("hi") -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0]).toMatchObject({ - index: 0, - title: "setup", - language: "python", - languageOrigin: "header", - timeoutMs: 10_000, - reset: false, - code: 'print("hi")', - }); - }); - - it("treats rst as a per-cell kernel wipe", () => { - const result = parseEvalInput(`*** Cell py:"bootstrap" -import json - -*** Cell js:"" rst -const x = 1; -`); - expect(result.cells).toHaveLength(2); - expect(result.cells[0].reset).toBe(false); - expect(result.cells[1].reset).toBe(true); - expect(result.cells[1].language).toBe("js"); - }); - - it("accepts case-insensitive language tokens (lenient)", () => { - const result = parseEvalInput(`*** Cell JS:"" -const a = 1; -*** Cell PY:"" -print("py") -`); - expect(result.cells).toHaveLength(2); - expect(result.cells[0].language).toBe("js"); - expect(result.cells[1].language).toBe("python"); - }); - - it("parses millisecond, second, and minute durations", () => { - const result = parseEvalInput(`*** Cell py:"a" t:500ms -a = 1 -*** Cell py:"b" t:5 -a = 2 -*** Cell py:"c" t:2m -a = 3 -`); - expect(result.cells.map(c => c.timeoutMs)).toEqual([500, 5000, 120_000]); - }); - - it("preserves blank lines inside the cell body", () => { - const result = parseEvalInput(`*** Cell js:"" -const x = 1; - -const y = 2; -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].code).toBe("const x = 1;\n\nconst y = 2;"); - }); - - it("treats blank lines between cells as separators, not code", () => { - const result = parseEvalInput(`*** Cell py:"" -print("a") - - -*** Cell py:"" -print("b") -`); - expect(result.cells).toHaveLength(2); - expect(result.cells[0].code).toBe('print("a")'); - expect(result.cells[1].code).toBe('print("b")'); - }); - - it("falls back to language sniffing when the header has no recognized language", () => { - // Bare `ruby` doesn't match LANG_TITLE, but the parser is lenient and - // falls back to body sniffing. - const result = parseEvalInput(`*** Cell ruby:"x" -const x = 1; -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].languageOrigin).toBe("default"); - expect(result.cells[0].language).toBe("js"); - }); - - it("accepts `**Cell` (two stars) as well as `***Cell`", () => { - const result = parseEvalInput(`**Cell py:"" -print(1) -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].language).toBe("python"); - }); - - it("implicitly closes a cell when a new *** Cell appears without *** End", () => { - const result = parseEvalInput(`*** Cell py:"" -print("a") -*** Cell js:"" -const x = 1; -`); - expect(result.cells).toHaveLength(2); - expect(result.cells[0].code).toBe('print("a")'); - expect(result.cells[1].code).toBe("const x = 1;"); - }); - - it("tolerates `*** End` as an optional cell terminator (GPT quirk)", () => { - const result = parseEvalInput(`*** Cell py:"" -print(1) -*** End -*** Cell js:"" -const x = 1; -*** End -`); - expect(result.cells).toHaveLength(2); - expect(result.cells[0].code).toBe("print(1)"); - expect(result.cells[1].code).toBe("const x = 1;"); - }); - - it("ignores anything trailing `*** End` (leniency)", () => { - const result = parseEvalInput(`*** Cell py:"" -print(1) -*** End py -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].code).toBe("print(1)"); - }); - - it("accepts long-form language aliases (Python, JavaScript, TypeScript)", () => { - const result = parseEvalInput(`*** Cell Python:"" -print(1) -*** Cell JavaScript:"" -const a = 1; -*** Cell TypeScript:"" -const b = 2; -`); - expect(result.cells.map(c => c.language)).toEqual(["python", "js", "js"]); - }); - - it("implicitly closes the final cell at EOF when *** End is missing", () => { - const result = parseEvalInput(`*** Cell py:"" -print(1) -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].code).toBe("print(1)"); - }); - - it("treats bare code without any *** Cell as a single implicit cell", () => { - const result = parseEvalInput(`def greet():\n print('hi')\ngreet()\n`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0]).toMatchObject({ - languageOrigin: "default", - language: "python", - code: "def greet():\n print('hi')\ngreet()", - }); - }); - - it("strips a markdown code fence wrapper and uses its language tag", () => { - const result = parseEvalInput("```js\nconst x = 1;\n```\n"); - expect(result.cells).toHaveLength(1); - expect(result.cells[0]).toMatchObject({ - language: "js", - languageOrigin: "header", - code: "const x = 1;", - }); - }); - - it("rejects invalid duration", () => { - expect(() => - parseEvalInput(`*** Cell py:"" t:forever -print(1) -`), - ).toThrow(/invalid duration/); - }); - - it("supports titles with embedded spaces", () => { - const result = parseEvalInput(`*** Cell py:"load and validate config" -print(1) -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].title).toBe("load and validate config"); - }); - - it('treats empty title (`py:""`) as no title', () => { - const result = parseEvalInput(`*** Cell py:"" -print(1) -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].title).toBeUndefined(); - expect(result.cells[0].language).toBe("python"); - }); - - it("accepts bare language token without title (lenient form)", () => { - // Parser is more permissive than the lark; bare `py` is accepted - // even though the canonical form is `py:"title"`. - const result = parseEvalInput(`*** Cell py -print(1) -`); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].language).toBe("python"); - expect(result.cells[0].title).toBeUndefined(); - }); - - describe("attribute leniency (accepted but not advertised)", () => { - it("accepts id aliases (title/name/cell/file/label)", () => { - const aliases = ["title", "name", "cell", "file", "label"]; - for (const alias of aliases) { - const result = parseEvalInput(`*** Cell py ${alias}:"hi"\nprint(1)\n`); - expect(result.cells[0].title).toBe("hi"); - } - }); - - it("accepts t aliases (timeout/duration/time)", () => { - const aliases = ["timeout", "duration", "time"]; - for (const alias of aliases) { - const result = parseEvalInput(`*** Cell py ${alias}:5s\nprint(1)\n`); - expect(result.cells[0].timeoutMs).toBe(5000); - } - }); - - it("accepts `reset` as an alias for `rst`", () => { - const result = parseEvalInput(`*** Cell py reset\nprint(1)\n`); - expect(result.cells[0].reset).toBe(true); - }); - - it("accepts `rst:true|false|1|0|yes|no|on|off`", () => { - for (const v of ["true", "1", "yes", "on"]) { - const r = parseEvalInput(`*** Cell py rst:${v}\nx\n`); - expect(r.cells[0].reset).toBe(true); - } - for (const v of ["false", "0", "no", "off"]) { - const r = parseEvalInput(`*** Cell py rst:${v}\nx\n`); - expect(r.cells[0].reset).toBe(false); - } - }); - - it("rejects an invalid rst value", () => { - expect(() => parseEvalInput(`*** Cell py rst:maybe\nx\n`)).toThrow(/invalid rst/); - }); - - it("accepts single-quoted titles (`id:'hi'`)", () => { - const result = parseEvalInput(`*** Cell py id:'hello world'\nprint(1)\n`); - expect(result.cells[0].title).toBe("hello world"); - }); - - it("accepts a bare positional duration token (e.g. `30s`)", () => { - const result = parseEvalInput(`*** Cell py 2m\nprint(1)\n`); - expect(result.cells[0].timeoutMs).toBe(120_000); - }); - - it("first occurrence wins for repeated keys (canonical or alias)", () => { - const result = parseEvalInput(`*** Cell py id:"first" name:"second" t:1s timeout:5s\nprint(1)\n`); - expect(result.cells[0].title).toBe("first"); - expect(result.cells[0].timeoutMs).toBe(1000); - }); - - it("unclassified bare tokens accumulate as a positional title", () => { - const result = parseEvalInput(`*** Cell py setup phase\nprint(1)\n`); - expect(result.cells[0].title).toBe("setup phase"); - }); - }); - - describe("*** Abort recovery sentinel (harmony-leak mitigation)", () => { - it("drops the in-progress cell and stops parsing", () => { - const result = parseEvalInput(`*** Cell py:"" -print("a") -*** Cell js:"" -const partial = 1; /* contamination starts mid-cell */ -*** Abort -*** Cell js:"" -const never_runs = 1; -`); - expect(result.aborted).toBe(true); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].language).toBe("python"); - expect(result.cells[0].code).toBe('print("a")'); - }); - - it("`*** End` before `*** Abort` preserves the closed cell", () => { - // Without `*** End`, the parser can't tell whether the cell was - // complete before contamination — by design, since `*** End` is - // optional and undocumented. Explicit `*** End` is the GPT quirk - // that signals "cell is closed, abort is between cells". - const result = parseEvalInput(`*** Cell py:"" -print("a") -*** End - -*** Abort - -*** Cell py:"" -print("never") -`); - expect(result.aborted).toBe(true); - expect(result.cells).toHaveLength(1); - expect(result.cells[0].code).toBe('print("a")'); - }); - - it("implicit-cell input containing *** Abort is rejected entirely", () => { - const result = parseEvalInput(`print("partial") -*** Abort -`); - expect(result.aborted).toBe(true); - expect(result.cells).toHaveLength(0); - }); - - it("appended sentinel from harmony-leak truncation: abort flag set, prior cell dropped", () => { - const truncated = `*** Cell py:""\nprint("ok")\n*** Abort\n`; - const result = parseEvalInput(truncated); - expect(result.aborted).toBe(true); - expect(result.cells).toHaveLength(0); - }); - - it("absent sentinel: aborted is undefined (not falsely set)", () => { - const result = parseEvalInput(`*** Cell py:"" -print(1) -`); - expect(result.aborted).toBeUndefined(); - }); - }); - - it("does not crash on stray non-marker lines between cells", () => { - // Regression for "null is not an object (evaluating - // 'BEGIN_RE.exec(lines[i])[1]')" — stray fragments must not crash. - // Without `*** End`, the stray junk folds into the prior cell's body; - // the contract for this test is just "don't crash". - const result = parseEvalInput(`*** Cell py:"" -print("a") -stray junk that is not a marker -*** Cell py:"" -print("b") -`); - expect(result.aborted).toBeUndefined(); - expect(result.cells).toHaveLength(2); - expect(result.cells[0].code).toContain('print("a")'); - expect(result.cells[1].code).toBe('print("b")'); - }); - - it("does not crash on trailing stray content after the final cell", () => { - const result = parseEvalInput(`*** Cell py:"" -print(1) -leftover model chatter -more junk -`); - expect(result.aborted).toBeUndefined(); - expect(result.cells).toHaveLength(1); - // Stray lines fold into the cell body (no terminator), which is fine — - // the contract is just "don't crash". - expect(result.cells[0].code).toContain("print(1)"); - }); -}); diff --git a/packages/coding-agent/test/tools/eval-display-text.test.ts b/packages/coding-agent/test/tools/eval-display-text.test.ts index 37acfdd7e..b442a25a9 100644 --- a/packages/coding-agent/test/tools/eval-display-text.test.ts +++ b/packages/coding-agent/test/tools/eval-display-text.test.ts @@ -46,7 +46,7 @@ describe("EvalTool display() text surfacing", () => { const tool = new EvalTool(makeSession()); const result = await tool.execute("call-display-json", { - input: "```js\ndisplay({ stdout: 'hi', exit_code: 0 });\n```\n", + cells: [{ language: "js", code: "```js\ndisplay({ stdout: 'hi', exit_code: 0 });\n```\n" }], }); const text = result.content.map(c => (c.type === "text" ? c.text : "")).join("\n"); @@ -67,7 +67,7 @@ describe("EvalTool display() text surfacing", () => { const tool = new EvalTool(makeSession()); const result = await tool.execute("call-mixed", { - input: "```js\nprint('before'); display([1,2,3]);\n```\n", + cells: [{ language: "js", code: "```js\nprint('before'); display([1,2,3]);\n```\n" }], }); const text = result.content.map(c => (c.type === "text" ? c.text : "")).join("\n"); @@ -87,7 +87,9 @@ describe("EvalTool display() text surfacing", () => { const tool = new EvalTool(makeSession()); const result = await tool.execute("call-image", { - input: "```js\ndisplay({ type: 'image', data: '...', mimeType: 'image/png' });\n```\n", + cells: [ + { language: "js", code: "```js\ndisplay({ type: 'image', data: '...', mimeType: 'image/png' });\n```\n" }, + ], }); const imageBlocks = result.content.filter(c => c.type === "image"); @@ -109,7 +111,7 @@ describe("EvalTool display() text surfacing", () => { const tool = new EvalTool(makeSession()); const result = await tool.execute("call-empty", { - input: "```js\nconst x = 1;\n```\n", + cells: [{ language: "js", code: "```js\nconst x = 1;\n```\n" }], }); const text = result.content.map(c => (c.type === "text" ? c.text : "")).join("\n"); @@ -127,7 +129,7 @@ describe("EvalTool display() text surfacing", () => { const tool = new EvalTool(makeSession()); const result = await tool.execute("call-huge", { - input: "```js\ndisplay({ payload: 'x'.repeat(20000) });\n```\n", + cells: [{ language: "js", code: "```js\ndisplay({ payload: 'x'.repeat(20000) });\n```\n" }], }); const text = result.content.map(c => (c.type === "text" ? c.text : "")).join("\n"); diff --git a/packages/coding-agent/test/tools/eval-fallback.test.ts b/packages/coding-agent/test/tools/eval-fallback.test.ts index 9fecf9abc..f442420ef 100644 --- a/packages/coding-agent/test/tools/eval-fallback.test.ts +++ b/packages/coding-agent/test/tools/eval-fallback.test.ts @@ -39,9 +39,7 @@ describe("EvalTool language resolution", () => { const pythonExecuteSpy = vi.spyOn(evalIndex.pythonBackend, "execute"); const tool = new EvalTool(makeSession()); - await tool.execute("call-1", { - input: "```js one\nconst x = 1;\n```\n", - }); + await tool.execute("call-1", { cells: [{ language: "js", code: "```js one\nconst x = 1;\n```\n" }] }); expect(jsExecuteSpy).toHaveBeenCalledTimes(1); expect(pythonExecuteSpy).not.toHaveBeenCalled(); @@ -53,9 +51,7 @@ describe("EvalTool language resolution", () => { const jsExecuteSpy = vi.spyOn(evalIndex.jsBackend, "execute"); const tool = new EvalTool(makeSession()); - await tool.execute("call-2", { - input: "```python one\nprint('hi')\n```\n", - }); + await tool.execute("call-2", { cells: [{ language: "js", code: "```python one\nprint('hi')\n```\n" }] }); expect(pythonExecuteSpy).toHaveBeenCalledTimes(1); expect(jsExecuteSpy).not.toHaveBeenCalled(); @@ -67,9 +63,7 @@ describe("EvalTool language resolution", () => { const jsExecuteSpy = vi.spyOn(evalIndex.jsBackend, "execute"); const tool = new EvalTool(makeSession()); - await tool.execute("call-3", { - input: "def greet():\n print('hi')\ngreet()\n", - }); + await tool.execute("call-3", { cells: [{ language: "js", code: "def greet():\n print('hi')\ngreet()\n" }] }); expect(pythonExecuteSpy).toHaveBeenCalledTimes(1); expect(jsExecuteSpy).not.toHaveBeenCalled();