diff --git a/packages/coding-agent/src/tools/browser/render.ts b/packages/coding-agent/src/tools/browser/render.ts index ccc404ac7..6b4919b3c 100644 --- a/packages/coding-agent/src/tools/browser/render.ts +++ b/packages/coding-agent/src/tools/browser/render.ts @@ -11,6 +11,7 @@ import type { RenderResultOptions } from "../../extensibility/custom-tools/types import type { Theme } from "../../modes/theme/theme"; import { Hasher, isFramedBlockComponent, markFramedBlockComponent, renderCodeCell, renderStatusLine } from "../../tui"; import type { BrowserToolDetails } from "../browser"; +import { formatJavaScriptForDisplay } from "../eval-format/javascript"; import { formatStyledTruncationWarning, stripOutputNotice } from "../output-meta"; import { replaceTabs, shortenPath } from "../render-utils"; @@ -90,7 +91,7 @@ function renderRunCell( isError: boolean, theme: Theme, ): Component { - const code = dropTrailingBlankLines(args.code ?? ""); + const code = formatJavaScriptForDisplay(dropTrailingBlankLines(args.code ?? "")); const status = cellStatus(options.isPartial, isError); const titleParts: string[] = [tabLabel(args, details)]; diff --git a/packages/coding-agent/src/tools/eval-format/index.ts b/packages/coding-agent/src/tools/eval-format/index.ts new file mode 100644 index 000000000..7e2c083bf --- /dev/null +++ b/packages/coding-agent/src/tools/eval-format/index.ts @@ -0,0 +1,24 @@ +import type { EvalLanguage } from "../../eval/types"; +import { formatJavaScriptForDisplay } from "./javascript"; +import { formatJuliaForDisplay } from "./julia"; +import { formatPythonForDisplay } from "./python"; +import { formatRubyForDisplay } from "./ruby"; + +export * from "./javascript"; +export * from "./julia"; +export * from "./python"; +export * from "./ruby"; + +/** Formats an arbitrary eval-code prefix for display without changing the executed source. */ +export function formatEvalCodeForDisplay(source: string, language: EvalLanguage): string { + switch (language) { + case "js": + return formatJavaScriptForDisplay(source); + case "ruby": + return formatRubyForDisplay(source); + case "julia": + return formatJuliaForDisplay(source); + case "python": + return formatPythonForDisplay(source); + } +} diff --git a/packages/coding-agent/src/tools/eval-format/javascript.ts b/packages/coding-agent/src/tools/eval-format/javascript.ts new file mode 100644 index 000000000..5ebb7c09f --- /dev/null +++ b/packages/coding-agent/src/tools/eval-format/javascript.ts @@ -0,0 +1,476 @@ +type PendingBreak = "brace" | "statement" | "close"; + +interface ParenFrame { + forHeader: boolean; + controlHeader: boolean; +} + +interface TemplateTextFrame { + kind: "text"; +} + +interface TemplateExpressionFrame { + kind: "expression"; + braceDepth: number; + regexAllowed: boolean; +} + +type TemplateFrame = TemplateTextFrame | TemplateExpressionFrame; + +const CONTROL_HEADER_WORDS: Record = { + catch: true, + for: true, + if: true, + switch: true, + while: true, + with: true, +}; +const REGEX_PREFIX_WORDS: Record = { + await: true, + case: true, + delete: true, + do: true, + else: true, + extends: true, + in: true, + instanceof: true, + new: true, + of: true, + return: true, + throw: true, + typeof: true, + void: true, + yield: true, +}; +const CLOSE_CONTINUATIONS = ["else", "catch", "finally"]; + +function isIdentifierStart(char: string): boolean { + if (!char) return false; + const code = char.charCodeAt(0); + return char === "$" || char === "_" || (code >= 65 && code <= 90) || (code >= 97 && code <= 122) || code >= 128; +} + +function isIdentifierPart(char: string): boolean { + if (!char) return false; + const code = char.charCodeAt(0); + return isIdentifierStart(char) || (code >= 48 && code <= 57); +} + +function scanIdentifier(source: string, start: number): number { + let index = start + 1; + while (index < source.length && isIdentifierPart(source[index])) index++; + return index; +} + +function scanNumber(source: string, start: number): number { + let index = start + 1; + while (index < source.length && /[\w.]/.test(source[index])) index++; + return index; +} + +function scanQuoted(source: string, start: number): number { + const quote = source[start]; + let index = start + 1; + while (index < source.length) { + if (source[index] === "\\") { + index += index + 1 < source.length ? 2 : 1; + continue; + } + if (source[index] === quote) return index + 1; + index++; + } + return source.length; +} + +function scanLineComment(source: string, start: number): number { + let index = start + 2; + while (index < source.length && source[index] !== "\n" && source[index] !== "\r") index++; + return index; +} + +function scanBlockComment(source: string, start: number): number { + let index = start + 2; + while (index < source.length) { + if (source[index] === "*" && source[index + 1] === "/") return index + 2; + index++; + } + return source.length; +} + +function scanRegex(source: string, start: number): number { + let index = start + 1; + let inCharacterClass = false; + while (index < source.length) { + const char = source[index]; + if (char === "\\") { + index += index + 1 < source.length ? 2 : 1; + continue; + } + if (char === "[") inCharacterClass = true; + else if (char === "]") inCharacterClass = false; + else if (char === "/" && !inCharacterClass) { + index++; + while (index < source.length && isIdentifierPart(source[index])) index++; + return index; + } + index++; + } + return source.length; +} + +function scanTemplate(source: string, start: number): number { + const frames: TemplateFrame[] = [{ kind: "text" }]; + let index = start + 1; + + while (index < source.length) { + const frame = frames[frames.length - 1]; + if (!frame) return index; + const char = source[index]; + const next = source[index + 1]; + + if (frame.kind === "text") { + if (char === "\\") { + index += index + 1 < source.length ? 2 : 1; + } else if (char === "`") { + frames.pop(); + index++; + if (frames.length === 0) return index; + const parent = frames[frames.length - 1]; + if (parent?.kind === "expression") parent.regexAllowed = false; + } else if (char === "$" && next === "{") { + frames.push({ kind: "expression", braceDepth: 1, regexAllowed: true }); + index += 2; + } else { + index++; + } + continue; + } + + if (char === "'" || char === '"') { + index = scanQuoted(source, index); + frame.regexAllowed = false; + continue; + } + if (char === "`") { + frames.push({ kind: "text" }); + index++; + continue; + } + if (char === "/" && next === "/") { + index = scanLineComment(source, index); + continue; + } + if (char === "/" && next === "*") { + index = scanBlockComment(source, index); + continue; + } + if (char === "/" && frame.regexAllowed) { + index = scanRegex(source, index); + frame.regexAllowed = false; + continue; + } + if (isIdentifierStart(char)) { + const end = scanIdentifier(source, index); + frame.regexAllowed = REGEX_PREFIX_WORDS[source.slice(index, end)] === true; + index = end; + continue; + } + if (char >= "0" && char <= "9") { + index = scanNumber(source, index); + frame.regexAllowed = false; + continue; + } + if (char === "{") { + frame.braceDepth++; + frame.regexAllowed = true; + index++; + continue; + } + if (char === "}") { + frame.braceDepth--; + index++; + if (frame.braceDepth === 0) frames.pop(); + else frame.regexAllowed = false; + continue; + } + if ((char === "+" && next === "+") || (char === "-" && next === "-")) { + frame.regexAllowed = false; + index += 2; + continue; + } + if (char === ")" || char === "]" || char === ".") frame.regexAllowed = false; + else if (!/\s/.test(char)) frame.regexAllowed = true; + index++; + } + + return source.length; +} + +function canJoinCloseWithWord(word: string, atSourceEnd: boolean): boolean { + return CLOSE_CONTINUATIONS.some( + (keyword) => keyword === word || (atSourceEnd && keyword.startsWith(word)), + ); +} + +function canAttachToClose(char: string): boolean { + return "();,.)]:?+-*/%&|^<>=!".includes(char); +} + +/** Formats JavaScript/TypeScript-like eval source for safe, stable display without requiring valid syntax. */ +export function formatJavaScriptForDisplay(source: string): string { + const output: string[] = []; + const parens: ParenFrame[] = []; + let index = 0; + let indent = 0; + let atLineStart = true; + let lastChar = ""; + let pendingWhitespace = ""; + let pendingBreak: PendingBreak | undefined; + let afterForSemicolon = false; + let regexAllowed = true; + let pendingFor = false; + let lastWord = ""; + let lastTokenWasWord = false; + + function append(text: string): void { + if (!text) return; + output.push(text); + lastChar = text[text.length - 1]; + const newline = Math.max(text.lastIndexOf("\n"), text.lastIndexOf("\r")); + atLineStart = newline >= 0 ? newline === text.length - 1 : false; + } + + function newline(): void { + output.push("\n"); + lastChar = "\n"; + atLineStart = true; + } + + function whitespaceWidth(text: string): number { + let width = 0; + for (const char of text) width += char === "\t" ? 4 - (width % 4) : 1; + return width; + } + + function flushWhitespace(): void { + if (atLineStart) { + const width = Math.max(indent * 4, whitespaceWidth(pendingWhitespace)); + if (width > 0) append(" ".repeat(width)); + } else { + append(pendingWhitespace); + } + pendingWhitespace = ""; + } + + function forceBreak(): void { + pendingWhitespace = ""; + if (!atLineStart) newline(); + pendingBreak = undefined; + } + + function prepareToken(kind: "word" | "punctuation" | "value", text: string, end: number): void { + if (pendingBreak === "close") { + if (kind === "word" && canJoinCloseWithWord(text, end === source.length)) { + pendingWhitespace = ""; + if (!atLineStart && lastChar !== " ") append(" "); + pendingBreak = undefined; + } else if (kind === "punctuation" && canAttachToClose(text[0])) { + pendingWhitespace = ""; + pendingBreak = undefined; + } else { + forceBreak(); + } + } else if (pendingBreak) { + forceBreak(); + } + + if (afterForSemicolon) { + if (text !== ";" && text !== ")" && !atLineStart && pendingWhitespace.length === 0) append(" "); + afterForSemicolon = false; + } + } + + function appendComment(end: number): void { + const comment = source.slice(index, end); + if (pendingBreak) { + if (pendingWhitespace.length > 0) append(pendingWhitespace); + else if (!atLineStart) append(" "); + pendingWhitespace = ""; + } else { + flushWhitespace(); + } + append(comment); + if (comment.includes("\n") || comment.includes("\r")) pendingBreak = undefined; + if (afterForSemicolon) afterForSemicolon = false; + } + + while (index < source.length) { + const char = source[index]; + const next = source[index + 1]; + + if (char === "\n" || char === "\r") { + pendingWhitespace = ""; + pendingBreak = undefined; + afterForSemicolon = false; + newline(); + index += char === "\r" && next === "\n" ? 2 : 1; + continue; + } + if (/\s/.test(char)) { + const start = index; + while (index < source.length && /[^\S\r\n]/.test(source[index])) index++; + pendingWhitespace += source.slice(start, index); + continue; + } + if (char === "/" && next === "/") { + const end = scanLineComment(source, index); + appendComment(end); + index = end; + continue; + } + if (char === "/" && next === "*") { + const end = scanBlockComment(source, index); + appendComment(end); + index = end; + continue; + } + if (char === "'" || char === '"') { + const end = scanQuoted(source, index); + prepareToken("value", char, end); + flushWhitespace(); + append(source.slice(index, end)); + index = end; + regexAllowed = false; + pendingFor = false; + lastTokenWasWord = false; + continue; + } + if (char === "`") { + const end = scanTemplate(source, index); + prepareToken("value", char, end); + flushWhitespace(); + append(source.slice(index, end)); + index = end; + regexAllowed = false; + pendingFor = false; + lastTokenWasWord = false; + continue; + } + if (char === "/" && regexAllowed) { + const end = scanRegex(source, index); + prepareToken("value", char, end); + flushWhitespace(); + append(source.slice(index, end)); + index = end; + regexAllowed = false; + pendingFor = false; + lastTokenWasWord = false; + continue; + } + if (isIdentifierStart(char)) { + const end = scanIdentifier(source, index); + const word = source.slice(index, end); + prepareToken("word", word, end); + flushWhitespace(); + append(word); + if (word === "for") pendingFor = true; + else if (!(pendingFor && word === "await")) pendingFor = false; + regexAllowed = REGEX_PREFIX_WORDS[word] === true; + lastWord = word; + lastTokenWasWord = true; + index = end; + continue; + } + if (char >= "0" && char <= "9") { + const end = scanNumber(source, index); + prepareToken("value", char, end); + flushWhitespace(); + append(source.slice(index, end)); + index = end; + regexAllowed = false; + pendingFor = false; + lastTokenWasWord = false; + continue; + } + if (char === "{") { + prepareToken("punctuation", char, index + 1); + const hadWhitespace = pendingWhitespace.length > 0; + flushWhitespace(); + if (!atLineStart && !hadWhitespace && !" ([{".includes(lastChar)) append(" "); + append(char); + indent++; + pendingBreak = "brace"; + regexAllowed = true; + pendingFor = false; + lastTokenWasWord = false; + index++; + continue; + } + if (char === "}") { + prepareToken("punctuation", char, index + 1); + pendingWhitespace = ""; + if (!atLineStart) newline(); + indent = Math.max(0, indent - 1); + flushWhitespace(); + append(char); + pendingBreak = "close"; + regexAllowed = false; + pendingFor = false; + lastTokenWasWord = false; + index++; + continue; + } + if (char === ";") { + prepareToken("punctuation", char, index + 1); + flushWhitespace(); + append(char); + const frame = parens[parens.length - 1]; + if (frame?.forHeader) afterForSemicolon = true; + else pendingBreak = "statement"; + regexAllowed = true; + pendingFor = false; + lastTokenWasWord = false; + index++; + continue; + } + if (char === "(") { + const forHeader = pendingFor; + const controlHeader = forHeader || (lastTokenWasWord && CONTROL_HEADER_WORDS[lastWord] === true); + prepareToken("punctuation", char, index + 1); + const needsSpace = controlHeader && pendingWhitespace.length === 0 && !atLineStart; + flushWhitespace(); + if (needsSpace) append(" "); + append(char); + parens.push({ forHeader, controlHeader }); + regexAllowed = true; + pendingFor = false; + lastTokenWasWord = false; + index++; + continue; + } + if (char === ")") { + prepareToken("punctuation", char, index + 1); + flushWhitespace(); + append(char); + const frame = parens.pop(); + regexAllowed = frame?.controlHeader ?? false; + pendingFor = false; + lastTokenWasWord = false; + index++; + continue; + } + + const doubledPostfix = (char === "+" && next === "+") || (char === "-" && next === "-"); + const token = doubledPostfix ? source.slice(index, index + 2) : char; + prepareToken("punctuation", token, index + token.length); + flushWhitespace(); + append(token); + if (doubledPostfix || char === "]" || char === ".") regexAllowed = false; + else regexAllowed = true; + pendingFor = false; + lastTokenWasWord = false; + index += token.length; + } + + return output.join(""); +} diff --git a/packages/coding-agent/src/tools/eval-format/julia.ts b/packages/coding-agent/src/tools/eval-format/julia.ts new file mode 100644 index 000000000..2439010dd --- /dev/null +++ b/packages/coding-agent/src/tools/eval-format/julia.ts @@ -0,0 +1,461 @@ +const INDENT = " "; + +const BLOCK_OPENERS: Record = { + function: true, + macro: true, + struct: true, + if: true, + for: true, + while: true, + let: true, + begin: true, + quote: true, + try: true, + module: true, + baremodule: true, + do: true, +}; + +const BRANCH_CLAUSES: Record = { + else: true, + elseif: true, + catch: true, + finally: true, +}; + +const EXPRESSION_PREFIX_WORDS: Record = { + baremodule: true, + begin: true, + catch: true, + const: true, + do: true, + else: true, + elseif: true, + finally: true, + for: true, + function: true, + global: true, + if: true, + in: true, + isa: true, + let: true, + local: true, + macro: true, + module: true, + mutable: true, + quote: true, + return: true, + struct: true, + throw: true, + try: true, + where: true, + while: true, +}; + +type QuoteKind = "double" | "triple" | "command" | "char"; + +interface QuoteFrame { + type: "quote"; + kind: QuoteKind; + escaped: boolean; +} + +interface InterpolationFrame { + type: "interpolation"; + closers: string[]; + canEndExpression: boolean; +} + +type LiteralFrame = QuoteFrame | InterpolationFrame; + +type PrefixToken = "none" | "dot" | "colon" | "at" | "other"; + +function isIdentifierStart(character: string): boolean { + const code = character.charCodeAt(0); + return ( + character === "_" || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + code >= 0x80 + ); +} + +function isIdentifierContinue(character: string): boolean { + const code = character.charCodeAt(0); + return ( + isIdentifierStart(character) || + (code >= 48 && code <= 57) || + character === "!" || + character === "?" + ); +} + +function isHorizontalWhitespace(character: string): boolean { + return character === " " || character === "\t" || character === "\v" || character === "\f"; +} + +function isExpressionSeparator(character: string): boolean { + return ( + character === "=" || + character === "," || + character === ";" || + character === ":" || + character === "." || + character === "@" || + character === "+" || + character === "-" || + character === "*" || + character === "/" || + character === "\\" || + character === "%" || + character === "^" || + character === "&" || + character === "|" || + character === "<" || + character === ">" || + character === "~" + ); +} + +function consumeLineComment(source: string, start: number): number { + let index = start + 1; + while (index < source.length && source[index] !== "\n" && source[index] !== "\r") index++; + return index; +} + +function consumeBlockComment(source: string, start: number): number { + let depth = 1; + let index = start + 2; + + while (index < source.length) { + if (source[index] === "#" && source[index + 1] === "=") { + depth++; + index += 2; + continue; + } + if (source[index] === "=" && source[index + 1] === "#") { + depth--; + index += 2; + if (depth === 0) return index; + continue; + } + index++; + } + + return source.length; +} + +function quoteWidth(kind: QuoteKind): number { + return kind === "triple" ? 3 : 1; +} + +function quoteCloses(source: string, index: number, kind: QuoteKind): boolean { + if (kind === "triple") { + return source[index] === '"' && source[index + 1] === '"' && source[index + 2] === '"'; + } + if (kind === "double") return source[index] === '"'; + if (kind === "command") return source[index] === "`"; + return source[index] === "'"; +} + +function pushQuote(frames: LiteralFrame[], kind: QuoteKind): void { + frames.push({ type: "quote", kind, escaped: false }); +} + +/** + * Finds the end of a quoted literal while treating interpolation as opaque code. + * The small lexer is deliberately independent from display layout: its only job + * is to keep separators and block words inside a literal out of the formatter. + */ +function consumeQuotedLiteral(source: string, start: number, kind: QuoteKind): number { + const frames: LiteralFrame[] = []; + pushQuote(frames, kind); + let index = start + quoteWidth(kind); + + while (index < source.length) { + const frame = frames.at(-1); + if (!frame) return index; + + if (frame.type === "quote") { + const character = source[index]; + if (frame.escaped) { + frame.escaped = false; + index++; + continue; + } + if (character === "\\") { + frame.escaped = true; + index++; + continue; + } + if (frame.kind !== "char" && character === "$" && source[index + 1] === "(") { + frames.push({ type: "interpolation", closers: [")"], canEndExpression: false }); + index += 2; + continue; + } + if (quoteCloses(source, index, frame.kind)) { + index += quoteWidth(frame.kind); + frames.pop(); + const parent = frames.at(-1); + if (!parent) return index; + if (parent.type === "interpolation") parent.canEndExpression = true; + continue; + } + index++; + continue; + } + + const character = source[index]; + if (character === "#") { + index = + source[index + 1] === "=" + ? consumeBlockComment(source, index) + : consumeLineComment(source, index); + continue; + } + if (character === '"') { + const nestedKind = source.startsWith('"""', index) ? "triple" : "double"; + pushQuote(frames, nestedKind); + index += quoteWidth(nestedKind); + continue; + } + if (character === "`") { + pushQuote(frames, "command"); + index++; + continue; + } + if (character === "'") { + if (frame.canEndExpression) { + index++; + continue; + } + pushQuote(frames, "char"); + index++; + continue; + } + if (isIdentifierStart(character)) { + const wordStart = index; + index++; + while (index < source.length && isIdentifierContinue(source[index])) index++; + frame.canEndExpression = EXPRESSION_PREFIX_WORDS[source.slice(wordStart, index)] !== true; + continue; + } + if (character === "(" || character === "[" || character === "{") { + frame.closers.push(character === "(" ? ")" : character === "[" ? "]" : "}"); + frame.canEndExpression = false; + index++; + continue; + } + if (character === ")" || character === "]" || character === "}") { + if (frame.closers.at(-1) === character) { + frame.closers.pop(); + index++; + if (frame.closers.length === 0) frames.pop(); + else frame.canEndExpression = true; + continue; + } + frame.canEndExpression = true; + index++; + continue; + } + if (character === "\n" || character === "\r" || isExpressionSeparator(character)) { + frame.canEndExpression = false; + index++; + continue; + } + if (!isHorizontalWhitespace(character)) frame.canEndExpression = true; + index++; + } + + return source.length; +} + +/** Formats an arbitrary Julia source prefix for stable, readable display. */ +export function formatJuliaForDisplay(source: string): string { + const output: string[] = []; + const delimiterClosers: string[] = []; + let index = 0; + let blockDepth = 0; + let atLineStart = true; + let pendingWhitespace = ""; + let suppressSourceNewline = false; + let canEndExpression = false; + let previousToken: PrefixToken = "none"; + + function beginContent(dedentBlock: boolean, dedentDelimiter: boolean): void { + if (atLineStart) { + pendingWhitespace = ""; + const indentation = Math.max( + 0, + blockDepth - (dedentBlock ? 1 : 0) + delimiterClosers.length - (dedentDelimiter ? 1 : 0), + ); + if (indentation > 0) output.push(INDENT.repeat(indentation)); + atLineStart = false; + } else if (pendingWhitespace.length > 0) { + output.push(pendingWhitespace); + pendingWhitespace = ""; + } + suppressSourceNewline = false; + } + + function appendOpaque(start: number, end: number): boolean { + output.push(source.slice(start, end)); + let containsNewline = false; + for (let cursor = start; cursor < end; cursor++) { + if (source[cursor] === "\n" || source[cursor] === "\r") { + containsNewline = true; + atLineStart = true; + } else { + atLineStart = false; + } + } + return containsNewline; + } + + while (index < source.length) { + const character = source[index]; + + if (isHorizontalWhitespace(character)) { + const whitespaceStart = index; + index++; + while (index < source.length && isHorizontalWhitespace(source[index])) index++; + if (!atLineStart) pendingWhitespace = source.slice(whitespaceStart, index); + continue; + } + + if (character === "\n" || character === "\r") { + pendingWhitespace = ""; + const newlineWidth = character === "\r" && source[index + 1] === "\n" ? 2 : 1; + index += newlineWidth; + if (suppressSourceNewline && atLineStart) { + suppressSourceNewline = false; + continue; + } + output.push("\n"); + atLineStart = true; + suppressSourceNewline = false; + canEndExpression = false; + previousToken = "none"; + continue; + } + + if (character === "#") { + beginContent(false, false); + const commentEnd = + source[index + 1] === "=" + ? consumeBlockComment(source, index) + : consumeLineComment(source, index); + const containsNewline = appendOpaque(index, commentEnd); + if (containsNewline) { + canEndExpression = false; + previousToken = "none"; + } + index = commentEnd; + continue; + } + + if (character === '"') { + beginContent(false, false); + const literalKind = source.startsWith('"""', index) ? "triple" : "double"; + const literalEnd = consumeQuotedLiteral(source, index, literalKind); + appendOpaque(index, literalEnd); + index = literalEnd; + canEndExpression = true; + previousToken = "other"; + continue; + } + + if (character === "`") { + beginContent(false, false); + const literalEnd = consumeQuotedLiteral(source, index, "command"); + appendOpaque(index, literalEnd); + index = literalEnd; + canEndExpression = true; + previousToken = "other"; + continue; + } + + if (character === "'") { + beginContent(false, false); + if (canEndExpression) { + output.push(character); + index++; + previousToken = "other"; + continue; + } + const literalEnd = consumeQuotedLiteral(source, index, "char"); + appendOpaque(index, literalEnd); + index = literalEnd; + canEndExpression = true; + previousToken = "other"; + continue; + } + + if (isIdentifierStart(character)) { + const wordStart = index; + index++; + while (index < source.length && isIdentifierContinue(source[index])) index++; + const word = source.slice(wordStart, index); + const isStructural = + delimiterClosers.length === 0 && + previousToken !== "dot" && + previousToken !== "colon" && + previousToken !== "at"; + const isEnd = isStructural && word === "end"; + const isBranch = isStructural && BRANCH_CLAUSES[word] === true; + beginContent(isEnd || isBranch, false); + output.push(word); + + if (isEnd) blockDepth = Math.max(0, blockDepth - 1); + else if (isStructural && BLOCK_OPENERS[word] === true) blockDepth++; + + canEndExpression = EXPRESSION_PREFIX_WORDS[word] !== true; + previousToken = "other"; + continue; + } + + if (character === "(" || character === "[" || character === "{") { + beginContent(false, false); + output.push(character); + delimiterClosers.push(character === "(" ? ")" : character === "[" ? "]" : "}"); + canEndExpression = false; + previousToken = "other"; + index++; + continue; + } + + if (character === ")" || character === "]" || character === "}") { + const matchesDelimiter = delimiterClosers.at(-1) === character; + beginContent(false, matchesDelimiter); + output.push(character); + if (matchesDelimiter) delimiterClosers.pop(); + canEndExpression = true; + previousToken = "other"; + index++; + continue; + } + + if (character === ";" && delimiterClosers.length === 0) { + beginContent(false, false); + output.push(";\n"); + atLineStart = true; + pendingWhitespace = ""; + suppressSourceNewline = true; + canEndExpression = false; + previousToken = "none"; + index++; + continue; + } + + beginContent(false, false); + output.push(character); + if (character === ".") previousToken = "dot"; + else if (character === ":") previousToken = "colon"; + else if (character === "@") previousToken = "at"; + else previousToken = "other"; + canEndExpression = !isExpressionSeparator(character); + index++; + } + + return output.join(""); +} diff --git a/packages/coding-agent/src/tools/eval-format/python.ts b/packages/coding-agent/src/tools/eval-format/python.ts new file mode 100644 index 000000000..c4dc0de9f --- /dev/null +++ b/packages/coding-agent/src/tools/eval-format/python.ts @@ -0,0 +1,547 @@ +type HeaderKind = + | "def" + | "class" + | "if" + | "elif" + | "else" + | "for" + | "while" + | "try" + | "except" + | "finally" + | "with" + | "match" + | "case" + | "async def" + | "async for" + | "async with"; + +type ChainKind = "if" | "loop" | "try" | "match"; + +interface Word { + text: string; + end: number; +} + +interface Header { + kind: HeaderKind; + end: number; +} + +interface BlockFrame { + kind: HeaderKind; + chain: ChainKind | null; + headerIndent: number; + sourceIndent: number; + previousChain: number; +} + +interface DelimiterFrame { + opener: string; + outputIndent: number; +} + +interface PendingSuiteColon { + kind: HeaderKind; + chain: ChainKind | null; + headerIndent: number; + sourceIndent: number; + afterColon: string[]; +} + +interface StringState { + quote: string; + triple: boolean; + escaped: boolean; +} + +interface ClauseAlignment { + indent: number; + chain: ChainKind | null; +} + +function isWordStart(char: string | undefined): boolean { + if (!char) return false; + const code = char.charCodeAt(0); + return char === "_" || (code >= 65 && code <= 90) || (code >= 97 && code <= 122); +} + +function isWordPart(char: string): boolean { + const code = char.charCodeAt(0); + return isWordStart(char) || (code >= 48 && code <= 57); +} + +function readWord(source: string, start: number): Word { + let end = start; + while (end < source.length && isWordPart(source[end])) end++; + return { text: source.slice(start, end), end }; +} + +function simpleHeader(word: Word): Header | null { + switch (word.text) { + case "def": + case "class": + case "if": + case "elif": + case "else": + case "for": + case "while": + case "try": + case "except": + case "finally": + case "with": + case "match": + case "case": + return { kind: word.text, end: word.end }; + default: + return null; + } +} + +function readHeader(source: string, start: number): Header | null { + if (!isWordStart(source[start])) return null; + const first = readWord(source, start); + if (first.text !== "async") return simpleHeader(first); + + let next = first.end; + while (source[next] === " " || source[next] === "\t") next++; + if (!isWordStart(source[next])) return null; + const second = readWord(source, next); + switch (second.text) { + case "def": + return { kind: "async def", end: second.end }; + case "for": + return { kind: "async for", end: second.end }; + case "with": + return { kind: "async with", end: second.end }; + default: + return null; + } +} + +function defaultChain(kind: HeaderKind): ChainKind | null { + switch (kind) { + case "if": + case "elif": + case "else": + return "if"; + case "for": + case "while": + case "async for": + return "loop"; + case "try": + case "except": + case "finally": + return "try"; + case "match": + case "case": + return "match"; + default: + return null; + } +} + +function canOpenSuite(kind: HeaderKind, hasPayload: boolean): boolean { + switch (kind) { + case "else": + case "try": + case "finally": + return !hasPayload; + case "except": + return true; + default: + return hasPayload; + } +} + +function matchingCloser(opener: string, closer: string): boolean { + return ( + (opener === "(" && closer === ")") || + (opener === "[" && closer === "]") || + (opener === "{" && closer === "}") + ); +} + +function formatPythonPrefix(source: string): string { + const chunks: string[] = []; + const blocks: BlockFrame[] = []; + const delimiters: DelimiterFrame[] = []; + const chainTops: Record = { if: -1, loop: -1, try: -1, match: -1 }; + const pendingHorizontal: string[] = []; + + let outputLineStart = true; + let lineIndent: number | null = null; + let currentOutputIndent = 0; + let sourceLineStart = true; + let sourceIndent = 0; + let currentSourceIndent = 0; + let skipGeneratedNewline = false; + + let statementPrepared = false; + let statementKind: HeaderKind | null = null; + let statementHeaderEnd = -1; + let statementHasPayload = false; + let statementIndent = 0; + let statementSourceIndent = 0; + let statementChain: ChainKind | null = null; + + let pendingColon: PendingSuiteColon | null = null; + let stringState: StringState | null = null; + let inComment = false; + + function currentBlockIndent(): number { + const top = blocks[blocks.length - 1]; + return top ? top.headerIndent + 1 : 0; + } + + function popBlock(): void { + const index = blocks.length - 1; + const frame = blocks.pop(); + if (frame?.chain && chainTops[frame.chain] === index) chainTops[frame.chain] = frame.previousChain; + } + + function popThrough(index: number): void { + while (blocks.length > index) popBlock(); + } + + function pushBlock(frame: PendingSuiteColon): void { + const previousChain = frame.chain ? chainTops[frame.chain] : -1; + blocks.push({ + kind: frame.kind, + chain: frame.chain, + headerIndent: frame.headerIndent, + sourceIndent: frame.sourceIndent, + previousChain, + }); + if (frame.chain) chainTops[frame.chain] = blocks.length - 1; + } + + function resetStatement(): void { + statementPrepared = false; + statementKind = null; + statementHeaderEnd = -1; + statementHasPayload = false; + statementIndent = currentBlockIndent(); + statementSourceIndent = currentSourceIndent; + statementChain = null; + } + + + function flushHorizontal(): void { + if (pendingHorizontal.length === 0) return; + chunks.push(pendingHorizontal.join("").replaceAll("\t", " ")); + pendingHorizontal.length = 0; + } + + function appendNormal(text: string): void { + if (outputLineStart) { + const indent = lineIndent ?? currentBlockIndent(); + if (indent > 0) chunks.push(" ".repeat(indent)); + currentOutputIndent = indent; + outputLineStart = false; + } + flushHorizontal(); + chunks.push(text); + } + + function appendRaw(text: string): void { + chunks.push(text); + if (outputLineStart) { + outputLineStart = false; + currentOutputIndent = 0; + } + } + + function finishOutputLine(): void { + pendingHorizontal.length = 0; + chunks.push("\n"); + outputLineStart = true; + lineIndent = null; + currentOutputIndent = 0; + } + + function consumeSourceNewline(): void { + sourceLineStart = true; + sourceIndent = 0; + currentSourceIndent = 0; + skipGeneratedNewline = false; + if (delimiters.length === 0) resetStatement(); + } + + function popSourceDedents(indent: number): void { + let top = blocks[blocks.length - 1]; + while (top && indent <= top.sourceIndent) { + popBlock(); + top = blocks[blocks.length - 1]; + } + } + + function alignTo(index: number, fallback: ChainKind): ClauseAlignment { + if (index < 0) return { indent: currentBlockIndent(), chain: fallback }; + const frame = blocks[index]; + const alignment = { indent: frame.headerIndent, chain: frame.chain ?? fallback }; + popThrough(index); + return alignment; + } + + function alignClause(kind: HeaderKind): ClauseAlignment | null { + switch (kind) { + case "elif": + return alignTo(chainTops.if, "if"); + case "except": + case "finally": + return alignTo(chainTops.try, "try"); + case "else": { + const target = Math.max(chainTops.if, chainTops.loop, chainTops.try); + return alignTo(target, "if"); + } + case "case": { + const target = chainTops.match; + if (target < 0) return { indent: currentBlockIndent(), chain: "match" }; + const frame = blocks[target]; + if (frame.kind === "match") { + while (blocks.length > target + 1) popBlock(); + return { indent: frame.headerIndent + 1, chain: "match" }; + } + const indent = frame.headerIndent; + popThrough(target); + return { indent, chain: "match" }; + } + default: + return null; + } + } + + function prepareStatement(index: number, physicalLineStart: boolean): void { + if (physicalLineStart) popSourceDedents(currentSourceIndent); + const header = readHeader(source, index); + const alignment = header ? alignClause(header.kind) : null; + statementPrepared = true; + statementKind = header?.kind ?? null; + statementHeaderEnd = header?.end ?? -1; + statementHasPayload = false; + statementIndent = alignment?.indent ?? currentBlockIndent(); + statementSourceIndent = currentSourceIndent; + statementChain = alignment?.chain ?? (header ? defaultChain(header.kind) : null); + lineIndent = statementIndent; + } + + function prepareToken(index: number, char: string, comment: boolean): void { + const physicalLineStart = sourceLineStart; + if (physicalLineStart) { + currentSourceIndent = sourceIndent; + sourceLineStart = false; + } + if (skipGeneratedNewline) skipGeneratedNewline = false; + if (!outputLineStart) return; + + const delimiter = delimiters[delimiters.length - 1]; + if (delimiter && statementPrepared) { + const closes = matchingCloser(delimiter.opener, char); + const structuralIndent = delimiter.outputIndent + (closes ? 0 : 1); + lineIndent = Math.max(structuralIndent, Math.ceil(currentSourceIndent / 4)); + return; + } + + if (comment) { + lineIndent = physicalLineStart + ? Math.min(currentBlockIndent(), Math.ceil(currentSourceIndent / 4)) + : currentBlockIndent(); + return; + } + if (!statementPrepared) prepareStatement(index, physicalLineStart); + else lineIndent = statementIndent; + } + + function notePayload(index: number): void { + if (statementKind && index >= statementHeaderEnd) statementHasPayload = true; + } + + function openPendingSuite(frame: PendingSuiteColon): void { + pushBlock(frame); + resetStatement(); + } + + let index = 0; + while (index < source.length) { + let char = source[index]; + + if (stringState) { + const newline = char === "\n" || char === "\r"; + if (newline) { + const width = char === "\r" && source[index + 1] === "\n" ? 2 : 1; + appendRaw(source.slice(index, index + width)); + outputLineStart = true; + lineIndent = null; + currentOutputIndent = 0; + sourceLineStart = true; + sourceIndent = 0; + stringState.escaped = false; + index += width; + continue; + } + + if ( + stringState.triple && + !stringState.escaped && + char === stringState.quote && + source[index + 1] === char && + source[index + 2] === char + ) { + appendRaw(source.slice(index, index + 3)); + sourceLineStart = false; + stringState = null; + index += 3; + continue; + } + + appendRaw(char); + sourceLineStart = false; + if (stringState.escaped) stringState.escaped = false; + else if (char === "\\") stringState.escaped = true; + else if (!stringState.triple && char === stringState.quote) stringState = null; + index++; + continue; + } + + if (inComment) { + if (char === "\n" || char === "\r") { + const width = char === "\r" && source[index + 1] === "\n" ? 2 : 1; + finishOutputLine(); + inComment = false; + consumeSourceNewline(); + index += width; + continue; + } + appendRaw(char); + index++; + continue; + } + + if (pendingColon) { + if (char === " " || char === "\t") { + pendingColon.afterColon.push(char); + index++; + continue; + } + if (char === "=" && pendingColon.afterColon.length === 0) { + appendNormal(":"); + pendingColon = null; + } else if (char === "#") { + const frame = pendingColon; + appendNormal(":"); + if (frame.afterColon.length > 0) chunks.push(frame.afterColon.join("").replaceAll("\t", " ")); + openPendingSuite(frame); + pendingColon = null; + } else if (char === "\n" || char === "\r") { + const frame = pendingColon; + const width = char === "\r" && source[index + 1] === "\n" ? 2 : 1; + appendNormal(":"); + openPendingSuite(frame); + pendingColon = null; + finishOutputLine(); + consumeSourceNewline(); + index += width; + continue; + } else { + const frame = pendingColon; + appendNormal(":"); + openPendingSuite(frame); + pendingColon = null; + finishOutputLine(); + skipGeneratedNewline = true; + continue; + } + char = source[index]; + } + + if (sourceLineStart && (char === " " || char === "\t")) { + if (char === "\t") sourceIndent += 4 - (sourceIndent % 4); + else sourceIndent++; + index++; + continue; + } + + if (char === "\n" || char === "\r") { + const width = char === "\r" && source[index + 1] === "\n" ? 2 : 1; + pendingHorizontal.length = 0; + if (!skipGeneratedNewline) finishOutputLine(); + consumeSourceNewline(); + index += width; + continue; + } + + if (char === " " || char === "\t") { + if (!skipGeneratedNewline && (!outputLineStart || statementPrepared)) pendingHorizontal.push(char); + index++; + continue; + } + + prepareToken(index, char, char === "#"); + + if (char === "#") { + appendNormal(char); + inComment = true; + index++; + continue; + } + + if (char === "'" || char === '"') { + notePayload(index); + const triple = source[index + 1] === char && source[index + 2] === char; + appendNormal(triple ? source.slice(index, index + 3) : char); + stringState = { quote: char, triple, escaped: false }; + index += triple ? 3 : 1; + continue; + } + + if ( + char === ":" && + delimiters.length === 0 && + statementKind && + canOpenSuite(statementKind, statementHasPayload) + ) { + flushHorizontal(); + pendingColon = { + kind: statementKind, + chain: statementChain, + headerIndent: statementIndent, + sourceIndent: statementSourceIndent, + afterColon: [], + }; + index++; + continue; + } + + if (char === ";" && delimiters.length === 0) { + appendNormal(char); + finishOutputLine(); + resetStatement(); + skipGeneratedNewline = true; + index++; + continue; + } + + notePayload(index); + appendNormal(char); + if (char === "(" || char === "[" || char === "{") { + delimiters.push({ opener: char, outputIndent: currentOutputIndent }); + } else { + const delimiter = delimiters[delimiters.length - 1]; + if (delimiter && matchingCloser(delimiter.opener, char)) delimiters.pop(); + } + index++; + } + + if (pendingColon) appendNormal(":"); + return chunks.join(""); +} + +/** Formats an arbitrary Python source prefix for stable, readable display. */ +export function formatPythonForDisplay(source: string): string { + try { + return formatPythonPrefix(source); + } catch { + return source; + } +} diff --git a/packages/coding-agent/src/tools/eval-format/ruby.ts b/packages/coding-agent/src/tools/eval-format/ruby.ts new file mode 100644 index 000000000..273cc4004 --- /dev/null +++ b/packages/coding-agent/src/tools/eval-format/ruby.ts @@ -0,0 +1,567 @@ +const INDENT = " "; + +const OPENING_KEYWORDS = new Set(["class", "module", "def", "if", "unless", "case", "begin", "while", "until", "for"]); +const BRANCH_KEYWORDS = new Set(["else", "elsif", "when", "rescue", "ensure"]); +const REGEXP_PREFIX_KEYWORDS = new Set([ + "and", + "begin", + "case", + "do", + "else", + "elsif", + "if", + "in", + "not", + "or", + "raise", + "rescue", + "return", + "then", + "unless", + "until", + "when", + "while", + "yield", +]); + +interface WordToken { + kind: "word"; + value: string; + eligible: boolean; +} + +interface PunctuationToken { + kind: "punctuation"; + value: string; +} + +interface LiteralToken { + kind: "literal"; +} + +type StructuralToken = WordToken | PunctuationToken | LiteralToken; + +type Quote = "'" | '"' | "`"; + +interface QuotedContext { + kind: "quoted"; + quote: Quote; + interpolated: boolean; + escaped: boolean; +} + +interface PercentContext { + kind: "percent"; + open: string; + close: string; + depth: number; + interpolated: boolean; + escaped: boolean; +} + +interface RegexpContext { + kind: "regexp"; + escaped: boolean; + inCharacterClass: boolean; +} + +interface InterpolationContext { + kind: "interpolation"; + braceDepth: number; + canStartExpression: boolean; +} + +interface CommentContext { + kind: "comment"; +} + +type LexicalContext = QuotedContext | PercentContext | RegexpContext | InterpolationContext | CommentContext; + +interface PercentLiteralStart { + end: number; + open: string; + close: string; + interpolated: boolean; +} + +interface LineLayout { + indent: number; + nextDepth: number; +} + +function isHorizontalWhitespace(character: string): boolean { + return character === " " || character === "\t" || character === "\f" || character === "\v"; +} + +function isIdentifierStart(character: string | undefined): boolean { + if (character === undefined) return false; + const code = character.charCodeAt(0); + return (code >= 65 && code <= 90) || (code >= 97 && code <= 122) || character === "_"; +} + +function isIdentifierPart(character: string | undefined): boolean { + if (character === undefined) return false; + const code = character.charCodeAt(0); + return isIdentifierStart(character) || (code >= 48 && code <= 57); +} + +function identifierEnd(source: string, start: number): number { + let end = start + 1; + while (isIdentifierPart(source[end])) end++; + if (source[end] === "?" || source[end] === "!") end++; + return end; +} + +function pairedDelimiter(open: string): string { + switch (open) { + case "(": + return ")"; + case "[": + return "]"; + case "{": + return "}"; + case "<": + return ">"; + default: + return open; + } +} + +function percentLiteralStart(source: string, start: number): PercentLiteralStart | undefined { + let delimiterIndex = start + 1; + let type = ""; + const candidateType = source[delimiterIndex]; + if (candidateType !== undefined && "qQwWiIxrs".includes(candidateType)) { + type = candidateType; + delimiterIndex++; + } + + const open = source[delimiterIndex]; + if (open === undefined || /[A-Za-z0-9_\s]/.test(open)) return undefined; + + return { + end: delimiterIndex + 1, + open, + close: pairedDelimiter(open), + interpolated: type === "" || type === "Q" || type === "W" || type === "I" || type === "x" || type === "r", + }; +} + +function isMatchingDelimiter(open: string, close: string): boolean { + return ( + (open === "(" && close === ")") || + (open === "[" && close === "]") || + (open === "{" && close === "}") + ); +} + +function isStandaloneAssignment(tokens: StructuralToken[], index: number): boolean { + const previous = tokens[index - 1]; + const next = tokens[index + 1]; + if (next?.kind === "punctuation" && (next.value === "=" || next.value === ">" || next.value === "(")) return false; + if ( + previous?.kind === "punctuation" && + (previous.value === "=" || previous.value === "!" || previous.value === "<" || previous.value === ">" || previous.value === "~") + ) { + return false; + } + return true; +} + +function lineLayout(tokens: StructuralToken[], depth: number): LineLayout { + const first = tokens[0]; + const leadingKeyword = first?.kind === "word" && first.eligible ? first.value : undefined; + const leadingEnd = leadingKeyword === "end"; + const branch = leadingKeyword !== undefined && BRANCH_KEYWORDS.has(leadingKeyword); + + let indent = depth; + if (leadingEnd || branch) indent = Math.max(0, depth - 1); + + let endCount = 0; + let hasDo = false; + for (const token of tokens) { + if (token.kind !== "word" || !token.eligible) continue; + if (token.value === "end") endCount++; + if (token.value === "do") hasDo = true; + } + + let opens = leadingKeyword !== undefined && OPENING_KEYWORDS.has(leadingKeyword); + if (leadingKeyword === "def") { + for (let index = 1; index < tokens.length; index++) { + const token = tokens[index]; + if (token.kind === "punctuation" && token.value === "=" && isStandaloneAssignment(tokens, index)) { + opens = false; + break; + } + } + } + if (!leadingEnd && !branch && hasDo) opens = true; + + return { + indent, + nextDepth: Math.max(0, depth + (opens ? 1 : 0) - endCount), + }; +} + +function formatRubyPrefix(source: string): string { + const output: string[] = []; + const contexts: LexicalContext[] = []; + const delimiters: string[] = []; + let tokens: StructuralToken[] = []; + let lineParts: string[] = []; + let lineHasVisibleText = false; + let preserveLeadingWhitespace = false; + let pendingVirtualBreak = false; + let blockDepth = 0; + let rootCanStartExpression = true; + + const append = (text: string): void => { + lineParts.push(text); + if (lineHasVisibleText) return; + for (let index = 0; index < text.length; index++) { + if (!isHorizontalWhitespace(text[index])) { + lineHasVisibleText = true; + return; + } + } + }; + + const currentCodeCanStartExpression = (): boolean => { + for (let index = contexts.length - 1; index >= 0; index--) { + const context = contexts[index]; + if (context.kind === "interpolation") return context.canStartExpression; + } + return rootCanStartExpression; + }; + + const setCurrentCodeCanStartExpression = (value: boolean): void => { + for (let index = contexts.length - 1; index >= 0; index--) { + const context = contexts[index]; + if (context.kind === "interpolation") { + context.canStartExpression = value; + return; + } + } + rootCanStartExpression = value; + }; + + const resetLine = (): void => { + tokens = []; + lineParts = []; + lineHasVisibleText = false; + preserveLeadingWhitespace = contexts.length > 0 || delimiters.length > 0; + }; + + const flushLine = (ending: string): void => { + const raw = lineParts.join(""); + const layout = lineLayout(tokens, blockDepth); + blockDepth = layout.nextDepth; + + if (preserveLeadingWhitespace) { + output.push(raw, ending); + return; + } + + let contentStart = 0; + while (contentStart < raw.length && isHorizontalWhitespace(raw[contentStart])) contentStart++; + const content = raw.slice(contentStart); + output.push(content.length === 0 ? "" : INDENT.repeat(layout.indent) + content, ending); + }; + + const addPunctuation = (value: string): void => { + if (contexts.length === 0 && delimiters.length === 0) tokens.push({ kind: "punctuation", value }); + }; + + const addLiteral = (): void => { + if (contexts.length === 0 && delimiters.length === 0) tokens.push({ kind: "literal" }); + }; + + for (let index = 0; index < source.length; ) { + const character = source[index]; + if (character === "\n" || character === "\r") { + const ending = character === "\r" && source[index + 1] === "\n" ? "\r\n" : character; + const top = contexts[contexts.length - 1]; + if (top?.kind === "comment") { + contexts.pop(); + } else if (top?.kind === "quoted" || top?.kind === "percent" || top?.kind === "regexp") { + top.escaped = false; + } + + const codeContext = contexts[contexts.length - 1]; + if (codeContext?.kind === "interpolation") { + codeContext.canStartExpression = true; + } else if (contexts.length === 0 && delimiters.length === 0) { + rootCanStartExpression = true; + } + + if (pendingVirtualBreak && !lineHasVisibleText) { + pendingVirtualBreak = false; + resetLine(); + } else { + flushLine(ending); + pendingVirtualBreak = false; + resetLine(); + } + index += ending.length; + continue; + } + + const top = contexts[contexts.length - 1]; + if (top?.kind === "comment") { + append(character); + index++; + continue; + } + + if (top?.kind === "quoted") { + append(character); + if (top.escaped) { + top.escaped = false; + index++; + continue; + } + if (character === "\\") { + top.escaped = true; + index++; + continue; + } + if (top.interpolated && character === "#" && source[index + 1] === "{") { + append("{"); + contexts.push({ kind: "interpolation", braceDepth: 1, canStartExpression: true }); + index += 2; + continue; + } + if (character === top.quote) { + contexts.pop(); + setCurrentCodeCanStartExpression(false); + } + index++; + continue; + } + + if (top?.kind === "percent") { + append(character); + if (top.escaped) { + top.escaped = false; + index++; + continue; + } + if (character === "\\") { + top.escaped = true; + index++; + continue; + } + if (top.interpolated && character === "#" && source[index + 1] === "{") { + append("{"); + contexts.push({ kind: "interpolation", braceDepth: 1, canStartExpression: true }); + index += 2; + continue; + } + if (top.open !== top.close && character === top.open) { + top.depth++; + } else if (character === top.close) { + top.depth--; + if (top.depth === 0) { + contexts.pop(); + setCurrentCodeCanStartExpression(false); + } + } + index++; + continue; + } + + if (top?.kind === "regexp") { + append(character); + if (top.escaped) { + top.escaped = false; + index++; + continue; + } + if (character === "\\") { + top.escaped = true; + index++; + continue; + } + if (character === "#" && source[index + 1] === "{") { + append("{"); + contexts.push({ kind: "interpolation", braceDepth: 1, canStartExpression: true }); + index += 2; + continue; + } + if (character === "[" && !top.inCharacterClass) { + top.inCharacterClass = true; + } else if (character === "]" && top.inCharacterClass) { + top.inCharacterClass = false; + } else if (character === "/" && !top.inCharacterClass) { + contexts.pop(); + setCurrentCodeCanStartExpression(false); + } + index++; + continue; + } + + const interpolation = top?.kind === "interpolation" ? top : undefined; + const inRootCode = interpolation === undefined; + + if (interpolation !== undefined && character === "}") { + append(character); + interpolation.braceDepth--; + if (interpolation.braceDepth === 0) { + contexts.pop(); + setCurrentCodeCanStartExpression(false); + } else { + interpolation.canStartExpression = false; + } + index++; + continue; + } + + if (character === "#") { + append(character); + contexts.push({ kind: "comment" }); + index++; + continue; + } + + if (character === "'" || character === '"' || character === "`") { + addLiteral(); + append(character); + contexts.push({ + kind: "quoted", + quote: character, + interpolated: character !== "'", + escaped: false, + }); + index++; + continue; + } + + if (character === "%") { + const start = percentLiteralStart(source, index); + if (start !== undefined) { + addLiteral(); + append(source.slice(index, start.end)); + contexts.push({ + kind: "percent", + open: start.open, + close: start.close, + depth: 1, + interpolated: start.interpolated, + escaped: false, + }); + index = start.end; + continue; + } + } + + if (character === "/" && currentCodeCanStartExpression()) { + addLiteral(); + append(character); + contexts.push({ kind: "regexp", escaped: false, inCharacterClass: false }); + index++; + continue; + } + + if (character === "?" && currentCodeCanStartExpression()) { + const next = source[index + 1]; + if (next !== undefined && !/\s/.test(next)) { + addLiteral(); + let end = index + 2; + if (next === "\\" && source[end] !== undefined) end++; + append(source.slice(index, end)); + setCurrentCodeCanStartExpression(false); + index = end; + continue; + } + } + + if (isIdentifierStart(character)) { + const end = identifierEnd(source, index); + const word = source.slice(index, end); + append(word); + + if (inRootCode && delimiters.length === 0) { + const previous = tokens[tokens.length - 1]; + const blockedByPrefix = + previous?.kind === "punctuation" && + (previous.value === ":" || previous.value === "." || previous.value === "@" || previous.value === "$"); + const label = source[end] === ":" && source[end + 1] !== ":"; + tokens.push({ kind: "word", value: word, eligible: !blockedByPrefix && !label }); + } + + setCurrentCodeCanStartExpression(REGEXP_PREFIX_KEYWORDS.has(word)); + index = end; + continue; + } + + if (interpolation !== undefined && character === "{") { + append(character); + interpolation.braceDepth++; + interpolation.canStartExpression = true; + index++; + continue; + } + + if (inRootCode && (character === "(" || character === "[" || character === "{")) { + addPunctuation(character); + append(character); + delimiters.push(character); + rootCanStartExpression = true; + index++; + continue; + } + + if (inRootCode && (character === ")" || character === "]" || character === "}")) { + append(character); + const open = delimiters[delimiters.length - 1]; + if (open !== undefined && isMatchingDelimiter(open, character)) delimiters.pop(); + addPunctuation(character); + rootCanStartExpression = false; + index++; + continue; + } + + if (character === ";") { + append(character); + if (inRootCode && delimiters.length === 0) { + addPunctuation(character); + rootCanStartExpression = true; + flushLine("\n"); + pendingVirtualBreak = true; + resetLine(); + } else { + setCurrentCodeCanStartExpression(true); + } + index++; + continue; + } + + append(character); + if (isHorizontalWhitespace(character)) { + index++; + continue; + } + + if (inRootCode && delimiters.length === 0) tokens.push({ kind: "punctuation", value: character }); + if (character === "." || character === ")" || character === "]" || character === "}") { + setCurrentCodeCanStartExpression(false); + } else if ("=,:!~+-*%&|^<>?".includes(character) || character === "/") { + setCurrentCodeCanStartExpression(true); + } else { + setCurrentCodeCanStartExpression(false); + } + index++; + } + + if (lineParts.length > 0 && !(pendingVirtualBreak && !lineHasVisibleText)) flushLine(""); + return output.join(""); +} + +/** Formats an arbitrary Ruby source prefix for display without requiring valid syntax. */ +export function formatRubyForDisplay(source: string): string { + try { + return formatRubyPrefix(source); + } catch { + return source; + } +} diff --git a/packages/coding-agent/src/tools/eval-render.ts b/packages/coding-agent/src/tools/eval-render.ts index b42468c11..141df3a5a 100644 --- a/packages/coding-agent/src/tools/eval-render.ts +++ b/packages/coding-agent/src/tools/eval-render.ts @@ -14,6 +14,7 @@ import { Markdown, Text } from "@oh-my-pi/pi-tui"; import { formatNumber } from "@oh-my-pi/pi-utils"; import { settings } from "../config/settings"; import type { EvalCellResult, EvalLanguage, EvalStatusEvent, EvalToolDetails } from "../eval/types"; +import { formatEvalCodeForDisplay } from "./eval-format"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import { formatContextUsage } from "../modes/components/status-line/context-thresholds"; import { truncateToVisualLines } from "../modes/components/visual-truncate"; @@ -89,10 +90,11 @@ function getRenderCells(args: EvalRenderArgs | undefined): EvalRenderCell[] { const out: EvalRenderCell[] = []; for (const cell of raw) { if (!cell || typeof cell !== "object") continue; + const language = normalizeRenderLanguage(typeof cell.language === "string" ? cell.language : undefined); const code = typeof cell.code === "string" ? cell.code : ""; out.push({ - language: normalizeRenderLanguage(typeof cell.language === "string" ? cell.language : undefined), - code, + language, + code: formatEvalCodeForDisplay(code, language), title: typeof cell.title === "string" ? cell.title : undefined, }); } @@ -587,6 +589,10 @@ export const evalToolRenderer = { const cellResults = details?.cells; if (cellResults && cellResults.length > 0) { + const displayCells = cellResults.map(cell => { + const language = cell.language ?? details?.language ?? "python"; + return { cell, code: formatEvalCodeForDisplay(cell.code, language), language }; + }); let cached: { key: string; width: number; result: string[] } | undefined; return markFramedBlockComponent({ @@ -602,8 +608,8 @@ export const evalToolRenderer = { } const lines: string[] = []; - for (let i = 0; i < cellResults.length; i++) { - const cell = cellResults[i]; + for (let i = 0; i < displayCells.length; i++) { + const { cell, code, language } = displayCells[i]; const allEvents = cell.statusEvents ?? []; const agentEvents = allEvents.filter(e => e.op === "agent"); const otherEvents = agentEvents.length > 0 ? allEvents.filter(e => e.op !== "agent") : allEvents; @@ -623,8 +629,8 @@ export const evalToolRenderer = { } const cellLines = renderCodeCell( { - code: cell.code, - language: languageForHighlighter(cell.language ?? details?.language), + code, + language: languageForHighlighter(language), showLanguage: true, index: i, total: cellResults.length, diff --git a/packages/coding-agent/test/tools/browser-display-format.test.ts b/packages/coding-agent/test/tools/browser-display-format.test.ts new file mode 100644 index 000000000..75e9d9fb2 --- /dev/null +++ b/packages/coding-agent/test/tools/browser-display-format.test.ts @@ -0,0 +1,33 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import { getThemeByName, setThemeInstance, type Theme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; +import { toolRenderers } from "@oh-my-pi/pi-coding-agent/tools/renderers"; + +describe("browser renderer: display-only streaming formatting", () => { + let theme: Theme; + + beforeAll(async () => { + resetSettingsForTest(); + await Settings.init({ inMemory: true, cwd: process.cwd() }); + theme = (await getThemeByName("dark"))!; + expect(theme).toBeDefined(); + setThemeInstance(theme); + }); + + afterAll(() => { + resetSettingsForTest(); + }); + + it("expands compact JavaScript without mutating the run source", () => { + const source = "if (ready) {run();finish();}"; + const args = { action: "run", code: source }; + const rendered = Bun.stripANSI( + toolRenderers.browser.renderCall(args, { expanded: true, isPartial: true }, theme).render(120).join("\n"), + ); + + expect(rendered).toContain("run();"); + expect(rendered).toContain("finish();"); + expect(rendered).not.toContain("run();finish();"); + expect(args.code).toBe(source); + }); +}); diff --git a/packages/coding-agent/test/tools/eval-display-format.test.ts b/packages/coding-agent/test/tools/eval-display-format.test.ts new file mode 100644 index 000000000..beefedbaa --- /dev/null +++ b/packages/coding-agent/test/tools/eval-display-format.test.ts @@ -0,0 +1,67 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import type { EvalToolDetails } from "@oh-my-pi/pi-coding-agent/eval/types"; +import { getThemeByName, setThemeInstance, type Theme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; +import { EvalTool, evalToolRenderer } from "@oh-my-pi/pi-coding-agent/tools/eval"; + +describe("eval renderer: display-only streaming formatting", () => { + let theme: Theme; + const source = "if (ready) {run();finish();}"; + + beforeAll(async () => { + resetSettingsForTest(); + await Settings.init({ inMemory: true, cwd: process.cwd() }); + theme = (await getThemeByName("dark"))!; + expect(theme).toBeDefined(); + setThemeInstance(theme); + }); + + afterAll(() => { + resetSettingsForTest(); + }); + + it("expands compact source in both pending and completed previews", () => { + const pending = Bun.stripANSI( + evalToolRenderer + .renderCall({ language: "js", code: source }, { expanded: true, isPartial: true }, theme) + .render(120) + .join("\n"), + ); + const details: EvalToolDetails = { + language: "js", + languages: ["js"], + cells: [{ index: 0, code: source, language: "js", output: "", status: "complete" }], + }; + const completed = Bun.stripANSI( + evalToolRenderer + .renderResult( + { content: [{ type: "text", text: "" }], details }, + { expanded: true, isPartial: false }, + theme, + ) + .render(120) + .join("\n"), + ); + + for (const rendered of [pending, completed]) { + expect(rendered).toContain("run();"); + expect(rendered).toContain("finish();"); + expect(rendered).not.toContain("run();finish();"); + } + expect(details.cells?.[0]?.code).toBe(source); + }); + + it("passes the original source to execution verbatim", async () => { + let executed = ""; + const tool = new EvalTool(null, { + proxyExecutor: async params => { + executed = params.code; + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + }); + + await tool.execute("call", { language: "js", code: source }); + + expect(executed).toBe(source); + }); +}); diff --git a/packages/coding-agent/test/tools/eval-format-javascript.test.ts b/packages/coding-agent/test/tools/eval-format-javascript.test.ts new file mode 100644 index 000000000..66fd51128 --- /dev/null +++ b/packages/coding-agent/test/tools/eval-format-javascript.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "bun:test"; +import { formatJavaScriptForDisplay } from "@oh-my-pi/pi-coding-agent/tools/eval-format/javascript"; + +describe("formatJavaScriptForDisplay", () => { + it("expands compact control flow, objects, and arrays", () => { + expect(formatJavaScriptForDisplay("if (ready){const item = { value: 1 };use(item);}else{fallback();}")).toBe( + [ + "if (ready) {", + " const item = {", + " value: 1", + " };", + " use(item);", + "} else {", + " fallback();", + "}", + ].join("\n"), + ); + + expect(formatJavaScriptForDisplay("const rows = [{ value: 1 },{ value: 2 }];")).toBe( + [ + "const rows = [{", + " value: 1", + "}, {", + " value: 2", + "}];", + ].join("\n"), + ); + }); + + it("keeps for-loop header semicolons inline", () => { + expect(formatJavaScriptForDisplay("for(;;){tick();}for(let i=0;i<2;i++){work(i);}")).toBe( + [ + "for (;;) {", + " tick();", + "}", + "for (let i=0; i<2; i++) {", + " work(i);", + "}", + ].join("\n"), + ); + }); + + it("does not split literals, templates, regexes, or comments", () => { + const doubleQuoted = String.raw`"a\";{b}"`; + const singleQuoted = String.raw`'a\';{b}'`; + const template = '`raw;{${fn({ value: "}" })}}`'; + const regex = "/[;{}]+/g"; + const lineComment = "// keep ; { }"; + const blockComment = "/* keep ; { } */"; + const source = `const double = ${doubleQuoted};const single = ${singleQuoted};const template = ${template};const regex = ${regex}; ${lineComment}\n${blockComment}done();`; + + expect(formatJavaScriptForDisplay(source)).toBe( + [ + `const double = ${doubleQuoted};`, + `const single = ${singleQuoted};`, + `const template = ${template};`, + `const regex = ${regex}; ${lineComment}`, + `${blockComment}done();`, + ].join("\n"), + ); + }); + + it("returns unfinished literals, comments, and blocks without inventing closers", () => { + const samples: Array<{ source: string; expected: string }> = [ + { source: "const value = `raw;${call({ x: 1", expected: "const value = `raw;${call({ x: 1" }, + { source: "/* unfinished ; {", expected: "/* unfinished ; {" }, + { source: "// unfinished ; {", expected: "// unfinished ; {" }, + { source: "const pattern = /[;{]", expected: "const pattern = /[;{]" }, + { source: "if (ready){work();", expected: "if (ready) {\n work();" }, + ]; + + for (const sample of samples) { + expect(() => formatJavaScriptForDisplay(sample.source)).not.toThrow(); + expect(formatJavaScriptForDisplay(sample.source)).toBe(sample.expected); + } + }); + + it("is idempotent", () => { + const source = "try{const result = { ok: true };use(result);}catch(error){report(error);}finally{cleanup();}"; + const formatted = formatJavaScriptForDisplay(source); + expect(formatJavaScriptForDisplay(formatted)).toBe(formatted); + }); + + it("never changes already committed lines while a prefix grows", () => { + const source = "if(flag){const value={text:`a;${item}`};run(value);}else{for(;;){tick();}}"; + let committed: string[] = []; + + for (let end = 1; end <= source.length; end++) { + let formatted = ""; + expect(() => { + formatted = formatJavaScriptForDisplay(source.slice(0, end)); + }).not.toThrow(); + const lines = formatted.split("\n"); + const nextCommitted = lines.slice(0, -1); + expect(nextCommitted.slice(0, committed.length)).toEqual(committed); + committed = nextCommitted; + } + }); +}); diff --git a/packages/coding-agent/test/tools/eval-format-julia.test.ts b/packages/coding-agent/test/tools/eval-format-julia.test.ts new file mode 100644 index 000000000..b356d28ef --- /dev/null +++ b/packages/coding-agent/test/tools/eval-format-julia.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "bun:test"; +import { formatJuliaForDisplay } from "@oh-my-pi/pi-coding-agent/tools/eval-format/julia"; + +describe("formatJuliaForDisplay", () => { + it("expands genuinely compact nested blocks", () => { + const source = + "module Demo;function classify(xs);for x in xs;if x > 0;println(x);else;map(xs) do y;println(y);end;end;end;end;end"; + + expect(formatJuliaForDisplay(source)).toBe( + [ + "module Demo;", + " function classify(xs);", + " for x in xs;", + " if x > 0;", + " println(x);", + " else;", + " map(xs) do y;", + " println(y);", + " end;", + " end;", + " end;", + " end;", + "end", + ].join("\n"), + ); + }); + + it("leaves separators and block words inside literals and comments untouched", () => { + const source = + String.raw`function demo();text = "if; end \"quoted; else\" $(join(["catch;", "finally"], ";"))";mark = ';';hash = '#';# elseif; end` + + "\n#= outer; #= inner; end =# catch =#;return text;end"; + + expect(formatJuliaForDisplay(source)).toBe( + [ + "function demo();", + String.raw` text = "if; end \"quoted; else\" $(join(["catch;", "finally"], ";"))";`, + " mark = ';';", + " hash = '#';", + " # elseif; end", + " #= outer; #= inner; end =# catch =#;", + " return text;", + "end", + ].join("\n"), + ); + }); + + it("aligns branches around nested begin and try blocks", () => { + const source = + "try;value = begin;if ready;1;elseif waiting;2;else;3;end;end;catch err;handle(err);finally;cleanup();end"; + + expect(formatJuliaForDisplay(source)).toBe( + [ + "try;", + " value = begin;", + " if ready;", + " 1;", + " elseif waiting;", + " 2;", + " else;", + " 3;", + " end;", + " end;", + "catch err;", + " handle(err);", + "finally;", + " cleanup();", + "end", + ].join("\n"), + ); + }); + + it("formats unfinished triple-string, nested-comment, and block prefixes without closing them", () => { + const prefixes = [ + { + source: 'function f();text = """if; end\nstill', + expected: 'function f();\n text = """if; end\nstill', + }, + { + source: "if ready;#= outer; #= inner; end", + expected: "if ready;\n #= outer; #= inner; end", + }, + { + source: "module Prefix;function run();if ready;work()", + expected: "module Prefix;\n function run();\n if ready;\n work()", + }, + ]; + + for (const prefix of prefixes) { + expect(() => formatJuliaForDisplay(prefix.source)).not.toThrow(); + expect(formatJuliaForDisplay(prefix.source)).toBe(prefix.expected); + } + }); + + it("is idempotent", () => { + const compact = + "baremodule Stable;mutable struct Box;value;end;function run(box);if box.value > 0;box.value;else;0;end;end;end"; + const formatted = formatJuliaForDisplay(compact); + + expect(formatJuliaForDisplay(formatted)).toBe(formatted); + }); + + it("never changes committed lines while a source prefix grows", () => { + const source = + String.raw`function stream();if ready;message = "end; $(join(["a;b"], ";"))";` + + "\n#= outer #= ; end =# catch =#\nwork();else;wait();end;end"; + let prefix = ""; + let committed = ""; + + for (const character of source) { + prefix += character; + const formatted = formatJuliaForDisplay(prefix); + expect(formatted.startsWith(committed)).toBe(true); + const lastNewline = formatted.lastIndexOf("\n"); + if (lastNewline >= 0) committed = formatted.slice(0, lastNewline + 1); + } + }); +}); diff --git a/packages/coding-agent/test/tools/eval-format-python.test.ts b/packages/coding-agent/test/tools/eval-format-python.test.ts new file mode 100644 index 000000000..20c87a3d6 --- /dev/null +++ b/packages/coding-agent/test/tools/eval-format-python.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "bun:test"; +import { formatPythonForDisplay } from "@oh-my-pi/pi-coding-agent/tools/eval-format/python"; + +const compact = + 'class Classifier:def classify(self,value):if value>0:return "positive";elif value<0:return "negative";else:return "zero"'; + +const expandedCompact = [ + "class Classifier:", + " def classify(self,value):", + " if value>0:", + ' return "positive";', + " elif value<0:", + ' return "negative";', + " else:", + ' return "zero"', +].join("\n"); + +const lexicalSafetySource = String.raw`if ready:# keep ; and : exactly + text = "semi; escaped \" quote"; result = call([1; 2], {"k;": 3}) # tail ; : +else:# other + result = None`; + +const lexicalSafetyExpected = [ + "if ready:# keep ; and : exactly", + String.raw` text = "semi; escaped \" quote";`, + ' result = call([1; 2], {"k;": 3}) # tail ; :', + "else:# other", + " result = None", +].join("\n"); + +describe("formatPythonForDisplay", () => { + it("expands genuinely compact nested suites and compound clauses", () => { + expect(formatPythonForDisplay(compact)).toBe(expandedCompact); + }); + + it("only splits top-level semicolons and leaves strings, delimiters, and comments intact", () => { + expect(formatPythonForDisplay(lexicalSafetySource)).toBe(lexicalSafetyExpected); + }); + + it("keeps unfinished strings and headers conservative", () => { + const unfinished = 'def build():value = """open;:#\nstill open'; + expect(formatPythonForDisplay(unfinished)).toBe('def build():\n value = """open;:#\nstill open'); + expect(formatPythonForDisplay("while waiting:")).toBe("while waiting:"); + }); + + it("is idempotent for compact, readable, and incomplete prefixes", () => { + const readable = [ + "def outer(value):", + " if value:", + ' return {"items": [value; 2]}', + " else:", + " return None", + ].join("\n"); + const unfinished = 'def build():value = """open;:#\nstill open'; + + for (const source of [compact, lexicalSafetySource, readable, unfinished, "if pending:"]) { + const formatted = formatPythonForDisplay(source); + expect(formatPythonForDisplay(formatted)).toBe(formatted); + } + }); + + it("never changes lines committed by an earlier sequential prefix", () => { + const source = + 'async def choose(values):if values:item="a;b";elif fallback:item=call([1;2]);else:item="""open\nstill'; + const expected = [ + "async def choose(values):", + " if values:", + ' item="a;b";', + " elif fallback:", + " item=call([1;2]);", + " else:", + ' item="""open', + "still", + ].join("\n"); + let committed: string[] = []; + + for (let end = 0; end <= source.length; end++) { + const formatted = formatPythonForDisplay(source.slice(0, end)); + const nextCommitted = formatted.split("\n").slice(0, -1); + expect(nextCommitted.slice(0, committed.length)).toEqual(committed); + committed = nextCommitted; + } + expect(formatPythonForDisplay(source)).toBe(expected); + }); +});