From e4225d68294843e11023c0f586041b4792b7b281 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 22 Feb 2026 00:46:31 +0100 Subject: [PATCH] refactor(coding-agent): consolidated output utilities into streaming-output module - Consolidated truncation and output utilities from tools/truncate.ts and tools/output-utils.ts into session/streaming-output.ts with improved UTF-8 boundary handling. - Renamed formatSize() to formatBytes() across codebase for consistency and clarity in byte-level formatting. - Refactored OutputSink to use windowed byte truncation instead of full-buffer encoding, improving memory efficiency on large outputs. - Migrated from Buffer to Uint8Array in web scrapers for better cross-platform compatibility and native browser support. - Added getArtifactManager() lazy-initialization method to ToolSession for deferred artifact manager instantiation. - Simplified API surface with wildcard exports from tools and session modules, reducing import complexity. --- packages/coding-agent/CHANGELOG.md | 31 + packages/coding-agent/DEVELOPMENT.md | 2 +- .../coding-agent/src/cli/file-processor.ts | 6 +- packages/coding-agent/src/debug/index.ts | 7 +- packages/coding-agent/src/index.ts | 25 +- .../src/modes/components/bash-execution.ts | 4 +- .../src/modes/components/python-execution.ts | 4 +- .../src/modes/utils/ui-helpers.ts | 4 +- packages/coding-agent/src/sdk.ts | 14 + .../src/session/streaming-output.ts | 657 +++++++++++++++--- packages/coding-agent/src/task/index.ts | 9 +- packages/coding-agent/src/tools/bash.ts | 7 +- packages/coding-agent/src/tools/fetch.ts | 7 +- packages/coding-agent/src/tools/find.ts | 2 +- packages/coding-agent/src/tools/grep.ts | 2 +- packages/coding-agent/src/tools/index.ts | 32 +- .../coding-agent/src/tools/output-meta.ts | 7 +- .../coding-agent/src/tools/output-utils.ts | 63 -- packages/coding-agent/src/tools/python.ts | 14 +- packages/coding-agent/src/tools/read.ts | 60 +- packages/coding-agent/src/tools/ssh.ts | 7 +- .../coding-agent/src/tools/tool-result.ts | 3 +- packages/coding-agent/src/tools/truncate.ts | 385 ---------- .../coding-agent/src/utils/file-mentions.ts | 20 +- .../src/web/scrapers/dockerhub.ts | 10 +- .../coding-agent/src/web/scrapers/ollama.ts | 12 +- .../coding-agent/src/web/scrapers/utils.ts | 8 +- .../coding-agent/test/bash-executor.test.ts | 2 +- .../core/python-executor-streaming.test.ts | 2 +- .../test/core/python-executor.test.ts | 2 +- .../model-registry-runtime-provider.test.ts | 9 + .../test/streaming-output.test.ts | 353 ++++++++++ packages/coding-agent/test/tools.test.ts | 3 + 33 files changed, 1068 insertions(+), 705 deletions(-) delete mode 100644 packages/coding-agent/src/tools/output-utils.ts delete mode 100644 packages/coding-agent/src/tools/truncate.ts create mode 100644 packages/coding-agent/test/streaming-output.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 2a864e329..95caa273d 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,6 +1,37 @@ # Changelog ## [Unreleased] +### Added + +- Exported truncation utilities and streaming output types from `session/streaming-output` module for public use +- Added `TailBuffer` class for efficient ring-style buffering with lazy joining and windowed truncation +- Added `truncateTailBytes` and `truncateHeadBytes` functions for UTF-8-aware byte-level truncation +- Added `formatTailTruncationNotice` and `formatHeadTruncationNotice` functions for consistent truncation notice formatting +- Added `allocateOutputArtifact` function to allocate artifact paths for tool output spilling +- Added `getArtifactManager()` method to `ToolSession` for lazy artifact manager access + +### Changed + +- Moved truncation logic from `tools/truncate.ts` to `session/streaming-output.ts` for better architectural separation +- Renamed `formatSize()` to `formatBytes()` across codebase for consistency +- Refactored `OutputSink` to use windowed byte truncation for memory efficiency when spilling to files +- Changed `ToolSession.artifactManager` from cached property to `getArtifactManager()` method for lazy initialization +- Updated `allocateOutputArtifact` return type to use `{ path, id }` instead of `{ artifactPath, artifactId }` +- Optimized newline counting to use native `indexOf` for V8-optimized string scanning +- Improved UTF-8 boundary detection in byte truncation with dedicated helper functions + +### Removed + +- Deleted `tools/truncate.ts` module (functionality moved to `session/streaming-output.ts`) +- Deleted `tools/output-utils.ts` module (functionality moved to `session/streaming-output.ts`) +- Removed `formatSize` export from public API (renamed to `formatBytes`) +- Removed individual truncation function exports from `tools/index.ts` (now exported via `session/streaming-output`) + +### Fixed + +- Fixed UTF-8 boundary handling in byte truncation to prevent invalid character sequences +- Fixed memory efficiency in `OutputSink` by using windowed truncation instead of full-buffer encoding +- Fixed line counting to handle chunk boundaries correctly across multiple `push()` calls ## [12.18.1] - 2026-02-21 ### Added diff --git a/packages/coding-agent/DEVELOPMENT.md b/packages/coding-agent/DEVELOPMENT.md index ec4f46789..4698000b0 100644 --- a/packages/coding-agent/DEVELOPMENT.md +++ b/packages/coding-agent/DEVELOPMENT.md @@ -424,7 +424,7 @@ Key meta blocks: `formatOutputNotice(meta)` converts metadata into appended textual notices (for model visibility), including: - shown line range and total line count -- byte-limit context via `formatSize(...)` +- byte-limit context via `formatBytes(...)` - pagination hint (`nextOffset`) for head-truncated output - artifact recovery hint (`Full: artifact://`) - limit and diagnostics notices diff --git a/packages/coding-agent/src/cli/file-processor.ts b/packages/coding-agent/src/cli/file-processor.ts index e266fbbfd..b81791aaf 100644 --- a/packages/coding-agent/src/cli/file-processor.ts +++ b/packages/coding-agent/src/cli/file-processor.ts @@ -8,7 +8,7 @@ import { isEnoent } from "@oh-my-pi/pi-utils"; import { getProjectDir } from "@oh-my-pi/pi-utils/dirs"; import chalk from "chalk"; import { resolveReadPath } from "../tools/path-utils"; -import { formatSize } from "../tools/truncate"; +import { formatBytes } from "../tools/render-utils"; import { formatDimensionNote, resizeImage } from "../utils/image-resize"; import { detectSupportedImageMimeTypeFromFile } from "../utils/mime"; @@ -47,9 +47,9 @@ export async function processFileArguments(fileArgs: string[], options?: Process const maxBytes = mimeType ? MAX_CLI_IMAGE_BYTES : MAX_CLI_TEXT_BYTES; if (stat.size > maxBytes) { console.error( - chalk.yellow(`Warning: Skipping file contents (too large: ${formatSize(stat.size)}): ${absolutePath}`), + chalk.yellow(`Warning: Skipping file contents (too large: ${formatBytes(stat.size)}): ${absolutePath}`), ); - text += `(skipped: too large, ${formatSize(stat.size)})\n`; + text += `(skipped: too large, ${formatBytes(stat.size)})\n`; continue; } diff --git a/packages/coding-agent/src/debug/index.ts b/packages/coding-agent/src/debug/index.ts index 289960fb9..8123c033f 100644 --- a/packages/coding-agent/src/debug/index.ts +++ b/packages/coding-agent/src/debug/index.ts @@ -11,6 +11,7 @@ import { getSessionsDir } from "@oh-my-pi/pi-utils/dirs"; import { DynamicBorder } from "../modes/components/dynamic-border"; import { getSelectListTheme, getSymbolTheme, theme } from "../modes/theme/theme"; import type { InteractiveModeContext } from "../modes/types"; +import { formatBytes } from "../tools/render-utils"; import { openPath } from "../utils/open"; import { DebugLogViewerComponent } from "./log-viewer"; import { generateHeapSnapshotData, type ProfilerSession, startCpuProfile } from "./profiler"; @@ -417,12 +418,6 @@ export class DebugSelectorComponent extends Container { } } -function formatBytes(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; -} - /** * Show the debug selector. */ diff --git a/packages/coding-agent/src/index.ts b/packages/coding-agent/src/index.ts index ec269e5ad..47f2c73f1 100644 --- a/packages/coding-agent/src/index.ts +++ b/packages/coding-agent/src/index.ts @@ -243,27 +243,4 @@ export { export { runSubprocess } from "./task/executor"; export type { AgentDefinition, AgentProgress, AgentSource, SingleResult, TaskParams } from "./task/types"; // Tools (detail types and utilities) -export { - type BashToolDetails, - type BashToolInput, - type BrowserToolDetails, - DEFAULT_MAX_BYTES, - DEFAULT_MAX_LINES, - type FindOperations, - type FindToolDetails, - type FindToolInput, - type FindToolOptions, - formatSize, - type GrepToolDetails, - type GrepToolInput, - type PythonToolDetails, - type ReadToolDetails, - type ReadToolInput, - type TruncationOptions, - type TruncationResult, - truncateHead, - truncateLine, - truncateTail, - type WriteToolDetails, - type WriteToolInput, -} from "./tools"; +export * from "./tools"; diff --git a/packages/coding-agent/src/modes/components/bash-execution.ts b/packages/coding-agent/src/modes/components/bash-execution.ts index 4a15207a2..ad15f1276 100644 --- a/packages/coding-agent/src/modes/components/bash-execution.ts +++ b/packages/coding-agent/src/modes/components/bash-execution.ts @@ -6,7 +6,7 @@ import { sanitizeText } from "@oh-my-pi/pi-natives"; import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui"; import { getSymbolTheme, theme } from "../../modes/theme/theme"; import type { TruncationMeta } from "../../tools/output-meta"; -import { formatSize } from "../../tools/truncate"; +import { formatBytes } from "../../tools/render-utils"; import { DynamicBorder } from "./dynamic-border"; import { truncateToVisualLines } from "./visual-truncate"; @@ -177,7 +177,7 @@ export class BashExecutionComponent extends Container { ); } else { warnings.push( - `Truncated: ${this.#truncation.outputLines} lines shown (${formatSize(this.#truncation.outputBytes)} limit)`, + `Truncated: ${this.#truncation.outputLines} lines shown (${formatBytes(this.#truncation.outputBytes)} limit)`, ); } statusParts.push(theme.fg("warning", warnings.join(". "))); diff --git a/packages/coding-agent/src/modes/components/python-execution.ts b/packages/coding-agent/src/modes/components/python-execution.ts index d9a69d808..11436dce9 100644 --- a/packages/coding-agent/src/modes/components/python-execution.ts +++ b/packages/coding-agent/src/modes/components/python-execution.ts @@ -7,7 +7,7 @@ import { sanitizeText } from "@oh-my-pi/pi-natives"; import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui"; import { getSymbolTheme, highlightCode, theme } from "../../modes/theme/theme"; import type { TruncationMeta } from "../../tools/output-meta"; -import { formatSize } from "../../tools/truncate"; +import { formatBytes } from "../../tools/render-utils"; import { DynamicBorder } from "./dynamic-border"; import { truncateToVisualLines } from "./visual-truncate"; @@ -161,7 +161,7 @@ export class PythonExecutionComponent extends Container { ); } else { warnings.push( - `Truncated: ${this.#truncation.outputLines} lines shown (${formatSize(this.#truncation.outputBytes)} limit)`, + `Truncated: ${this.#truncation.outputLines} lines shown (${formatBytes(this.#truncation.outputBytes)} limit)`, ); } statusParts.push(theme.fg("warning", warnings.join(". "))); diff --git a/packages/coding-agent/src/modes/utils/ui-helpers.ts b/packages/coding-agent/src/modes/utils/ui-helpers.ts index 380ef1c34..5fd7b85eb 100644 --- a/packages/coding-agent/src/modes/utils/ui-helpers.ts +++ b/packages/coding-agent/src/modes/utils/ui-helpers.ts @@ -17,7 +17,7 @@ import { theme } from "../../modes/theme/theme"; import type { CompactionQueuedMessage, InteractiveModeContext } from "../../modes/types"; import { type CustomMessage, SKILL_PROMPT_MESSAGE_TYPE, type SkillPromptDetails } from "../../session/messages"; import type { SessionContext } from "../../session/session-manager"; -import { formatSize } from "../../tools/truncate"; +import { formatBytes } from "../../tools/render-utils"; type TextBlock = { type: "text"; text: string }; @@ -130,7 +130,7 @@ export class UiHelpers { for (const file of message.files) { let suffix: string; if (file.skippedReason === "tooLarge") { - const size = typeof file.byteSize === "number" ? formatSize(file.byteSize) : "unknown size"; + const size = typeof file.byteSize === "number" ? formatBytes(file.byteSize) : "unknown size"; suffix = `(skipped: ${size})`; } else { suffix = file.image diff --git a/packages/coding-agent/src/sdk.ts b/packages/coding-agent/src/sdk.ts index b79f98f09..2e4ca2917 100644 --- a/packages/coding-agent/src/sdk.ts +++ b/packages/coding-agent/src/sdk.ts @@ -13,6 +13,7 @@ import { loadPromptTemplates as loadPromptTemplatesInternal, type PromptTemplate import { Settings, type SkillsSettings } from "./config/settings"; import { CursorExecHandlers } from "./cursor"; import "./discovery"; +import { ArtifactManager } from "@oh-my-pi/pi-coding-agent/session/artifacts"; import { initializeWithSettings } from "./discovery"; import { TtsrManager } from "./export/ttsr"; import { @@ -715,6 +716,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} const enableLsp = options.enableLsp ?? true; + let artifactManager: ArtifactManager | null = null; const toolSession: ToolSession = { cwd, hasUI: options.hasUI ?? false, @@ -742,6 +744,18 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} }, getPlanModeState: () => session.getPlanModeState(), getCompactContext: () => session.formatCompactContext(), + getArtifactManager: () => { + if (artifactManager) { + return artifactManager; + } + const sessionFile = sessionManager.getSessionFile(); + if (!sessionFile) { + return null; + } + const manager = new ArtifactManager(sessionFile); + artifactManager = manager; + return manager; + }, settings, authStorage, modelRegistry, diff --git a/packages/coding-agent/src/session/streaming-output.ts b/packages/coding-agent/src/session/streaming-output.ts index 0cd01a59a..cf5bd38f8 100644 --- a/packages/coding-agent/src/session/streaming-output.ts +++ b/packages/coding-agent/src/session/streaming-output.ts @@ -1,5 +1,18 @@ import { sanitizeText } from "@oh-my-pi/pi-natives"; -import { DEFAULT_MAX_BYTES } from "../tools/truncate"; +import type { ToolSession } from "../tools"; +import { formatBytes } from "../tools/render-utils"; + +// ============================================================================= +// Constants +// ============================================================================= + +export const DEFAULT_MAX_LINES = 3000; +export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB +export const DEFAULT_MAX_COLUMN = 1024; // Max chars per grep match line + +// ============================================================================= +// Interfaces +// ============================================================================= export interface OutputSummary { output: string; @@ -19,40 +32,438 @@ export interface OutputSinkOptions { onChunk?: (chunk: string) => void; } +export interface TruncationResult { + content: string; + truncated: boolean; + truncatedBy: "lines" | "bytes" | null; + totalLines: number; + totalBytes: number; + outputLines: number; + outputBytes: number; + lastLinePartial: boolean; + firstLineExceedsLimit: boolean; + maxLines: number; + maxBytes: number; +} + +export interface TruncationOptions { + /** Maximum number of lines (default: 3000) */ + maxLines?: number; + /** Maximum number of bytes (default: 50KB) */ + maxBytes?: number; +} + +/** Result from byte-level truncation helpers. */ +export interface ByteTruncationResult { + text: string; + bytes: number; +} + +export interface TailTruncationNoticeOptions { + fullOutputPath?: string; + originalContent?: string; + suffix?: string; +} + +export interface HeadTruncationNoticeOptions { + startLine?: number; + totalFileLines?: number; +} + +// ============================================================================= +// Low-level byte utilities +// +// Both use a "windowed encoding" strategy for strings: instead of encoding the +// entire input with Buffer.from(), we encode a window of at most `maxBytes` +// code units from the relevant end. Every JS code unit produces ≥1 UTF-8 byte, +// so this window is guaranteed to contain ≥ maxBytes encoded bytes (unless the +// input is shorter), avoiding O(N) encoding cost on large strings. +// ============================================================================= + +/** Advance past UTF-8 continuation bytes (10xxxxxx) to a leading byte. */ +function findUtf8BoundaryForward(buf: Buffer, pos: number): number { + while (pos < buf.length && (buf[pos] & 0xc0) === 0x80) pos++; + return pos; +} + +/** Retreat past UTF-8 continuation bytes to land on a leading byte. */ +function findUtf8BoundaryBackward(buf: Buffer, pos: number): number { + while (pos > 0 && (buf[pos] & 0xc0) === 0x80) pos--; + return pos; +} + +/** + * Truncate a string/buffer to fit within a byte limit, keeping the tail. + * Handles multi-byte UTF-8 boundaries correctly. + */ +export function truncateTailBytes(data: string | Uint8Array, maxBytes: number): ByteTruncationResult { + if (typeof data === "string") { + const len = Buffer.byteLength(data, "utf-8"); + if (len <= maxBytes) return { text: data, bytes: len }; + + // Windowed: encode only the last `maxBytes` code units (≥ maxBytes bytes). + const window = data.substring(Math.max(0, data.length - maxBytes)); + const buf = Buffer.from(window, "utf-8"); + const start = findUtf8BoundaryForward(buf, buf.length - maxBytes); + const slice = buf.subarray(start); + return { text: slice.toString("utf-8"), bytes: slice.length }; + } + + // Uint8Array / Buffer path + if (data.length <= maxBytes) return { text: Buffer.from(data).toString("utf-8"), bytes: data.length }; + const buf = Buffer.isBuffer(data) ? data : Buffer.from(data); + const start = findUtf8BoundaryForward(buf, buf.length - maxBytes); + const slice = buf.subarray(start); + return { text: slice.toString("utf-8"), bytes: slice.length }; +} + +/** + * Truncate a string/buffer to fit within a byte limit, keeping the head. + * Handles multi-byte UTF-8 boundaries correctly. + */ +export function truncateHeadBytes(data: string | Uint8Array, maxBytes: number): ByteTruncationResult { + if (typeof data === "string") { + const len = Buffer.byteLength(data, "utf-8"); + if (len <= maxBytes) return { text: data, bytes: len }; + + // Windowed: encode only the first `maxBytes` code units. + const window = data.substring(0, maxBytes); + const buf = Buffer.from(window, "utf-8"); + const end = findUtf8BoundaryBackward(buf, maxBytes); + if (end <= 0) return { text: "", bytes: 0 }; + const slice = buf.subarray(0, end); + return { text: slice.toString("utf-8"), bytes: slice.length }; + } + + if (data.length <= maxBytes) return { text: Buffer.from(data).toString("utf-8"), bytes: data.length }; + const buf = Buffer.isBuffer(data) ? data : Buffer.from(data); + const end = findUtf8BoundaryBackward(buf, maxBytes); + if (end <= 0) return { text: "", bytes: 0 }; + const slice = buf.subarray(0, end); + return { text: slice.toString("utf-8"), bytes: slice.length }; +} + +// ============================================================================= +// Line-level utilities +// ============================================================================= + +/** + * Count newline characters. Uses indexOf for V8-optimized native string scanning. + */ function countNewlines(text: string): number { let count = 0; - for (let i = 0; i < text.length; i++) { - if (text.charCodeAt(i) === 10) count += 1; + let pos = text.indexOf("\n", 0); + while (pos !== -1) { + count++; + pos = text.indexOf("\n", pos + 1); } return count; } -function countLines(text: string): number { - if (text.length === 0) return 0; - return countNewlines(text) + 1; +/** + * Truncate a single line to max characters, appending '…' if truncated. + */ +export function truncateLine( + line: string, + maxChars: number = DEFAULT_MAX_COLUMN, +): { text: string; wasTruncated: boolean } { + if (line.length <= maxChars) return { text: line, wasTruncated: false }; + return { text: `${line.slice(0, maxChars)}…`, wasTruncated: true }; } -function truncateStringToBytesFromEnd(text: string, maxBytes: number): { text: string; bytes: number } { - const buf = Buffer.from(text, "utf-8"); - if (buf.length <= maxBytes) { - return { text, bytes: buf.length }; - } +// ============================================================================= +// Content truncation (line + byte aware) +// +// Both truncateHead and truncateTail encode the content to a Buffer once and +// collect newline byte-offsets in a single forward pass, avoiding split("\n") +// which would allocate N intermediate strings for N lines. +// ============================================================================= - let start = buf.length - maxBytes; - while (start < buf.length && (buf[start] & 0xc0) === 0x80) { - start++; - } - - const sliced = buf.subarray(start).toString("utf-8"); - return { text: sliced, bytes: Buffer.byteLength(sliced, "utf-8") }; +/** Shared helper to build a no-truncation result. */ +function noTruncResult( + content: string, + totalLines: number, + totalBytes: number, + maxLines: number, + maxBytes: number, +): TruncationResult { + return { + content, + truncated: false, + truncatedBy: null, + totalLines, + totalBytes, + outputLines: totalLines, + outputBytes: totalBytes, + lastLinePartial: false, + firstLineExceedsLimit: false, + maxLines, + maxBytes, + }; } /** - * Line-buffered output sink with file spill support. + * Collect byte-offsets of every 0x0a in a Buffer. Avoids re-scanning. * - * Uses a single string buffer with line position tracking. - * When memory limit exceeded, spills ~half to file in one batch operation. + * For a buffer with lines [L0 \n L1 \n L2]: + * nlOffsets = [offset_of_first_\n, offset_of_second_\n] + * Line k starts at (k === 0 ? 0 : nlOffsets[k-1] + 1) + * Line k ends at (k < nlOffsets.length ? nlOffsets[k] - 1 : buf.length - 1) */ +function collectNewlineOffsets(buf: Buffer): number[] { + const offsets: number[] = []; + let pos = Bun.indexOfLine(buf, 0); + while (pos !== -1) { + offsets.push(pos); + pos = Bun.indexOfLine(buf, pos + 1); + } + return offsets; +} + +/** + * Truncate content from the head (keep first N lines/bytes). + * Suitable for file reads where you want to see the beginning. + * + * Uses indexOfLine to scan forward incrementally — stops as soon as the + * line or byte limit is hit without ever collecting all newline positions. + * + * Never returns partial lines. If the first line exceeds the byte limit, + * returns empty content with firstLineExceedsLimit=true. + */ +export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult { + const maxLines = options.maxLines ?? DEFAULT_MAX_LINES; + const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; + const totalBytes = Buffer.byteLength(content, "utf-8"); + + // Fast path: if under byte limit, only need a string-level newline count + if (totalBytes <= maxBytes) { + const totalLines = countNewlines(content) + 1; + if (totalLines <= maxLines) { + return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes); + } + } + + // Slow path: encode once, scan with native indexOfLine + const totalLines = countNewlines(content) + 1; + if (totalLines <= maxLines && totalBytes <= maxBytes) { + return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes); + } + + // Forward scan: include complete lines that fit within both limits. + // Uses indexOfLine incrementally — no offset array needed. + let includedLines = 0; + let cutByte = 0; + let truncatedBy: "lines" | "bytes" = "lines"; + let scanPos = 0; + + const buf = Buffer.from(content, "utf-8"); + while (includedLines < maxLines) { + const nlPos = Bun.indexOfLine(buf, scanPos); + // Byte position right after this line's trailing \n, or end-of-buffer + const lineEnd = nlPos === -1 ? buf.length : nlPos + 1; + // Output strips a trailing \n from the final slice; enforce limit on displayed bytes + const outputEnd = nlPos === -1 ? lineEnd : lineEnd - 1; + + if (outputEnd > maxBytes) { + truncatedBy = "bytes"; + if (includedLines === 0) { + return { + content: "", + truncated: true, + truncatedBy: "bytes", + totalLines, + totalBytes, + outputLines: 0, + outputBytes: 0, + lastLinePartial: false, + firstLineExceedsLimit: true, + maxLines, + maxBytes, + }; + } + break; + } + + cutByte = lineEnd; + includedLines++; + + if (nlPos === -1) break; // no more lines + scanPos = nlPos + 1; + } + + if (includedLines >= maxLines && cutByte <= maxBytes) { + truncatedBy = "lines"; + } + + // Strip trailing \n so output matches join("\n") semantics + const sliceEnd = cutByte > 0 && buf[cutByte - 1] === 0x0a ? cutByte - 1 : cutByte; + const outputContent = buf.subarray(0, sliceEnd).toString("utf-8"); + + return { + content: outputContent, + truncated: true, + truncatedBy, + totalLines, + totalBytes, + outputLines: includedLines, + outputBytes: sliceEnd, + lastLinePartial: false, + firstLineExceedsLimit: false, + maxLines, + maxBytes, + }; +} + +/** + * Truncate content from the tail (keep last N lines/bytes). + * Suitable for bash output where you want to see the end (errors, final results). + * + * May return a partial first line if the last line exceeds the byte limit. + */ +export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult { + const maxLines = options.maxLines ?? DEFAULT_MAX_LINES; + const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; + const totalBytes = Buffer.byteLength(content, "utf-8"); + const totalLines = countNewlines(content) + 1; + + if (totalLines <= maxLines && totalBytes <= maxBytes) { + return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes); + } + + const buf = Buffer.from(content, "utf-8"); + const nlOffsets = collectNewlineOffsets(buf); + const lineCount = nlOffsets.length + 1; + + // Walk backward, accumulating complete lines that fit. + let includedLines = 0; + let startByte = buf.length; + let truncatedBy: "lines" | "bytes" = "lines"; + + for (let lineIdx = lineCount - 1; lineIdx >= 0 && includedLines < maxLines; lineIdx--) { + const lineStart = lineIdx === 0 ? 0 : nlOffsets[lineIdx - 1] + 1; + const spanBytes = buf.length - lineStart; + + if (spanBytes > maxBytes) { + truncatedBy = "bytes"; + if (includedLines === 0) { + // Last line alone exceeds byte limit — take its tail + const tailResult = truncateTailBytes(buf, maxBytes); + return { + content: tailResult.text, + truncated: true, + truncatedBy: "bytes", + totalLines, + totalBytes, + outputLines: 1, + outputBytes: tailResult.bytes, + lastLinePartial: true, + firstLineExceedsLimit: false, + maxLines, + maxBytes, + }; + } + break; + } + + startByte = lineStart; + includedLines++; + } + + if (includedLines >= maxLines && buf.length - startByte <= maxBytes) { + truncatedBy = "lines"; + } + + const outputBytes = buf.length - startByte; + const outputContent = buf.subarray(startByte).toString("utf-8"); + + return { + content: outputContent, + truncated: true, + truncatedBy, + totalLines, + totalBytes, + outputLines: includedLines, + outputBytes, + lastLinePartial: false, + firstLineExceedsLimit: false, + maxLines, + maxBytes, + }; +} + +// ============================================================================= +// TailBuffer — ring-style tail buffer with lazy joining +// +// Uses windowed truncateTailBytes for trimming, so only a suffix of the +// accumulated string is ever encoded — not the entire buffer. +// ============================================================================= + +const MAX_PENDING = 10; + +export class TailBuffer { + #pending: string[] = []; + #pos = 0; // tracked byte count (approximate after trims) + + constructor(readonly maxBytes: number) {} + + append(text: string): void { + if (!text) return; + const n = Buffer.byteLength(text, "utf-8"); + this.#pos += n; + + if (this.#pending.length > 0) { + this.#pending.push(text); + if (this.#pending.length > MAX_PENDING) this.#compact(); + // Trim when we exceed 2× budget to amortise cost + if (this.#pos > this.maxBytes * 2) this.#trim(); + } else { + this.#pending[0] = text; + this.#pending.length = 1; + } + } + + text(): string { + return this.#trim(); + } + + bytes(): number { + return this.#pos; + } + + // -- private --------------------------------------------------------------- + + #compact(): void { + this.#pending[0] = this.#pending.join(""); + this.#pending.length = 1; + } + + #flush(): string { + if (this.#pending.length === 0) return ""; + if (this.#pending.length > 1) this.#compact(); + return this.#pending[0]; + } + + /** Trim the buffer to maxBytes using windowed tail truncation. */ + #trim(): string { + if (this.#pos <= this.maxBytes) return this.#flush(); + + const joined = this.#flush(); + const { text, bytes } = truncateTailBytes(joined, this.maxBytes); + this.#pos = bytes; + this.#pending[0] = text; + this.#pending.length = 1; + return text; + } +} + +// ============================================================================= +// OutputSink — line-buffered output with file spill support +// +// Uses a string buffer with byte tracking. When the spill threshold is +// exceeded, all data is written to a file sink and the in-memory buffer is +// trimmed to a tail window via windowed truncateTailBytes. +// ============================================================================= + export class OutputSink { #buffer = ""; #bufferBytes = 0; @@ -60,11 +471,13 @@ export class OutputSink { #totalBytes = 0; #sawData = false; #truncated = false; + #file?: { path: string; artifactId?: string; sink: Bun.FileSink; }; + readonly #artifactPath?: string; readonly #artifactId?: string; readonly #spillThreshold: number; @@ -72,66 +485,44 @@ export class OutputSink { constructor(options?: OutputSinkOptions) { const { artifactPath, artifactId, spillThreshold = DEFAULT_MAX_BYTES, onChunk } = options ?? {}; - this.#artifactPath = artifactPath; this.#artifactId = artifactId; this.#spillThreshold = spillThreshold; this.#onChunk = onChunk; } - async #pushSanitized(data: string): Promise { - this.#onChunk?.(data); - - const dataBytes = Buffer.byteLength(data, "utf-8"); - this.#totalBytes += dataBytes; - if (data.length > 0) { - this.#sawData = true; - this.#totalLines += countNewlines(data); - } - - const bufferOverflow = this.#bufferBytes + dataBytes > this.#spillThreshold; - const overflow = this.#file || bufferOverflow; - const sink = overflow ? await this.#fileSink() : null; - - this.#buffer += data; - this.#bufferBytes += dataBytes; - await sink?.write(data); - - if (bufferOverflow) { - this.#truncated = true; - const trimmed = truncateStringToBytesFromEnd(this.#buffer, this.#spillThreshold); - this.#buffer = trimmed.text; - this.#bufferBytes = trimmed.bytes; - } - if (this.#file) { - this.#truncated = true; - } - } - - async #fileSink(): Promise { - if (!this.#artifactPath) return null; - if (!this.#file) { - try { - this.#file = { - path: this.#artifactPath, - artifactId: this.#artifactId, - sink: Bun.file(this.#artifactPath).writer(), - }; - await this.#file.sink.write(this.#buffer); - } catch { - try { - await this.#file?.sink?.end(); - } catch {} - this.#file = undefined; - return null; - } - } - return this.#file.sink; - } - async push(chunk: string): Promise { chunk = sanitizeText(chunk); - await this.#pushSanitized(chunk); + this.#onChunk?.(chunk); + + const dataBytes = Buffer.byteLength(chunk, "utf-8"); + this.#totalBytes += dataBytes; + + if (chunk.length > 0) { + this.#sawData = true; + this.#totalLines += countNewlines(chunk); + } + + const willOverflow = this.#bufferBytes + dataBytes > this.#spillThreshold; + + // Write to file if already spilling or about to overflow + if (this.#file != null || willOverflow) { + const sink = await this.#ensureFileSink(); + await sink?.write(chunk); + } + + this.#buffer += chunk; + this.#bufferBytes += dataBytes; + + // Keep only a tail window in memory when overflowing + if (willOverflow) { + this.#truncated = true; + const { text, bytes } = truncateTailBytes(this.#buffer, this.#spillThreshold); + this.#buffer = text; + this.#bufferBytes = bytes; + } + + if (this.#file) this.#truncated = true; } createInput(): WritableStream { @@ -139,14 +530,9 @@ export class OutputSink { const finalize = async () => { await this.push(dec.decode()); }; - return new WritableStream({ write: async chunk => { - if (typeof chunk === "string") { - await this.push(chunk); - } else { - await this.push(dec.decode(chunk, { stream: true })); - } + await this.push(typeof chunk === "string" ? chunk : dec.decode(chunk, { stream: true })); }, close: finalize, abort: finalize, @@ -155,23 +541,120 @@ export class OutputSink { async dump(notice?: string): Promise { const noticeLine = notice ? `[${notice}]\n` : ""; - const outputLines = countLines(this.#buffer); - const outputBytes = this.#bufferBytes; + const outputLines = this.#buffer.length > 0 ? countNewlines(this.#buffer) + 1 : 0; const totalLines = this.#sawData ? this.#totalLines + 1 : 0; - const totalBytes = this.#totalBytes; - if (this.#file) { - await this.#file.sink.end(); - } + if (this.#file) await this.#file.sink.end(); return { output: `${noticeLine}${this.#buffer}`, truncated: this.#truncated, totalLines, - totalBytes, + totalBytes: this.#totalBytes, outputLines, - outputBytes, + outputBytes: this.#bufferBytes, artifactId: this.#file?.artifactId, }; } + + // -- private --------------------------------------------------------------- + + async #ensureFileSink(): Promise { + if (!this.#artifactPath) return null; + if (this.#file) return this.#file.sink; + + try { + const sink = Bun.file(this.#artifactPath).writer(); + this.#file = { path: this.#artifactPath, artifactId: this.#artifactId, sink }; + // Flush existing buffer to file BEFORE it gets trimmed + await sink.write(this.#buffer); + return sink; + } catch { + try { + await this.#file?.sink?.end(); + } catch { + /* ignore */ + } + this.#file = undefined; + return null; + } + } +} + +// ============================================================================= +// Session helpers +// ============================================================================= + +const kEmpty = Object.freeze({} as { id?: string; path?: string }); + +/** Allocate a new artifact path and ID without writing content. */ +export async function allocateOutputArtifact(session: ToolSession, toolType: string) { + const manager = session.getArtifactManager?.(); + if (!manager) return kEmpty; + + try { + return await manager.allocatePath(toolType); + } catch { + return kEmpty; + } +} + +// ============================================================================= +// Truncation notice formatting +// ============================================================================= + +/** + * Format a truncation notice for tail-truncated output (bash, python, ssh). + * Returns empty string if not truncated. + */ +export function formatTailTruncationNotice( + truncation: TruncationResult, + options: TailTruncationNoticeOptions = {}, +): string { + if (!truncation.truncated) return ""; + + const { fullOutputPath, originalContent, suffix = "" } = options; + const startLine = truncation.totalLines - truncation.outputLines + 1; + const endLine = truncation.totalLines; + const fullOutputPart = fullOutputPath ? `. Full output: ${fullOutputPath}` : ""; + + let notice: string; + if (truncation.lastLinePartial) { + let lastLineSizePart = ""; + if (originalContent) { + const lastNl = originalContent.lastIndexOf("\n"); + const lastLine = lastNl === -1 ? originalContent : originalContent.substring(lastNl + 1); + lastLineSizePart = ` (line is ${formatBytes(Buffer.byteLength(lastLine, "utf-8"))})`; + } + notice = `[Showing last ${formatBytes(truncation.outputBytes)} of line ${endLine}${lastLineSizePart}${fullOutputPart}${suffix}]`; + } else if (truncation.truncatedBy === "lines") { + notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}${fullOutputPart}${suffix}]`; + } else { + notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatBytes(truncation.maxBytes)} limit)${fullOutputPart}${suffix}]`; + } + + return `\n\n${notice}`; +} + +/** + * Format a truncation notice for head-truncated output (read tool). + * Returns empty string if not truncated. + */ +export function formatHeadTruncationNotice( + truncation: TruncationResult, + options: HeadTruncationNoticeOptions = {}, +): string { + if (!truncation.truncated) return ""; + + const startLineDisplay = options.startLine ?? 1; + const totalFileLines = options.totalFileLines ?? truncation.totalLines; + const endLineDisplay = startLineDisplay + truncation.outputLines - 1; + const nextOffset = endLineDisplay + 1; + + const notice = + truncation.truncatedBy === "lines" + ? `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]` + : `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines} (${formatBytes(truncation.maxBytes)} limit). Use offset=${nextOffset} to continue]`; + + return `\n\n${notice}`; } diff --git a/packages/coding-agent/src/task/index.ts b/packages/coding-agent/src/task/index.ts index 05b14f1e4..6d88ac500 100644 --- a/packages/coding-agent/src/task/index.ts +++ b/packages/coding-agent/src/task/index.ts @@ -26,7 +26,7 @@ import type { Theme } from "../modes/theme/theme"; import planModeSubagentPrompt from "../prompts/system/plan-mode-subagent.md" with { type: "text" }; import taskDescriptionTemplate from "../prompts/tools/task.md" with { type: "text" }; import taskSummaryTemplate from "../prompts/tools/task-summary.md" with { type: "text" }; -import { formatDuration } from "../tools/render-utils"; +import { formatBytes, formatDuration } from "../tools/render-utils"; // Import review tools for side effects (registers subagent tool handlers) import "../tools/review"; import { discoverAgents, getAgent } from "./discovery"; @@ -55,13 +55,6 @@ import { type WorktreeBaseline, } from "./worktree"; -/** Format byte count for display */ -function formatBytes(bytes: number): string { - if (bytes < 1024) return `${bytes}B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}K`; - return `${(bytes / (1024 * 1024)).toFixed(1)}M`; -} - function createUsageTotals(): Usage { return { input: 0, diff --git a/packages/coding-agent/src/tools/bash.ts b/packages/coding-agent/src/tools/bash.ts index 73bcf5de2..e20c24245 100644 --- a/packages/coding-agent/src/tools/bash.ts +++ b/packages/coding-agent/src/tools/bash.ts @@ -12,6 +12,7 @@ import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import { truncateToVisualLines } from "../modes/components/visual-truncate"; import type { Theme } from "../modes/theme/theme"; import bashDescription from "../prompts/tools/bash.md" with { type: "text" }; +import { allocateOutputArtifact, DEFAULT_MAX_BYTES, TailBuffer } from "../session/streaming-output"; import { renderStatusLine } from "../tui"; import { CachedOutputBlock } from "../tui/output-block"; import type { ToolSession } from "."; @@ -20,12 +21,10 @@ import { checkBashInterception } from "./bash-interceptor"; import { applyHeadTail } from "./bash-normalize"; import { expandInternalUrls } from "./bash-skill-urls"; import type { OutputMeta } from "./output-meta"; -import { allocateOutputArtifact, createTailBuffer } from "./output-utils"; import { resolveToCwd } from "./path-utils"; import { formatBytes, replaceTabs, wrapBrackets } from "./render-utils"; import { ToolAbortError, ToolError } from "./tool-errors"; import { toolResult } from "./tool-result"; -import { DEFAULT_MAX_BYTES } from "./truncate"; export const BASH_DEFAULT_PREVIEW_LINES = 10; @@ -114,12 +113,12 @@ export class BashTool implements AgentTool { const timeoutMs = timeoutSec * 1000; // Track output for streaming updates (tail only) - const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES); + const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES); // Set up artifacts environment and allocation const artifactsDir = this.session.getArtifactsDir?.(); const extraEnv = artifactsDir ? { ARTIFACTS: artifactsDir } : undefined; - const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "bash"); + const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "bash"); const usePty = this.session.settings.get("bash.virtualTerminal") === "on" && diff --git a/packages/coding-agent/src/tools/fetch.ts b/packages/coding-agent/src/tools/fetch.ts index 76654b1f0..a9eecbdb9 100644 --- a/packages/coding-agent/src/tools/fetch.ts +++ b/packages/coding-agent/src/tools/fetch.ts @@ -10,6 +10,7 @@ import { renderPromptTemplate } from "../config/prompt-templates"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import { type Theme, theme } from "../modes/theme/theme"; import fetchDescription from "../prompts/tools/fetch.md" with { type: "text" }; +import { allocateOutputArtifact, DEFAULT_MAX_BYTES, truncateHead } from "../session/streaming-output"; import { renderStatusLine } from "../tui"; import { CachedOutputBlock } from "../tui/output-block"; import { ensureTool } from "../utils/tools-manager"; @@ -20,11 +21,9 @@ import { convertWithMarkitdown, fetchBinary } from "../web/scrapers/utils"; import type { ToolSession } from "."; import { applyListLimit } from "./list-limit"; import type { OutputMeta } from "./output-meta"; -import { allocateOutputArtifact } from "./output-utils"; import { formatExpandHint } from "./render-utils"; import { ToolAbortError } from "./tool-errors"; import { toolResult } from "./tool-result"; -import { DEFAULT_MAX_BYTES, truncateHead } from "./truncate"; // ============================================================================= // Types and Constants @@ -900,10 +899,10 @@ export class FetchTool implements AgentTool string | null; /** Get session ID */ getSessionId?: () => string | null; - /** Cached artifact manager (allocated per ToolSession) */ - artifactManager?: ArtifactManager; + /** Get artifact manager (allocated per ToolSession) */ + getArtifactManager?: () => ArtifactManager | null; /** Get artifacts directory for artifact:// URLs and $ARTIFACTS env var */ getArtifactsDir?: () => string | null; /** Get session spawns */ diff --git a/packages/coding-agent/src/tools/output-meta.ts b/packages/coding-agent/src/tools/output-meta.ts index db049e4b1..d789fe81c 100644 --- a/packages/coding-agent/src/tools/output-meta.ts +++ b/packages/coding-agent/src/tools/output-meta.ts @@ -12,10 +12,9 @@ import type { AgentToolUpdateCallback, } from "@oh-my-pi/pi-agent-core"; import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai"; -import type { OutputSummary } from "../session/streaming-output"; +import type { OutputSummary, TruncationResult } from "../session/streaming-output"; +import { formatBytes } from "./render-utils"; import { renderError } from "./tool-errors"; -import type { TruncationResult } from "./truncate"; -import { formatSize } from "./truncate"; /** * Truncation metadata for the output notice. @@ -336,7 +335,7 @@ export function formatOutputNotice(meta: OutputMeta | undefined): string { if (t.truncatedBy === "bytes") { const maxBytes = t.maxBytes ?? t.outputBytes; - notice += ` (${formatSize(maxBytes)} limit)`; + notice += ` (${formatBytes(maxBytes)} limit)`; } if (t.nextOffset != null) { diff --git a/packages/coding-agent/src/tools/output-utils.ts b/packages/coding-agent/src/tools/output-utils.ts deleted file mode 100644 index 5f4acc028..000000000 --- a/packages/coding-agent/src/tools/output-utils.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { ArtifactManager } from "../session/artifacts"; -import type { ToolSession } from "."; - -export interface TailBuffer { - append(chunk: string): void; - text(): string; - bytes(): number; -} - -export function createTailBuffer(maxBytes: number): TailBuffer { - let buffer = ""; - let bufferBytes = 0; - - const append = (text: string) => { - if (!text) return; - const chunkBytes = Buffer.byteLength(text, "utf-8"); - buffer += text; - bufferBytes += chunkBytes; - - if (bufferBytes > maxBytes) { - const buf = Buffer.from(buffer, "utf-8"); - let start = Math.max(0, buf.length - maxBytes); - while (start < buf.length && (buf[start] & 0xc0) === 0x80) { - start++; - } - buffer = buf.subarray(start).toString("utf-8"); - bufferBytes = Buffer.byteLength(buffer, "utf-8"); - } - }; - - return { - append, - text: () => buffer, - bytes: () => bufferBytes, - }; -} - -export function getArtifactManager(session: ToolSession): ArtifactManager | null { - if (session.artifactManager) { - return session.artifactManager; - } - const sessionFile = session.getSessionFile(); - if (!sessionFile) { - return null; - } - const manager = new ArtifactManager(sessionFile); - session.artifactManager = manager; - return manager; -} - -export async function allocateOutputArtifact( - session: ToolSession, - toolType: string, -): Promise<{ artifactPath?: string; artifactId?: string }> { - const manager = getArtifactManager(session); - if (!manager) return {}; - try { - const allocation = await manager.allocatePath(toolType); - return { artifactPath: allocation.path, artifactId: allocation.id }; - } catch { - return {}; - } -} diff --git a/packages/coding-agent/src/tools/python.ts b/packages/coding-agent/src/tools/python.ts index bc2fbb186..c6398dba1 100644 --- a/packages/coding-agent/src/tools/python.ts +++ b/packages/coding-agent/src/tools/python.ts @@ -13,16 +13,20 @@ import type { PreludeHelper, PythonStatusEvent } from "../ipy/kernel"; import { truncateToVisualLines } from "../modes/components/visual-truncate"; import type { Theme } from "../modes/theme/theme"; import pythonDescription from "../prompts/tools/python.md" with { type: "text" }; -import { OutputSink, type OutputSummary } from "../session/streaming-output"; +import { + allocateOutputArtifact, + DEFAULT_MAX_BYTES, + OutputSink, + type OutputSummary, + TailBuffer, +} from "../session/streaming-output"; import { getTreeBranch, getTreeContinuePrefix, renderCodeCell } from "../tui"; import type { ToolSession } from "."; import type { OutputMeta } from "./output-meta"; -import { allocateOutputArtifact, createTailBuffer } from "./output-utils"; import { resolveToCwd } from "./path-utils"; import { replaceTabs, shortenPath, ToolUIKit, truncateToWidth } from "./render-utils"; import { ToolAbortError, ToolError } from "./tool-errors"; import { toolResult } from "./tool-result"; -import { DEFAULT_MAX_BYTES } from "./truncate"; export const PYTHON_DEFAULT_PREVIEW_LINES = 10; @@ -207,7 +211,7 @@ export class PythonTool implements AgentTool { throw new ToolError(`Working directory is not a directory: ${commandCwd}`); } - const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES * 2); + const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES * 2); const jsonOutputs: unknown[] = []; const images: ImageContent[] = []; const statusEvents: PythonStatusEvent[] = []; @@ -255,7 +259,7 @@ export class PythonTool implements AgentTool { const sessionFile = this.session.getSessionFile?.() ?? undefined; const artifactsDir = this.session.getArtifactsDir?.() ?? undefined; - const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "python"); + const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "python"); outputSink = new OutputSink({ artifactPath, artifactId, diff --git a/packages/coding-agent/src/tools/read.ts b/packages/coding-agent/src/tools/read.ts index f8c4fd664..4f9ae7225 100644 --- a/packages/coding-agent/src/tools/read.ts +++ b/packages/coding-agent/src/tools/read.ts @@ -14,6 +14,13 @@ import { getLanguageFromPath, type Theme } from "../modes/theme/theme"; import { computeLineHash } from "../patch/hashline"; import readDescription from "../prompts/tools/read.md" with { type: "text" }; import type { ToolSession } from "../sdk"; +import { + DEFAULT_MAX_BYTES, + DEFAULT_MAX_LINES, + type TruncationResult, + truncateHead, + truncateHeadBytes, +} from "../session/streaming-output"; import { renderCodeCell, renderStatusLine } from "../tui"; import { CachedOutputBlock } from "../tui/output-block"; import { resolveFileDisplayMode } from "../utils/file-display-mode"; @@ -23,17 +30,9 @@ import { ensureTool } from "../utils/tools-manager"; import { applyListLimit } from "./list-limit"; import type { OutputMeta } from "./output-meta"; import { resolveReadPath, resolveToCwd } from "./path-utils"; -import { formatAge, shortenPath, wrapBrackets } from "./render-utils"; +import { formatAge, formatBytes, shortenPath, wrapBrackets } from "./render-utils"; import { ToolAbortError, ToolError, throwIfAborted } from "./tool-errors"; import { toolResult } from "./tool-result"; -import { - DEFAULT_MAX_BYTES, - DEFAULT_MAX_LINES, - formatSize, - type TruncationResult, - truncateHead, - truncateStringToBytesFromStart, -} from "./truncate"; // Document types convertible via markitdown const CONVERTIBLE_EXTENSIONS = new Set([".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".rtf", ".epub"]); @@ -211,17 +210,8 @@ async function streamLinesFromFile( let firstLinePreview: { text: string; bytes: number } | undefined; if (firstLinePreviewBytes > 0) { - const buf = Buffer.concat(firstLinePreviewChunks, firstLinePreviewBytes); - let end = Math.min(buf.length, maxBytes); - while (end > 0 && (buf[end] & 0xc0) === 0x80) { - end--; - } - if (end > 0) { - const text = buf.slice(0, end).toString("utf-8"); - firstLinePreview = { text, bytes: Buffer.byteLength(text, "utf-8") }; - } else { - firstLinePreview = { text: "", bytes: 0 }; - } + const { text, bytes } = truncateHeadBytes(Buffer.concat(firstLinePreviewChunks, firstLinePreviewBytes), maxBytes); + firstLinePreview = { text, bytes }; } return { @@ -623,8 +613,8 @@ export class ReadTool implements AgentTool { if (mimeType) { if (fileSize > MAX_IMAGE_SIZE) { - const sizeStr = formatSize(fileSize); - const maxStr = formatSize(MAX_IMAGE_SIZE); + const sizeStr = formatBytes(fileSize); + const maxStr = formatBytes(MAX_IMAGE_SIZE); throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`); } else { // Read as image (binary) @@ -633,8 +623,8 @@ export class ReadTool implements AgentTool { // Check actual buffer size after reading to prevent OOM during serialization if (buffer.byteLength > MAX_IMAGE_SIZE) { - const sizeStr = formatSize(buffer.byteLength); - const maxStr = formatSize(MAX_IMAGE_SIZE); + const sizeStr = formatBytes(buffer.byteLength); + const maxStr = formatBytes(MAX_IMAGE_SIZE); throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`); } else { const base64 = new Uint8Array(buffer).toBase64(); @@ -788,16 +778,16 @@ export class ReadTool implements AgentTool { const snippet = firstLinePreview ?? { text: "", bytes: 0 }; if (shouldAddHashLines) { - outputText = `[Line ${startLineDisplay} is ${formatSize( + outputText = `[Line ${startLineDisplay} is ${formatBytes( firstLineBytes, - )}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`; + )}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`; } else { outputText = formatText(snippet.text, startLineDisplay); } if (snippet.text.length === 0) { - outputText = `[Line ${startLineDisplay} is ${formatSize( + outputText = `[Line ${startLineDisplay} is ${formatBytes( firstLineBytes, - )}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`; + )}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`; } details = { truncation }; sourcePath = absolutePath; @@ -946,19 +936,19 @@ export class ReadTool implements AgentTool { if (truncation.firstLineExceedsLimit) { const firstLine = allLines[startLine] ?? ""; const firstLineBytes = Buffer.byteLength(firstLine, "utf-8"); - const snippet = truncateStringToBytesFromStart(firstLine, DEFAULT_MAX_BYTES); + const snippet = truncateHeadBytes(firstLine, DEFAULT_MAX_BYTES); if (shouldAddHashLines) { - outputText = `[Line ${startLineDisplay} is ${formatSize( + outputText = `[Line ${startLineDisplay} is ${formatBytes( firstLineBytes, - )}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`; + )}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`; } else { outputText = formatText(snippet.text, startLineDisplay); } if (snippet.text.length === 0) { - outputText = `[Line ${startLineDisplay} is ${formatSize( + outputText = `[Line ${startLineDisplay} is ${formatBytes( firstLineBytes, - )}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`; + )}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`; } details = { truncation }; truncationInfo = { @@ -1116,12 +1106,12 @@ export const readToolRenderer = { if (truncation) { let warning: string; if (fallback?.firstLineExceedsLimit) { - warning = `First line exceeds ${formatSize(fallback.maxBytes ?? DEFAULT_MAX_BYTES)} limit`; + warning = `First line exceeds ${formatBytes(fallback.maxBytes ?? DEFAULT_MAX_BYTES)} limit`; } else if (truncation.truncatedBy === "lines") { warning = `Truncated: ${truncation.outputLines} of ${truncation.totalLines} lines (${DEFAULT_MAX_LINES} line limit)`; } else { const maxBytes = fallback?.maxBytes ?? DEFAULT_MAX_BYTES; - warning = `Truncated: ${truncation.outputLines} lines (${formatSize(maxBytes)} limit)`; + warning = `Truncated: ${truncation.outputLines} lines (${formatBytes(maxBytes)} limit)`; } if (truncation.artifactId) { warning += `. Full output: artifact://${truncation.artifactId}`; diff --git a/packages/coding-agent/src/tools/ssh.ts b/packages/coding-agent/src/tools/ssh.ts index c37c2a1d9..07bbe081d 100644 --- a/packages/coding-agent/src/tools/ssh.ts +++ b/packages/coding-agent/src/tools/ssh.ts @@ -9,6 +9,7 @@ import { loadCapability } from "../discovery"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import type { Theme } from "../modes/theme/theme"; import sshDescriptionBase from "../prompts/tools/ssh.md" with { type: "text" }; +import { allocateOutputArtifact, DEFAULT_MAX_BYTES, TailBuffer } from "../session/streaming-output"; import type { SSHHostInfo } from "../ssh/connection-manager"; import { ensureHostInfo, getHostInfoForHost } from "../ssh/connection-manager"; import { executeSSH } from "../ssh/ssh-executor"; @@ -16,11 +17,9 @@ import { renderStatusLine } from "../tui"; import { CachedOutputBlock } from "../tui/output-block"; import type { ToolSession } from "."; import type { OutputMeta } from "./output-meta"; -import { allocateOutputArtifact, createTailBuffer } from "./output-utils"; import { formatBytes, wrapBrackets } from "./render-utils"; import { ToolError } from "./tool-errors"; import { toolResult } from "./tool-result"; -import { DEFAULT_MAX_BYTES } from "./truncate"; const sshSchema = Type.Object({ host: Type.String({ description: "Host name from managed SSH config or discovered ssh.json files" }), @@ -159,8 +158,8 @@ export class SshTool implements AgentTool { const timeoutSec = Math.max(1, Math.min(3600, rawTimeout)); const timeoutMs = timeoutSec * 1000; - const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES); - const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "ssh"); + const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES); + const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "ssh"); const result = await executeSSH(hostConfig, remoteCommand, { timeout: timeoutMs, diff --git a/packages/coding-agent/src/tools/tool-result.ts b/packages/coding-agent/src/tools/tool-result.ts index f7a065d78..a83022826 100644 --- a/packages/coding-agent/src/tools/tool-result.ts +++ b/packages/coding-agent/src/tools/tool-result.ts @@ -1,9 +1,8 @@ import type { AgentToolResult } from "@oh-my-pi/pi-agent-core"; import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai"; -import type { OutputSummary } from "../session/streaming-output"; +import type { OutputSummary, TruncationResult } from "../session/streaming-output"; import type { OutputMeta, TruncationOptions, TruncationSummaryOptions, TruncationTextOptions } from "./output-meta"; import { outputMeta } from "./output-meta"; -import type { TruncationResult } from "./truncate"; type ToolContent = Array; diff --git a/packages/coding-agent/src/tools/truncate.ts b/packages/coding-agent/src/tools/truncate.ts deleted file mode 100644 index b31ea0716..000000000 --- a/packages/coding-agent/src/tools/truncate.ts +++ /dev/null @@ -1,385 +0,0 @@ -/** - * Shared truncation utilities for tool outputs. - * - * Truncation is based on two independent limits - whichever is hit first wins: - * - Line limit (default: 4000 lines) - * - Byte limit (default: 50KB) - * - * Never returns partial lines (except bash tail truncation edge case - * and the read tool's long-line snippet fallback). - */ - -export const DEFAULT_MAX_LINES = 3000; -export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB -export const DEFAULT_MAX_COLUMN = 1024; // Max chars per grep match line - -export interface TruncationResult { - /** The truncated content */ - content: string; - /** Whether truncation occurred */ - truncated: boolean; - /** Which limit was hit: "lines", "bytes", or null if not truncated */ - truncatedBy: "lines" | "bytes" | null; - /** Total number of lines in the original content */ - totalLines: number; - /** Total number of bytes in the original content */ - totalBytes: number; - /** Number of complete lines in the truncated output */ - outputLines: number; - /** Number of bytes in the truncated output */ - outputBytes: number; - /** Whether the last line was partially truncated (only for tail truncation edge case) */ - lastLinePartial: boolean; - /** Whether the first line exceeded the byte limit (for head truncation) */ - firstLineExceedsLimit: boolean; - /** The max lines limit that was applied */ - maxLines: number; - /** The max bytes limit that was applied */ - maxBytes: number; -} - -export interface TruncationOptions { - /** Maximum number of lines (default: 2000) */ - maxLines?: number; - /** Maximum number of bytes (default: 50KB) */ - maxBytes?: number; -} - -/** - * Format bytes as human-readable size. - */ -export function formatSize(bytes: number): string { - if (bytes < 1024) { - return `${bytes}B`; - } else if (bytes < 1024 * 1024) { - return `${(bytes / 1024).toFixed(1)}KB`; - } else if (bytes < 1024 * 1024 * 1024) { - return `${(bytes / (1024 * 1024)).toFixed(1)}MB`; - } else { - return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`; - } -} - -/** - * Truncate content from the head (keep first N lines/bytes). - * Suitable for file reads where you want to see the beginning. - * - * Never returns partial lines. If first line exceeds byte limit, - * returns empty content with firstLineExceedsLimit=true. - */ -export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult { - const maxLines = options.maxLines ?? DEFAULT_MAX_LINES; - const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; - - const totalBytes = Buffer.byteLength(content, "utf-8"); - const lines = content.split("\n"); - const totalLines = lines.length; - - // Check if no truncation needed - if (totalLines <= maxLines && totalBytes <= maxBytes) { - return { - content, - truncated: false, - truncatedBy: null, - totalLines, - totalBytes, - outputLines: totalLines, - outputBytes: totalBytes, - lastLinePartial: false, - firstLineExceedsLimit: false, - maxLines, - maxBytes, - }; - } - - // Check if first line alone exceeds byte limit - const firstLineBytes = Buffer.byteLength(lines[0], "utf-8"); - if (firstLineBytes > maxBytes) { - return { - content: "", - truncated: true, - truncatedBy: "bytes", - totalLines, - totalBytes, - outputLines: 0, - outputBytes: 0, - lastLinePartial: false, - firstLineExceedsLimit: true, - maxLines, - maxBytes, - }; - } - - // Collect complete lines that fit - const outputLinesArr: string[] = []; - let outputBytesCount = 0; - let truncatedBy: "lines" | "bytes" = "lines"; - - for (let i = 0; i < lines.length && i < maxLines; i++) { - const line = lines[i]; - const lineBytes = Buffer.byteLength(line, "utf-8") + (i > 0 ? 1 : 0); // +1 for newline - - if (outputBytesCount + lineBytes > maxBytes) { - truncatedBy = "bytes"; - break; - } - - outputLinesArr.push(line); - outputBytesCount += lineBytes; - } - - // If we exited due to line limit - if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) { - truncatedBy = "lines"; - } - - const outputContent = outputLinesArr.join("\n"); - const finalOutputBytes = Buffer.byteLength(outputContent, "utf-8"); - - return { - content: outputContent, - truncated: true, - truncatedBy, - totalLines, - totalBytes, - outputLines: outputLinesArr.length, - outputBytes: finalOutputBytes, - lastLinePartial: false, - firstLineExceedsLimit: false, - maxLines, - maxBytes, - }; -} - -/** - * Truncate content from the tail (keep last N lines/bytes). - * Suitable for bash output where you want to see the end (errors, final results). - * - * May return partial first line if the last line of original content exceeds byte limit. - */ -export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult { - const maxLines = options.maxLines ?? DEFAULT_MAX_LINES; - const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; - - const totalBytes = Buffer.byteLength(content, "utf-8"); - const lines = content.split("\n"); - const totalLines = lines.length; - - // Check if no truncation needed - if (totalLines <= maxLines && totalBytes <= maxBytes) { - return { - content, - truncated: false, - truncatedBy: null, - totalLines, - totalBytes, - outputLines: totalLines, - outputBytes: totalBytes, - lastLinePartial: false, - firstLineExceedsLimit: false, - maxLines, - maxBytes, - }; - } - - // Work backwards from the end - const outputLinesArr: string[] = []; - let outputBytesCount = 0; - let truncatedBy: "lines" | "bytes" = "lines"; - let lastLinePartial = false; - - for (let i = lines.length - 1; i >= 0 && outputLinesArr.length < maxLines; i--) { - const line = lines[i]; - const lineBytes = Buffer.byteLength(line, "utf-8") + (outputLinesArr.length > 0 ? 1 : 0); // +1 for newline - - if (outputBytesCount + lineBytes > maxBytes) { - truncatedBy = "bytes"; - // Edge case: if we haven't added ANY lines yet and this line exceeds maxBytes, - // take the end of the line (partial) - if (outputLinesArr.length === 0) { - const truncatedLine = truncateStringToBytesFromEnd(line, maxBytes); - outputLinesArr.unshift(truncatedLine); - outputBytesCount = Buffer.byteLength(truncatedLine, "utf-8"); - lastLinePartial = true; - } - break; - } - - outputLinesArr.unshift(line); - outputBytesCount += lineBytes; - } - - // If we exited due to line limit - if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) { - truncatedBy = "lines"; - } - - const outputContent = outputLinesArr.join("\n"); - const finalOutputBytes = Buffer.byteLength(outputContent, "utf-8"); - - return { - content: outputContent, - truncated: true, - truncatedBy, - totalLines, - totalBytes, - outputLines: outputLinesArr.length, - outputBytes: finalOutputBytes, - lastLinePartial, - firstLineExceedsLimit: false, - maxLines, - maxBytes, - }; -} - -/** - * Truncate a string to fit within a byte limit (from the end). - * Handles multi-byte UTF-8 characters correctly. - */ -function truncateStringToBytesFromEnd(str: string, maxBytes: number): string { - const buf = Buffer.from(str, "utf-8"); - if (buf.length <= maxBytes) { - return str; - } - - // Start from the end, skip maxBytes back - let start = buf.length - maxBytes; - - // Find a valid UTF-8 boundary (start of a character) - while (start < buf.length && (buf[start] & 0xc0) === 0x80) { - start++; - } - - return buf.slice(start).toString("utf-8"); -} - -/** - * Truncate a string to fit within a byte limit (from the start). - * Handles multi-byte UTF-8 characters correctly. - */ -export function truncateStringToBytesFromStart(str: string, maxBytes: number): { text: string; bytes: number } { - const buf = Buffer.from(str, "utf-8"); - if (buf.length <= maxBytes) { - return { text: str, bytes: buf.length }; - } - - let end = maxBytes; - - // Find a valid UTF-8 boundary (start of a character) - while (end > 0 && (buf[end] & 0xc0) === 0x80) { - end--; - } - - if (end <= 0) { - return { text: "", bytes: 0 }; - } - - const text = buf.slice(0, end).toString("utf-8"); - return { text, bytes: Buffer.byteLength(text, "utf-8") }; -} - -/** - * Truncate a single line to max characters, adding [truncated] suffix. - * Used for grep match lines. - */ -export function truncateLine( - line: string, - maxChars: number = DEFAULT_MAX_COLUMN, -): { text: string; wasTruncated: boolean } { - if (line.length <= maxChars) { - return { text: line, wasTruncated: false }; - } - return { text: `${line.slice(0, maxChars)}…`, wasTruncated: true }; -} - -// ============================================================================= -// Truncation notice formatting -// ============================================================================= - -export interface TailTruncationNoticeOptions { - /** Path to full output file (e.g., from bash/python executor) */ - fullOutputPath?: string; - /** Original content for computing last line size when lastLinePartial */ - originalContent?: string; - /** Additional suffix to append inside the brackets */ - suffix?: string; -} - -/** - * Format a truncation notice for tail-truncated output (bash, python, ssh). - * Returns empty string if not truncated. - * - * Examples: - * - "[Showing last 50KB of line 1000 (line is 2.1MB). Full output: /tmp/out.txt]" - * - "[Showing lines 500-1000 of 1000. Full output: /tmp/out.txt]" - * - "[Showing lines 500-1000 of 1000 (50KB limit). Full output: /tmp/out.txt]" - */ -export function formatTailTruncationNotice( - truncation: TruncationResult, - options: TailTruncationNoticeOptions = {}, -): string { - if (!truncation.truncated) { - return ""; - } - - const { fullOutputPath, originalContent, suffix = "" } = options; - const startLine = truncation.totalLines - truncation.outputLines + 1; - const endLine = truncation.totalLines; - const fullOutputPart = fullOutputPath ? `. Full output: ${fullOutputPath}` : ""; - - let notice: string; - - if (truncation.lastLinePartial) { - let lastLineSizePart = ""; - if (originalContent) { - const lastLine = originalContent.split("\n").pop() || ""; - lastLineSizePart = ` (line is ${formatSize(Buffer.byteLength(lastLine, "utf-8"))})`; - } - notice = `[Showing last ${formatSize(truncation.outputBytes)} of line ${endLine}${lastLineSizePart}${fullOutputPart}${suffix}]`; - } else if (truncation.truncatedBy === "lines") { - notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}${fullOutputPart}${suffix}]`; - } else { - notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatSize(truncation.maxBytes)} limit)${fullOutputPart}${suffix}]`; - } - - return `\n\n${notice}`; -} - -export interface HeadTruncationNoticeOptions { - /** 1-indexed start line number (default: 1) */ - startLine?: number; - /** Total lines in the original file (for "of N" display) */ - totalFileLines?: number; -} - -/** - * Format a truncation notice for head-truncated output (read tool). - * Returns empty string if not truncated. - * - * Examples: - * - "[Showing lines 1-2000 of 5000. Use offset=2001 to continue]" - * - "[Showing lines 100-2099 of 5000 (50KB limit). Use offset=2100 to continue]" - */ -export function formatHeadTruncationNotice( - truncation: TruncationResult, - options: HeadTruncationNoticeOptions = {}, -): string { - if (!truncation.truncated) { - return ""; - } - - const startLineDisplay = options.startLine ?? 1; - const totalFileLines = options.totalFileLines ?? truncation.totalLines; - const endLineDisplay = startLineDisplay + truncation.outputLines - 1; - const nextOffset = endLineDisplay + 1; - - let notice: string; - - if (truncation.truncatedBy === "lines") { - notice = `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`; - } else { - notice = `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines} (${formatSize(truncation.maxBytes)} limit). Use offset=${nextOffset} to continue]`; - } - - return `\n\n${notice}`; -} diff --git a/packages/coding-agent/src/utils/file-mentions.ts b/packages/coding-agent/src/utils/file-mentions.ts index d93598e23..d9d46030c 100644 --- a/packages/coding-agent/src/utils/file-mentions.ts +++ b/packages/coding-agent/src/utils/file-mentions.ts @@ -11,9 +11,9 @@ import type { AgentMessage } from "@oh-my-pi/pi-agent-core"; import { glob } from "@oh-my-pi/pi-natives"; import { formatHashLines } from "../patch/hashline"; import type { FileMentionMessage } from "../session/messages"; +import { DEFAULT_MAX_BYTES, truncateHead, truncateHeadBytes } from "../session/streaming-output"; import { resolveReadPath } from "../tools/path-utils"; -import { formatAge } from "../tools/render-utils"; -import { DEFAULT_MAX_BYTES, formatSize, truncateHead, truncateStringToBytesFromStart } from "../tools/truncate"; +import { formatAge, formatBytes } from "../tools/render-utils"; import { fuzzyMatch } from "./fuzzy"; import { formatDimensionNote, resizeImage } from "./image-resize"; import { detectSupportedImageMimeTypeFromFile } from "./mime"; @@ -162,15 +162,15 @@ function buildTextOutput(textContent: string): { output: string; lineCount: numb if (truncation.firstLineExceedsLimit) { const firstLine = allLines[0] ?? ""; const firstLineBytes = Buffer.byteLength(firstLine, "utf-8"); - const snippet = truncateStringToBytesFromStart(firstLine, DEFAULT_MAX_BYTES); + const snippet = truncateHeadBytes(firstLine, DEFAULT_MAX_BYTES); let outputText = snippet.text; if (outputText.length > 0) { - outputText += `\n\n[Line 1 is ${formatSize(firstLineBytes)}, exceeds ${formatSize( + outputText += `\n\n[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes( DEFAULT_MAX_BYTES, - )} limit. Showing first ${formatSize(snippet.bytes)} of the line.]`; + )} limit. Showing first ${formatBytes(snippet.bytes)} of the line.]`; } else { - outputText = `[Line 1 is ${formatSize(firstLineBytes)}, exceeds ${formatSize( + outputText = `[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes( DEFAULT_MAX_BYTES, )} limit. Unable to display a valid UTF-8 snippet.]`; } @@ -187,7 +187,7 @@ function buildTextOutput(textContent: string): { output: string; lineCount: numb if (truncation.truncatedBy === "lines") { outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`; } else { - outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines} (${formatSize( + outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines} (${formatBytes( DEFAULT_MAX_BYTES, )} limit). Use offset=${nextOffset} to continue]`; } @@ -247,7 +247,7 @@ async function buildDirectoryListing(absolutePath: string): Promise<{ output: st notices.push(`${DEFAULT_DIR_LIMIT} entries limit reached. Use limit=${DEFAULT_DIR_LIMIT * 2} for more`); } if (truncation.truncated) { - notices.push(`${formatSize(DEFAULT_MAX_BYTES)} limit reached`); + notices.push(`${formatBytes(DEFAULT_MAX_BYTES)} limit reached`); } if (notices.length > 0) { output += `\n\n[${notices.join(". ")}]`; @@ -313,7 +313,7 @@ export async function generateFileMentionMessages( if (stat.size > MAX_AUTO_READ_IMAGE_BYTES) { files.push({ path: resolvedPath, - content: `(skipped auto-read: too large, ${formatSize(stat.size)})`, + content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`, byteSize: stat.size, skippedReason: "tooLarge", }); @@ -349,7 +349,7 @@ export async function generateFileMentionMessages( if (stat.size > MAX_AUTO_READ_TEXT_BYTES) { files.push({ path: resolvedPath, - content: `(skipped auto-read: too large, ${formatSize(stat.size)})`, + content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`, byteSize: stat.size, skippedReason: "tooLarge", }); diff --git a/packages/coding-agent/src/web/scrapers/dockerhub.ts b/packages/coding-agent/src/web/scrapers/dockerhub.ts index 774709d85..c91b24184 100644 --- a/packages/coding-agent/src/web/scrapers/dockerhub.ts +++ b/packages/coding-agent/src/web/scrapers/dockerhub.ts @@ -1,3 +1,4 @@ +import { formatBytes } from "../../tools/render-utils"; import type { RenderResult, SpecialHandler } from "./types"; import { finalizeOutput, formatCount, loadPage } from "./types"; @@ -29,13 +30,6 @@ interface DockerHubTagsResponse { results?: DockerHubTag[]; } -function formatSize(bytes: number): string { - if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`; - if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`; - if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`; - return `${bytes}B`; -} - /** * Handle Docker Hub URLs via API */ @@ -131,7 +125,7 @@ export const handleDockerHub: SpecialHandler = async ( md += "|-----|------|---------------|--------|\n"; for (const tag of tags) { - const size = tag.full_size ? formatSize(tag.full_size) : "-"; + const size = tag.full_size ? formatBytes(tag.full_size) : "-"; const archs = tag.images ?.map(img => img.architecture) diff --git a/packages/coding-agent/src/web/scrapers/ollama.ts b/packages/coding-agent/src/web/scrapers/ollama.ts index 64aa7585c..b4c9268ad 100644 --- a/packages/coding-agent/src/web/scrapers/ollama.ts +++ b/packages/coding-agent/src/web/scrapers/ollama.ts @@ -1,3 +1,4 @@ +import { formatBytes } from "../../tools/render-utils"; import type { RenderResult, SpecialHandler } from "./types"; import { finalizeOutput, loadPage } from "./types"; @@ -100,13 +101,6 @@ function extractTagsFromHtml(html: string, baseRef: string): string[] { return Array.from(tags); } -function formatSize(bytes: number): string { - if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`; - if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`; - if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`; - return `${bytes}B`; -} - function buildModelPath(parts: string[]): string { return parts.map(part => encodeURIComponent(part)).join("/"); } @@ -227,11 +221,11 @@ export const handleOllama: SpecialHandler = async ( let sizeLine: string | null = null; if (selectedTag?.size) { - sizeLine = formatSize(selectedTag.size); + sizeLine = formatBytes(selectedTag.size); } else if (sizes.length > 0) { const minSize = Math.min(...sizes); const maxSize = Math.max(...sizes); - sizeLine = minSize === maxSize ? formatSize(minSize) : `${formatSize(minSize)} - ${formatSize(maxSize)}`; + sizeLine = minSize === maxSize ? formatBytes(minSize) : `${formatBytes(minSize)} - ${formatBytes(maxSize)}`; } let md = `# ${baseRef}\n\n`; diff --git a/packages/coding-agent/src/web/scrapers/utils.ts b/packages/coding-agent/src/web/scrapers/utils.ts index d0d052f4f..e330f6769 100644 --- a/packages/coding-agent/src/web/scrapers/utils.ts +++ b/packages/coding-agent/src/web/scrapers/utils.ts @@ -13,7 +13,7 @@ export interface ConvertResult { } export interface BinaryFetchResult { - buffer: Buffer; + buffer: Uint8Array; contentType: string; contentDisposition?: string; ok: boolean; @@ -22,7 +22,7 @@ export interface BinaryFetchResult { } export async function convertWithMarkitdown( - content: Buffer, + content: Uint8Array, extensionHint: string, timeout: number, userSignal?: AbortSignal, @@ -68,7 +68,7 @@ export async function convertWithMarkitdown( } } -const kEmptyBuffer = Buffer.alloc(0); +const kEmptyBuffer = new Uint8Array(0); export async function fetchBinary(url: string, timeout: number, userSignal?: AbortSignal): Promise { if (userSignal?.aborted) { @@ -115,7 +115,7 @@ export async function fetchBinary(url: string, timeout: number, userSignal?: Abo } } - const buffer = Buffer.from(await response.arrayBuffer()); + const buffer = await response.bytes(); if (buffer.length > MAX_BYTES) { return { buffer: kEmptyBuffer, diff --git a/packages/coding-agent/test/bash-executor.test.ts b/packages/coding-agent/test/bash-executor.test.ts index b7f618f44..71f3db0ce 100644 --- a/packages/coding-agent/test/bash-executor.test.ts +++ b/packages/coding-agent/test/bash-executor.test.ts @@ -4,7 +4,7 @@ import * as os from "node:os"; import * as path from "node:path"; import { _resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { executeBash } from "@oh-my-pi/pi-coding-agent/exec/bash-executor"; -import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate"; +import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output"; import * as shellSnapshot from "@oh-my-pi/pi-coding-agent/utils/shell-snapshot"; function makeTempDir(): string { diff --git a/packages/coding-agent/test/core/python-executor-streaming.test.ts b/packages/coding-agent/test/core/python-executor-streaming.test.ts index 069662ee3..1b8d8f1df 100644 --- a/packages/coding-agent/test/core/python-executor-streaming.test.ts +++ b/packages/coding-agent/test/core/python-executor-streaming.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "bun:test"; import { executePythonWithKernel, type PythonKernelExecutor } from "@oh-my-pi/pi-coding-agent/ipy/executor"; import type { KernelExecuteOptions, KernelExecuteResult } from "@oh-my-pi/pi-coding-agent/ipy/kernel"; -import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate"; +import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output"; class FakeKernel implements PythonKernelExecutor { private result: KernelExecuteResult; diff --git a/packages/coding-agent/test/core/python-executor.test.ts b/packages/coding-agent/test/core/python-executor.test.ts index 235ccda0e..fc1dd76c2 100644 --- a/packages/coding-agent/test/core/python-executor.test.ts +++ b/packages/coding-agent/test/core/python-executor.test.ts @@ -14,7 +14,7 @@ import { type PreludeHelper, PythonKernel, } from "@oh-my-pi/pi-coding-agent/ipy/kernel"; -import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate"; +import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output"; import { TempDir } from "@oh-my-pi/pi-utils"; class FakeKernel implements PythonKernelExecutor { diff --git a/packages/coding-agent/test/model-registry-runtime-provider.test.ts b/packages/coding-agent/test/model-registry-runtime-provider.test.ts index 1191e88c2..29132100a 100644 --- a/packages/coding-agent/test/model-registry-runtime-provider.test.ts +++ b/packages/coding-agent/test/model-registry-runtime-provider.test.ts @@ -51,6 +51,15 @@ describe("ModelRegistry runtime provider registration", () => { const streamSimple: NonNullable = () => ({}) as unknown as AssistantMessageEventStream; + test("loads built-in GitLab Duo models and OAuth provider metadata", () => { + const registry = new ModelRegistry(authStorage, modelsJsonPath); + const model = registry.find("gitlab-duo", "claude-sonnet-4-5-20250929"); + + expect(model).toBeDefined(); + expect(model?.api).toBe("anthropic-messages"); + expect(getOAuthProviders().some(provider => provider.id === "gitlab-duo")).toBe(true); + }); + test("validates provider config before mutating custom API state", () => { const registry = new ModelRegistry(authStorage, modelsJsonPath); const beforeAnthropicCount = registry.getAll().filter(model => model.provider === "anthropic").length; diff --git a/packages/coding-agent/test/streaming-output.test.ts b/packages/coding-agent/test/streaming-output.test.ts new file mode 100644 index 000000000..05133c080 --- /dev/null +++ b/packages/coding-agent/test/streaming-output.test.ts @@ -0,0 +1,353 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { ArtifactManager } from "../src/session/artifacts"; +import { + allocateOutputArtifact, + DEFAULT_MAX_BYTES, + DEFAULT_MAX_COLUMN, + DEFAULT_MAX_LINES, + formatHeadTruncationNotice, + formatTailTruncationNotice, + OutputSink, + TailBuffer, + truncateHead, + truncateHeadBytes, + truncateLine, + truncateTail, + truncateTailBytes, +} from "../src/session/streaming-output"; +import type { ToolSession } from "../src/tools"; + +const createdTempDirs: string[] = []; + +async function createTempDir(): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "streaming-output-test-")); + createdTempDirs.push(dir); + return dir; +} + +function byteLength(text: string): number { + return Buffer.byteLength(text, "utf-8"); +} + +function createSession(overrides: Partial = {}): ToolSession { + return { + cwd: "/", + hasUI: false, + getSessionFile: () => null, + getSessionSpawns: () => null, + settings: {} as ToolSession["settings"], + ...overrides, + }; +} + +afterEach(async () => { + for (const dir of createdTempDirs.splice(0)) { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +describe("streaming-output exports", () => { + test("exports expected default limits", () => { + expect(DEFAULT_MAX_LINES).toBe(3000); + expect(DEFAULT_MAX_BYTES).toBe(50 * 1024); + expect(DEFAULT_MAX_COLUMN).toBe(1024); + }); +}); + +describe("truncateTailBytes", () => { + test("returns source when already under limit", () => { + const text = "hello"; + expect(truncateTailBytes(text, 10)).toEqual({ text: "hello", bytes: 5 }); + }); + + test("truncates from end without breaking UTF-8 boundaries", () => { + const text = "a😀b"; + const result = truncateTailBytes(text, 4); + expect(result).toEqual({ text: "b", bytes: 1 }); + expect(result.text).not.toContain("\uFFFD"); + }); + + test("accepts Uint8Array input", () => { + const bytes = new TextEncoder().encode("abc😀"); + const result = truncateTailBytes(bytes, 4); + expect(result.text).toBe("😀"); + expect(result.bytes).toBe(4); + }); +}); + +describe("truncateHeadBytes", () => { + test("returns source when already under limit", () => { + const text = "hello"; + expect(truncateHeadBytes(text, 10)).toEqual({ text: "hello", bytes: 5 }); + }); + + test("truncates from start without breaking UTF-8 boundaries", () => { + const text = "a😀b"; + const result = truncateHeadBytes(text, 2); + expect(result).toEqual({ text: "a", bytes: 1 }); + expect(result.text).not.toContain("\uFFFD"); + }); + + test("returns empty when maxBytes is zero", () => { + const result = truncateHeadBytes("abc", 0); + expect(result).toEqual({ text: "", bytes: 0 }); + }); +}); + +describe("truncateHead", () => { + test("returns unmodified content when within limits", () => { + const content = "a\nb"; + const result = truncateHead(content, { maxLines: 10, maxBytes: 20 }); + expect(result.truncated).toBe(false); + expect(result.content).toBe(content); + expect(result.truncatedBy).toBe(null); + }); + + test("handles first line exceeding byte limit", () => { + const result = truncateHead("abcdef\nnext", { maxBytes: 3, maxLines: 10 }); + expect(result.content).toBe(""); + expect(result.truncated).toBe(true); + expect(result.truncatedBy).toBe("bytes"); + expect(result.firstLineExceedsLimit).toBe(true); + }); + + + test("includes first line when text fits exact byte budget", () => { + const result = truncateHead("abc\nx", { maxBytes: 3, maxLines: 10 }); + expect(result.content).toBe("abc"); + expect(result.truncated).toBe(true); + expect(result.truncatedBy).toBe("bytes"); + expect(result.firstLineExceedsLimit).toBe(false); + expect(result.outputBytes).toBe(byteLength("abc")); + }); + test("truncates by line count", () => { + const result = truncateHead("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 }); + expect(result.content).toBe("l1\nl2"); + expect(result.truncatedBy).toBe("lines"); + expect(result.outputLines).toBe(2); + }); + + test("truncates by byte budget using complete lines", () => { + const result = truncateHead("12345\nabc\nz", { maxLines: 10, maxBytes: 7 }); + expect(result.content).toBe("12345"); + expect(result.truncatedBy).toBe("bytes"); + expect(result.lastLinePartial).toBe(false); + expect(result.outputBytes).toBe(byteLength("12345")); + }); +}); + +describe("truncateTail", () => { + test("returns unmodified content when within limits", () => { + const content = "a\nb"; + const result = truncateTail(content, { maxLines: 10, maxBytes: 20 }); + expect(result.truncated).toBe(false); + expect(result.content).toBe(content); + expect(result.truncatedBy).toBe(null); + }); + + test("truncates by line count", () => { + const result = truncateTail("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 }); + expect(result.content).toBe("l2\nl3"); + expect(result.truncatedBy).toBe("lines"); + expect(result.outputLines).toBe(2); + }); + + test("truncates by byte budget while preserving line boundaries", () => { + const result = truncateTail("aaa\nbbbb\ncc", { maxLines: 10, maxBytes: 6 }); + expect(result.content).toBe("cc"); + expect(result.truncatedBy).toBe("bytes"); + expect(result.lastLinePartial).toBe(false); + }); + + test("returns partial single line when last line exceeds byte limit", () => { + const result = truncateTail("abcdefghij", { maxLines: 10, maxBytes: 4 }); + expect(result.content).toBe("ghij"); + expect(result.truncatedBy).toBe("bytes"); + expect(result.lastLinePartial).toBe(true); + }); +}); + +describe("truncateLine", () => { + test("does not truncate short lines", () => { + expect(truncateLine("hello", 10)).toEqual({ text: "hello", wasTruncated: false }); + }); + + test("truncates long lines with ellipsis", () => { + expect(truncateLine("abcdefgh", 5)).toEqual({ text: "abcde…", wasTruncated: true }); + }); +}); + +describe("TailBuffer", () => { + test("keeps trailing bytes under budget", () => { + const tail = new TailBuffer(5); + tail.append("abc"); + tail.append("def"); + expect(tail.text()).toBe("bcdef"); + expect(tail.bytes()).toBe(5); + }); + + test("handles multibyte data and empty appends", () => { + const tail = new TailBuffer(4); + tail.append(""); + tail.append("😀"); + tail.append("x"); + expect(tail.text()).toBe("x"); + expect(tail.bytes()).toBe(1); + }); +}); + +describe("OutputSink", () => { + test("tracks totals and adds notice in dump", async () => { + const sink = new OutputSink(); + await sink.push("hello\nworld"); + const dumped = await sink.dump("notice"); + + expect(dumped.output).toBe("[notice]\nhello\nworld"); + expect(dumped.truncated).toBe(false); + expect(dumped.totalLines).toBe(2); + expect(dumped.totalBytes).toBe(byteLength("hello\nworld")); + expect(dumped.outputLines).toBe(2); + expect(dumped.outputBytes).toBe(byteLength("hello\nworld")); + }); + + + test("counts lines correctly when chunks contain no newlines", async () => { + const sink = new OutputSink(); + await sink.push("abc"); + await sink.push("def"); + const dumped = await sink.dump(); + + expect(dumped.totalLines).toBe(1); + expect(dumped.outputLines).toBe(1); + }); + + test("counts all newline boundaries across chunk splits", async () => { + const sink = new OutputSink(); + await sink.push("a\n"); + await sink.push("b\n\n"); + await sink.push("c"); + const dumped = await sink.dump(); + + expect(dumped.output).toBe("a\nb\n\nc"); + expect(dumped.totalLines).toBe(4); + expect(dumped.outputLines).toBe(4); + }); + test("invokes onChunk callback with sanitized text", async () => { + const chunks: string[] = []; + const sink = new OutputSink({ onChunk: chunk => chunks.push(chunk) }); + await sink.push("abc"); + await sink.push("def"); + expect(chunks).toEqual(["abc", "def"]); + }); + + test("truncates in-memory output when spill threshold is exceeded", async () => { + const sink = new OutputSink({ spillThreshold: 5 }); + await sink.push("abc"); + await sink.push("def"); + + const dumped = await sink.dump(); + expect(dumped.truncated).toBe(true); + expect(dumped.output).toBe("bcdef"); + expect(dumped.totalBytes).toBe(6); + expect(dumped.outputBytes).toBe(5); + }); + + test("spills full output to artifact file when artifact path is provided", async () => { + const dir = await createTempDir(); + const artifactPath = path.join(dir, "output.log"); + const sink = new OutputSink({ + artifactPath, + artifactId: "artifact-1", + spillThreshold: 5, + }); + + await sink.push("abc"); + await sink.push("def"); + const dumped = await sink.dump(); + const artifactText = await Bun.file(artifactPath).text(); + + expect(dumped.truncated).toBe(true); + expect(dumped.artifactId).toBe("artifact-1"); + expect(artifactText).toBe("abcdef"); + expect(dumped.output).toBe("bcdef"); + }); + + test("createInput decodes streamed UTF-8 chunks correctly", async () => { + const sink = new OutputSink(); + const writer = sink.createInput().getWriter(); + const bytes = new TextEncoder().encode("😀X"); + + await writer.write(bytes.subarray(0, 2)); + await writer.write(bytes.subarray(2)); + await writer.close(); + + const dumped = await sink.dump(); + expect(dumped.output).toBe("😀X"); + expect(dumped.totalBytes).toBe(byteLength("😀X")); + }); +}); + +describe("allocateOutputArtifact", () => { + test("returns empty object when session has no artifact manager", async () => { + const session = createSession({ getArtifactManager: undefined }); + expect(await allocateOutputArtifact(session, "bash")).toEqual({}); + }); + + test("allocates artifact path via session artifact manager", async () => { + const dir = await createTempDir(); + const sessionFile = path.join(dir, "session.jsonl"); + await Bun.write(sessionFile, ""); + const manager = new ArtifactManager(sessionFile); + const session = createSession({ getArtifactManager: () => manager }); + + const result = await allocateOutputArtifact(session, "bash"); + expect(result.id).toBeDefined(); + expect(result.path).toContain(`${path.sep}${result.id}.bash.log`); + }); +}); + +describe("truncation notice formatting", () => { + test("formatTailTruncationNotice returns empty string for non-truncated results", () => { + const truncation = truncateTail("a\nb", { maxLines: 10, maxBytes: 50 }); + expect(formatTailTruncationNotice(truncation)).toBe(""); + }); + + test("formatTailTruncationNotice supports partial-line notices", () => { + const truncation = truncateTail("abcdefghij", { maxLines: 10, maxBytes: 4 }); + const notice = formatTailTruncationNotice(truncation, { + fullOutputPath: "/tmp/full.log", + originalContent: "abcdefghij", + suffix: " [suffix]", + }); + expect(notice).toBe("\n\n[Showing last 4B of line 1 (line is 10B). Full output: /tmp/full.log [suffix]]"); + }); + + test("formatTailTruncationNotice supports line-based and byte-based notices", () => { + const lineTruncation = truncateTail("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 }); + expect(formatTailTruncationNotice(lineTruncation)).toBe("\n\n[Showing lines 2-3 of 3]"); + + const byteTruncation = truncateTail("aaa\nbbbb\ncc", { maxLines: 10, maxBytes: 6 }); + expect(formatTailTruncationNotice(byteTruncation)).toBe("\n\n[Showing lines 3-3 of 3 (6B limit)]"); + }); + + test("formatHeadTruncationNotice returns empty string for non-truncated results", () => { + const truncation = truncateHead("a\nb", { maxLines: 10, maxBytes: 50 }); + expect(formatHeadTruncationNotice(truncation)).toBe(""); + }); + + test("formatHeadTruncationNotice supports line and byte truncation", () => { + const lineTruncation = truncateHead("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 }); + expect(formatHeadTruncationNotice(lineTruncation)).toBe("\n\n[Showing lines 1-2 of 3. Use offset=3 to continue]"); + + const byteTruncation = truncateHead("12345\nabc\nz", { maxLines: 10, maxBytes: 7 }); + expect( + formatHeadTruncationNotice(byteTruncation, { + startLine: 100, + totalFileLines: 500, + }), + ).toBe("\n\n[Showing lines 100-100 of 500 (7B limit). Use offset=101 to continue]"); + }); +}); diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index 050b003d9..8da1d3022 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -4,6 +4,7 @@ import * as os from "node:os"; import * as path from "node:path"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { EditTool } from "@oh-my-pi/pi-coding-agent/patch"; +import { ArtifactManager } from "@oh-my-pi/pi-coding-agent/session/artifacts"; import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools"; import { BashTool } from "@oh-my-pi/pi-coding-agent/tools/bash"; import { FindTool } from "@oh-my-pi/pi-coding-agent/tools/find"; @@ -25,11 +26,13 @@ function getTextOutput(result: any): string { function createTestToolSession(cwd: string): ToolSession { const sessionFile = path.join(cwd, "session.jsonl"); + const artifactManager = new ArtifactManager(sessionFile); return { cwd, hasUI: false, getSessionFile: () => sessionFile, getSessionSpawns: () => "*", + getArtifactManager: () => artifactManager, settings: Settings.isolated(), }; }