refactor(coding-agent): consolidated output utilities into streaming-output module

- Consolidated truncation and output utilities from tools/truncate.ts and tools/output-utils.ts into session/streaming-output.ts with improved UTF-8 boundary handling.
- Renamed formatSize() to formatBytes() across codebase for consistency and clarity in byte-level formatting.
- Refactored OutputSink to use windowed byte truncation instead of full-buffer encoding, improving memory efficiency on large outputs.
- Migrated from Buffer to Uint8Array in web scrapers for better cross-platform compatibility and native browser support.
- Added getArtifactManager() lazy-initialization method to ToolSession for deferred artifact manager instantiation.
- Simplified API surface with wildcard exports from tools and session modules, reducing import complexity.
This commit is contained in:
can1357
2026-02-22 00:46:31 +01:00
parent 9dbf13513a
commit e4225d6829
33 changed files with 1068 additions and 705 deletions
+31
View File
@@ -1,6 +1,37 @@
# Changelog
## [Unreleased]
### Added
- Exported truncation utilities and streaming output types from `session/streaming-output` module for public use
- Added `TailBuffer` class for efficient ring-style buffering with lazy joining and windowed truncation
- Added `truncateTailBytes` and `truncateHeadBytes` functions for UTF-8-aware byte-level truncation
- Added `formatTailTruncationNotice` and `formatHeadTruncationNotice` functions for consistent truncation notice formatting
- Added `allocateOutputArtifact` function to allocate artifact paths for tool output spilling
- Added `getArtifactManager()` method to `ToolSession` for lazy artifact manager access
### Changed
- Moved truncation logic from `tools/truncate.ts` to `session/streaming-output.ts` for better architectural separation
- Renamed `formatSize()` to `formatBytes()` across codebase for consistency
- Refactored `OutputSink` to use windowed byte truncation for memory efficiency when spilling to files
- Changed `ToolSession.artifactManager` from cached property to `getArtifactManager()` method for lazy initialization
- Updated `allocateOutputArtifact` return type to use `{ path, id }` instead of `{ artifactPath, artifactId }`
- Optimized newline counting to use native `indexOf` for V8-optimized string scanning
- Improved UTF-8 boundary detection in byte truncation with dedicated helper functions
### Removed
- Deleted `tools/truncate.ts` module (functionality moved to `session/streaming-output.ts`)
- Deleted `tools/output-utils.ts` module (functionality moved to `session/streaming-output.ts`)
- Removed `formatSize` export from public API (renamed to `formatBytes`)
- Removed individual truncation function exports from `tools/index.ts` (now exported via `session/streaming-output`)
### Fixed
- Fixed UTF-8 boundary handling in byte truncation to prevent invalid character sequences
- Fixed memory efficiency in `OutputSink` by using windowed truncation instead of full-buffer encoding
- Fixed line counting to handle chunk boundaries correctly across multiple `push()` calls
## [12.18.1] - 2026-02-21
### Added
+1 -1
View File
@@ -424,7 +424,7 @@ Key meta blocks:
`formatOutputNotice(meta)` converts metadata into appended textual notices (for model visibility), including:
- shown line range and total line count
- byte-limit context via `formatSize(...)`
- byte-limit context via `formatBytes(...)`
- pagination hint (`nextOffset`) for head-truncated output
- artifact recovery hint (`Full: artifact://<id>`)
- limit and diagnostics notices
@@ -8,7 +8,7 @@ import { isEnoent } from "@oh-my-pi/pi-utils";
import { getProjectDir } from "@oh-my-pi/pi-utils/dirs";
import chalk from "chalk";
import { resolveReadPath } from "../tools/path-utils";
import { formatSize } from "../tools/truncate";
import { formatBytes } from "../tools/render-utils";
import { formatDimensionNote, resizeImage } from "../utils/image-resize";
import { detectSupportedImageMimeTypeFromFile } from "../utils/mime";
@@ -47,9 +47,9 @@ export async function processFileArguments(fileArgs: string[], options?: Process
const maxBytes = mimeType ? MAX_CLI_IMAGE_BYTES : MAX_CLI_TEXT_BYTES;
if (stat.size > maxBytes) {
console.error(
chalk.yellow(`Warning: Skipping file contents (too large: ${formatSize(stat.size)}): ${absolutePath}`),
chalk.yellow(`Warning: Skipping file contents (too large: ${formatBytes(stat.size)}): ${absolutePath}`),
);
text += `<file name="${absolutePath}">(skipped: too large, ${formatSize(stat.size)})</file>\n`;
text += `<file name="${absolutePath}">(skipped: too large, ${formatBytes(stat.size)})</file>\n`;
continue;
}
+1 -6
View File
@@ -11,6 +11,7 @@ import { getSessionsDir } from "@oh-my-pi/pi-utils/dirs";
import { DynamicBorder } from "../modes/components/dynamic-border";
import { getSelectListTheme, getSymbolTheme, theme } from "../modes/theme/theme";
import type { InteractiveModeContext } from "../modes/types";
import { formatBytes } from "../tools/render-utils";
import { openPath } from "../utils/open";
import { DebugLogViewerComponent } from "./log-viewer";
import { generateHeapSnapshotData, type ProfilerSession, startCpuProfile } from "./profiler";
@@ -417,12 +418,6 @@ export class DebugSelectorComponent extends Container {
}
}
function formatBytes(bytes: number): string {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
/**
* Show the debug selector.
*/
+1 -24
View File
@@ -243,27 +243,4 @@ export {
export { runSubprocess } from "./task/executor";
export type { AgentDefinition, AgentProgress, AgentSource, SingleResult, TaskParams } from "./task/types";
// Tools (detail types and utilities)
export {
type BashToolDetails,
type BashToolInput,
type BrowserToolDetails,
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
type FindOperations,
type FindToolDetails,
type FindToolInput,
type FindToolOptions,
formatSize,
type GrepToolDetails,
type GrepToolInput,
type PythonToolDetails,
type ReadToolDetails,
type ReadToolInput,
type TruncationOptions,
type TruncationResult,
truncateHead,
truncateLine,
truncateTail,
type WriteToolDetails,
type WriteToolInput,
} from "./tools";
export * from "./tools";
@@ -6,7 +6,7 @@ import { sanitizeText } from "@oh-my-pi/pi-natives";
import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui";
import { getSymbolTheme, theme } from "../../modes/theme/theme";
import type { TruncationMeta } from "../../tools/output-meta";
import { formatSize } from "../../tools/truncate";
import { formatBytes } from "../../tools/render-utils";
import { DynamicBorder } from "./dynamic-border";
import { truncateToVisualLines } from "./visual-truncate";
@@ -177,7 +177,7 @@ export class BashExecutionComponent extends Container {
);
} else {
warnings.push(
`Truncated: ${this.#truncation.outputLines} lines shown (${formatSize(this.#truncation.outputBytes)} limit)`,
`Truncated: ${this.#truncation.outputLines} lines shown (${formatBytes(this.#truncation.outputBytes)} limit)`,
);
}
statusParts.push(theme.fg("warning", warnings.join(". ")));
@@ -7,7 +7,7 @@ import { sanitizeText } from "@oh-my-pi/pi-natives";
import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui";
import { getSymbolTheme, highlightCode, theme } from "../../modes/theme/theme";
import type { TruncationMeta } from "../../tools/output-meta";
import { formatSize } from "../../tools/truncate";
import { formatBytes } from "../../tools/render-utils";
import { DynamicBorder } from "./dynamic-border";
import { truncateToVisualLines } from "./visual-truncate";
@@ -161,7 +161,7 @@ export class PythonExecutionComponent extends Container {
);
} else {
warnings.push(
`Truncated: ${this.#truncation.outputLines} lines shown (${formatSize(this.#truncation.outputBytes)} limit)`,
`Truncated: ${this.#truncation.outputLines} lines shown (${formatBytes(this.#truncation.outputBytes)} limit)`,
);
}
statusParts.push(theme.fg("warning", warnings.join(". ")));
@@ -17,7 +17,7 @@ import { theme } from "../../modes/theme/theme";
import type { CompactionQueuedMessage, InteractiveModeContext } from "../../modes/types";
import { type CustomMessage, SKILL_PROMPT_MESSAGE_TYPE, type SkillPromptDetails } from "../../session/messages";
import type { SessionContext } from "../../session/session-manager";
import { formatSize } from "../../tools/truncate";
import { formatBytes } from "../../tools/render-utils";
type TextBlock = { type: "text"; text: string };
@@ -130,7 +130,7 @@ export class UiHelpers {
for (const file of message.files) {
let suffix: string;
if (file.skippedReason === "tooLarge") {
const size = typeof file.byteSize === "number" ? formatSize(file.byteSize) : "unknown size";
const size = typeof file.byteSize === "number" ? formatBytes(file.byteSize) : "unknown size";
suffix = `(skipped: ${size})`;
} else {
suffix = file.image
+14
View File
@@ -13,6 +13,7 @@ import { loadPromptTemplates as loadPromptTemplatesInternal, type PromptTemplate
import { Settings, type SkillsSettings } from "./config/settings";
import { CursorExecHandlers } from "./cursor";
import "./discovery";
import { ArtifactManager } from "@oh-my-pi/pi-coding-agent/session/artifacts";
import { initializeWithSettings } from "./discovery";
import { TtsrManager } from "./export/ttsr";
import {
@@ -715,6 +716,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
const enableLsp = options.enableLsp ?? true;
let artifactManager: ArtifactManager | null = null;
const toolSession: ToolSession = {
cwd,
hasUI: options.hasUI ?? false,
@@ -742,6 +744,18 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
},
getPlanModeState: () => session.getPlanModeState(),
getCompactContext: () => session.formatCompactContext(),
getArtifactManager: () => {
if (artifactManager) {
return artifactManager;
}
const sessionFile = sessionManager.getSessionFile();
if (!sessionFile) {
return null;
}
const manager = new ArtifactManager(sessionFile);
artifactManager = manager;
return manager;
},
settings,
authStorage,
modelRegistry,
@@ -1,5 +1,18 @@
import { sanitizeText } from "@oh-my-pi/pi-natives";
import { DEFAULT_MAX_BYTES } from "../tools/truncate";
import type { ToolSession } from "../tools";
import { formatBytes } from "../tools/render-utils";
// =============================================================================
// Constants
// =============================================================================
export const DEFAULT_MAX_LINES = 3000;
export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB
export const DEFAULT_MAX_COLUMN = 1024; // Max chars per grep match line
// =============================================================================
// Interfaces
// =============================================================================
export interface OutputSummary {
output: string;
@@ -19,40 +32,438 @@ export interface OutputSinkOptions {
onChunk?: (chunk: string) => void;
}
export interface TruncationResult {
content: string;
truncated: boolean;
truncatedBy: "lines" | "bytes" | null;
totalLines: number;
totalBytes: number;
outputLines: number;
outputBytes: number;
lastLinePartial: boolean;
firstLineExceedsLimit: boolean;
maxLines: number;
maxBytes: number;
}
export interface TruncationOptions {
/** Maximum number of lines (default: 3000) */
maxLines?: number;
/** Maximum number of bytes (default: 50KB) */
maxBytes?: number;
}
/** Result from byte-level truncation helpers. */
export interface ByteTruncationResult {
text: string;
bytes: number;
}
export interface TailTruncationNoticeOptions {
fullOutputPath?: string;
originalContent?: string;
suffix?: string;
}
export interface HeadTruncationNoticeOptions {
startLine?: number;
totalFileLines?: number;
}
// =============================================================================
// Low-level byte utilities
//
// Both use a "windowed encoding" strategy for strings: instead of encoding the
// entire input with Buffer.from(), we encode a window of at most `maxBytes`
// code units from the relevant end. Every JS code unit produces ≥1 UTF-8 byte,
// so this window is guaranteed to contain ≥ maxBytes encoded bytes (unless the
// input is shorter), avoiding O(N) encoding cost on large strings.
// =============================================================================
/** Advance past UTF-8 continuation bytes (10xxxxxx) to a leading byte. */
function findUtf8BoundaryForward(buf: Buffer, pos: number): number {
while (pos < buf.length && (buf[pos] & 0xc0) === 0x80) pos++;
return pos;
}
/** Retreat past UTF-8 continuation bytes to land on a leading byte. */
function findUtf8BoundaryBackward(buf: Buffer, pos: number): number {
while (pos > 0 && (buf[pos] & 0xc0) === 0x80) pos--;
return pos;
}
/**
* Truncate a string/buffer to fit within a byte limit, keeping the tail.
* Handles multi-byte UTF-8 boundaries correctly.
*/
export function truncateTailBytes(data: string | Uint8Array, maxBytes: number): ByteTruncationResult {
if (typeof data === "string") {
const len = Buffer.byteLength(data, "utf-8");
if (len <= maxBytes) return { text: data, bytes: len };
// Windowed: encode only the last `maxBytes` code units (≥ maxBytes bytes).
const window = data.substring(Math.max(0, data.length - maxBytes));
const buf = Buffer.from(window, "utf-8");
const start = findUtf8BoundaryForward(buf, buf.length - maxBytes);
const slice = buf.subarray(start);
return { text: slice.toString("utf-8"), bytes: slice.length };
}
// Uint8Array / Buffer path
if (data.length <= maxBytes) return { text: Buffer.from(data).toString("utf-8"), bytes: data.length };
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
const start = findUtf8BoundaryForward(buf, buf.length - maxBytes);
const slice = buf.subarray(start);
return { text: slice.toString("utf-8"), bytes: slice.length };
}
/**
* Truncate a string/buffer to fit within a byte limit, keeping the head.
* Handles multi-byte UTF-8 boundaries correctly.
*/
export function truncateHeadBytes(data: string | Uint8Array, maxBytes: number): ByteTruncationResult {
if (typeof data === "string") {
const len = Buffer.byteLength(data, "utf-8");
if (len <= maxBytes) return { text: data, bytes: len };
// Windowed: encode only the first `maxBytes` code units.
const window = data.substring(0, maxBytes);
const buf = Buffer.from(window, "utf-8");
const end = findUtf8BoundaryBackward(buf, maxBytes);
if (end <= 0) return { text: "", bytes: 0 };
const slice = buf.subarray(0, end);
return { text: slice.toString("utf-8"), bytes: slice.length };
}
if (data.length <= maxBytes) return { text: Buffer.from(data).toString("utf-8"), bytes: data.length };
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
const end = findUtf8BoundaryBackward(buf, maxBytes);
if (end <= 0) return { text: "", bytes: 0 };
const slice = buf.subarray(0, end);
return { text: slice.toString("utf-8"), bytes: slice.length };
}
// =============================================================================
// Line-level utilities
// =============================================================================
/**
* Count newline characters. Uses indexOf for V8-optimized native string scanning.
*/
function countNewlines(text: string): number {
let count = 0;
for (let i = 0; i < text.length; i++) {
if (text.charCodeAt(i) === 10) count += 1;
let pos = text.indexOf("\n", 0);
while (pos !== -1) {
count++;
pos = text.indexOf("\n", pos + 1);
}
return count;
}
function countLines(text: string): number {
if (text.length === 0) return 0;
return countNewlines(text) + 1;
/**
* Truncate a single line to max characters, appending '…' if truncated.
*/
export function truncateLine(
line: string,
maxChars: number = DEFAULT_MAX_COLUMN,
): { text: string; wasTruncated: boolean } {
if (line.length <= maxChars) return { text: line, wasTruncated: false };
return { text: `${line.slice(0, maxChars)}…`, wasTruncated: true };
}
function truncateStringToBytesFromEnd(text: string, maxBytes: number): { text: string; bytes: number } {
const buf = Buffer.from(text, "utf-8");
if (buf.length <= maxBytes) {
return { text, bytes: buf.length };
}
// =============================================================================
// Content truncation (line + byte aware)
//
// Both truncateHead and truncateTail encode the content to a Buffer once and
// collect newline byte-offsets in a single forward pass, avoiding split("\n")
// which would allocate N intermediate strings for N lines.
// =============================================================================
let start = buf.length - maxBytes;
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
start++;
}
const sliced = buf.subarray(start).toString("utf-8");
return { text: sliced, bytes: Buffer.byteLength(sliced, "utf-8") };
/** Shared helper to build a no-truncation result. */
function noTruncResult(
content: string,
totalLines: number,
totalBytes: number,
maxLines: number,
maxBytes: number,
): TruncationResult {
return {
content,
truncated: false,
truncatedBy: null,
totalLines,
totalBytes,
outputLines: totalLines,
outputBytes: totalBytes,
lastLinePartial: false,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
/**
* Line-buffered output sink with file spill support.
* Collect byte-offsets of every 0x0a in a Buffer. Avoids re-scanning.
*
* Uses a single string buffer with line position tracking.
* When memory limit exceeded, spills ~half to file in one batch operation.
* For a buffer with lines [L0 \n L1 \n L2]:
* nlOffsets = [offset_of_first_\n, offset_of_second_\n]
* Line k starts at (k === 0 ? 0 : nlOffsets[k-1] + 1)
* Line k ends at (k < nlOffsets.length ? nlOffsets[k] - 1 : buf.length - 1)
*/
function collectNewlineOffsets(buf: Buffer): number[] {
const offsets: number[] = [];
let pos = Bun.indexOfLine(buf, 0);
while (pos !== -1) {
offsets.push(pos);
pos = Bun.indexOfLine(buf, pos + 1);
}
return offsets;
}
/**
* Truncate content from the head (keep first N lines/bytes).
* Suitable for file reads where you want to see the beginning.
*
* Uses indexOfLine to scan forward incrementally — stops as soon as the
* line or byte limit is hit without ever collecting all newline positions.
*
* Never returns partial lines. If the first line exceeds the byte limit,
* returns empty content with firstLineExceedsLimit=true.
*/
export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult {
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
const totalBytes = Buffer.byteLength(content, "utf-8");
// Fast path: if under byte limit, only need a string-level newline count
if (totalBytes <= maxBytes) {
const totalLines = countNewlines(content) + 1;
if (totalLines <= maxLines) {
return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes);
}
}
// Slow path: encode once, scan with native indexOfLine
const totalLines = countNewlines(content) + 1;
if (totalLines <= maxLines && totalBytes <= maxBytes) {
return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes);
}
// Forward scan: include complete lines that fit within both limits.
// Uses indexOfLine incrementally — no offset array needed.
let includedLines = 0;
let cutByte = 0;
let truncatedBy: "lines" | "bytes" = "lines";
let scanPos = 0;
const buf = Buffer.from(content, "utf-8");
while (includedLines < maxLines) {
const nlPos = Bun.indexOfLine(buf, scanPos);
// Byte position right after this line's trailing \n, or end-of-buffer
const lineEnd = nlPos === -1 ? buf.length : nlPos + 1;
// Output strips a trailing \n from the final slice; enforce limit on displayed bytes
const outputEnd = nlPos === -1 ? lineEnd : lineEnd - 1;
if (outputEnd > maxBytes) {
truncatedBy = "bytes";
if (includedLines === 0) {
return {
content: "",
truncated: true,
truncatedBy: "bytes",
totalLines,
totalBytes,
outputLines: 0,
outputBytes: 0,
lastLinePartial: false,
firstLineExceedsLimit: true,
maxLines,
maxBytes,
};
}
break;
}
cutByte = lineEnd;
includedLines++;
if (nlPos === -1) break; // no more lines
scanPos = nlPos + 1;
}
if (includedLines >= maxLines && cutByte <= maxBytes) {
truncatedBy = "lines";
}
// Strip trailing \n so output matches join("\n") semantics
const sliceEnd = cutByte > 0 && buf[cutByte - 1] === 0x0a ? cutByte - 1 : cutByte;
const outputContent = buf.subarray(0, sliceEnd).toString("utf-8");
return {
content: outputContent,
truncated: true,
truncatedBy,
totalLines,
totalBytes,
outputLines: includedLines,
outputBytes: sliceEnd,
lastLinePartial: false,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
/**
* Truncate content from the tail (keep last N lines/bytes).
* Suitable for bash output where you want to see the end (errors, final results).
*
* May return a partial first line if the last line exceeds the byte limit.
*/
export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult {
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
const totalBytes = Buffer.byteLength(content, "utf-8");
const totalLines = countNewlines(content) + 1;
if (totalLines <= maxLines && totalBytes <= maxBytes) {
return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes);
}
const buf = Buffer.from(content, "utf-8");
const nlOffsets = collectNewlineOffsets(buf);
const lineCount = nlOffsets.length + 1;
// Walk backward, accumulating complete lines that fit.
let includedLines = 0;
let startByte = buf.length;
let truncatedBy: "lines" | "bytes" = "lines";
for (let lineIdx = lineCount - 1; lineIdx >= 0 && includedLines < maxLines; lineIdx--) {
const lineStart = lineIdx === 0 ? 0 : nlOffsets[lineIdx - 1] + 1;
const spanBytes = buf.length - lineStart;
if (spanBytes > maxBytes) {
truncatedBy = "bytes";
if (includedLines === 0) {
// Last line alone exceeds byte limit — take its tail
const tailResult = truncateTailBytes(buf, maxBytes);
return {
content: tailResult.text,
truncated: true,
truncatedBy: "bytes",
totalLines,
totalBytes,
outputLines: 1,
outputBytes: tailResult.bytes,
lastLinePartial: true,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
break;
}
startByte = lineStart;
includedLines++;
}
if (includedLines >= maxLines && buf.length - startByte <= maxBytes) {
truncatedBy = "lines";
}
const outputBytes = buf.length - startByte;
const outputContent = buf.subarray(startByte).toString("utf-8");
return {
content: outputContent,
truncated: true,
truncatedBy,
totalLines,
totalBytes,
outputLines: includedLines,
outputBytes,
lastLinePartial: false,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
// =============================================================================
// TailBuffer — ring-style tail buffer with lazy joining
//
// Uses windowed truncateTailBytes for trimming, so only a suffix of the
// accumulated string is ever encoded — not the entire buffer.
// =============================================================================
const MAX_PENDING = 10;
export class TailBuffer {
#pending: string[] = [];
#pos = 0; // tracked byte count (approximate after trims)
constructor(readonly maxBytes: number) {}
append(text: string): void {
if (!text) return;
const n = Buffer.byteLength(text, "utf-8");
this.#pos += n;
if (this.#pending.length > 0) {
this.#pending.push(text);
if (this.#pending.length > MAX_PENDING) this.#compact();
// Trim when we exceed 2× budget to amortise cost
if (this.#pos > this.maxBytes * 2) this.#trim();
} else {
this.#pending[0] = text;
this.#pending.length = 1;
}
}
text(): string {
return this.#trim();
}
bytes(): number {
return this.#pos;
}
// -- private ---------------------------------------------------------------
#compact(): void {
this.#pending[0] = this.#pending.join("");
this.#pending.length = 1;
}
#flush(): string {
if (this.#pending.length === 0) return "";
if (this.#pending.length > 1) this.#compact();
return this.#pending[0];
}
/** Trim the buffer to maxBytes using windowed tail truncation. */
#trim(): string {
if (this.#pos <= this.maxBytes) return this.#flush();
const joined = this.#flush();
const { text, bytes } = truncateTailBytes(joined, this.maxBytes);
this.#pos = bytes;
this.#pending[0] = text;
this.#pending.length = 1;
return text;
}
}
// =============================================================================
// OutputSink — line-buffered output with file spill support
//
// Uses a string buffer with byte tracking. When the spill threshold is
// exceeded, all data is written to a file sink and the in-memory buffer is
// trimmed to a tail window via windowed truncateTailBytes.
// =============================================================================
export class OutputSink {
#buffer = "";
#bufferBytes = 0;
@@ -60,11 +471,13 @@ export class OutputSink {
#totalBytes = 0;
#sawData = false;
#truncated = false;
#file?: {
path: string;
artifactId?: string;
sink: Bun.FileSink;
};
readonly #artifactPath?: string;
readonly #artifactId?: string;
readonly #spillThreshold: number;
@@ -72,66 +485,44 @@ export class OutputSink {
constructor(options?: OutputSinkOptions) {
const { artifactPath, artifactId, spillThreshold = DEFAULT_MAX_BYTES, onChunk } = options ?? {};
this.#artifactPath = artifactPath;
this.#artifactId = artifactId;
this.#spillThreshold = spillThreshold;
this.#onChunk = onChunk;
}
async #pushSanitized(data: string): Promise<void> {
this.#onChunk?.(data);
const dataBytes = Buffer.byteLength(data, "utf-8");
this.#totalBytes += dataBytes;
if (data.length > 0) {
this.#sawData = true;
this.#totalLines += countNewlines(data);
}
const bufferOverflow = this.#bufferBytes + dataBytes > this.#spillThreshold;
const overflow = this.#file || bufferOverflow;
const sink = overflow ? await this.#fileSink() : null;
this.#buffer += data;
this.#bufferBytes += dataBytes;
await sink?.write(data);
if (bufferOverflow) {
this.#truncated = true;
const trimmed = truncateStringToBytesFromEnd(this.#buffer, this.#spillThreshold);
this.#buffer = trimmed.text;
this.#bufferBytes = trimmed.bytes;
}
if (this.#file) {
this.#truncated = true;
}
}
async #fileSink(): Promise<Bun.FileSink | null> {
if (!this.#artifactPath) return null;
if (!this.#file) {
try {
this.#file = {
path: this.#artifactPath,
artifactId: this.#artifactId,
sink: Bun.file(this.#artifactPath).writer(),
};
await this.#file.sink.write(this.#buffer);
} catch {
try {
await this.#file?.sink?.end();
} catch {}
this.#file = undefined;
return null;
}
}
return this.#file.sink;
}
async push(chunk: string): Promise<void> {
chunk = sanitizeText(chunk);
await this.#pushSanitized(chunk);
this.#onChunk?.(chunk);
const dataBytes = Buffer.byteLength(chunk, "utf-8");
this.#totalBytes += dataBytes;
if (chunk.length > 0) {
this.#sawData = true;
this.#totalLines += countNewlines(chunk);
}
const willOverflow = this.#bufferBytes + dataBytes > this.#spillThreshold;
// Write to file if already spilling or about to overflow
if (this.#file != null || willOverflow) {
const sink = await this.#ensureFileSink();
await sink?.write(chunk);
}
this.#buffer += chunk;
this.#bufferBytes += dataBytes;
// Keep only a tail window in memory when overflowing
if (willOverflow) {
this.#truncated = true;
const { text, bytes } = truncateTailBytes(this.#buffer, this.#spillThreshold);
this.#buffer = text;
this.#bufferBytes = bytes;
}
if (this.#file) this.#truncated = true;
}
createInput(): WritableStream<Uint8Array | string> {
@@ -139,14 +530,9 @@ export class OutputSink {
const finalize = async () => {
await this.push(dec.decode());
};
return new WritableStream({
write: async chunk => {
if (typeof chunk === "string") {
await this.push(chunk);
} else {
await this.push(dec.decode(chunk, { stream: true }));
}
await this.push(typeof chunk === "string" ? chunk : dec.decode(chunk, { stream: true }));
},
close: finalize,
abort: finalize,
@@ -155,23 +541,120 @@ export class OutputSink {
async dump(notice?: string): Promise<OutputSummary> {
const noticeLine = notice ? `[${notice}]\n` : "";
const outputLines = countLines(this.#buffer);
const outputBytes = this.#bufferBytes;
const outputLines = this.#buffer.length > 0 ? countNewlines(this.#buffer) + 1 : 0;
const totalLines = this.#sawData ? this.#totalLines + 1 : 0;
const totalBytes = this.#totalBytes;
if (this.#file) {
await this.#file.sink.end();
}
if (this.#file) await this.#file.sink.end();
return {
output: `${noticeLine}${this.#buffer}`,
truncated: this.#truncated,
totalLines,
totalBytes,
totalBytes: this.#totalBytes,
outputLines,
outputBytes,
outputBytes: this.#bufferBytes,
artifactId: this.#file?.artifactId,
};
}
// -- private ---------------------------------------------------------------
async #ensureFileSink(): Promise<Bun.FileSink | null> {
if (!this.#artifactPath) return null;
if (this.#file) return this.#file.sink;
try {
const sink = Bun.file(this.#artifactPath).writer();
this.#file = { path: this.#artifactPath, artifactId: this.#artifactId, sink };
// Flush existing buffer to file BEFORE it gets trimmed
await sink.write(this.#buffer);
return sink;
} catch {
try {
await this.#file?.sink?.end();
} catch {
/* ignore */
}
this.#file = undefined;
return null;
}
}
}
// =============================================================================
// Session helpers
// =============================================================================
const kEmpty = Object.freeze({} as { id?: string; path?: string });
/** Allocate a new artifact path and ID without writing content. */
export async function allocateOutputArtifact(session: ToolSession, toolType: string) {
const manager = session.getArtifactManager?.();
if (!manager) return kEmpty;
try {
return await manager.allocatePath(toolType);
} catch {
return kEmpty;
}
}
// =============================================================================
// Truncation notice formatting
// =============================================================================
/**
* Format a truncation notice for tail-truncated output (bash, python, ssh).
* Returns empty string if not truncated.
*/
export function formatTailTruncationNotice(
truncation: TruncationResult,
options: TailTruncationNoticeOptions = {},
): string {
if (!truncation.truncated) return "";
const { fullOutputPath, originalContent, suffix = "" } = options;
const startLine = truncation.totalLines - truncation.outputLines + 1;
const endLine = truncation.totalLines;
const fullOutputPart = fullOutputPath ? `. Full output: ${fullOutputPath}` : "";
let notice: string;
if (truncation.lastLinePartial) {
let lastLineSizePart = "";
if (originalContent) {
const lastNl = originalContent.lastIndexOf("\n");
const lastLine = lastNl === -1 ? originalContent : originalContent.substring(lastNl + 1);
lastLineSizePart = ` (line is ${formatBytes(Buffer.byteLength(lastLine, "utf-8"))})`;
}
notice = `[Showing last ${formatBytes(truncation.outputBytes)} of line ${endLine}${lastLineSizePart}${fullOutputPart}${suffix}]`;
} else if (truncation.truncatedBy === "lines") {
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}${fullOutputPart}${suffix}]`;
} else {
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatBytes(truncation.maxBytes)} limit)${fullOutputPart}${suffix}]`;
}
return `\n\n${notice}`;
}
/**
* Format a truncation notice for head-truncated output (read tool).
* Returns empty string if not truncated.
*/
export function formatHeadTruncationNotice(
truncation: TruncationResult,
options: HeadTruncationNoticeOptions = {},
): string {
if (!truncation.truncated) return "";
const startLineDisplay = options.startLine ?? 1;
const totalFileLines = options.totalFileLines ?? truncation.totalLines;
const endLineDisplay = startLineDisplay + truncation.outputLines - 1;
const nextOffset = endLineDisplay + 1;
const notice =
truncation.truncatedBy === "lines"
? `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`
: `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines} (${formatBytes(truncation.maxBytes)} limit). Use offset=${nextOffset} to continue]`;
return `\n\n${notice}`;
}
+1 -8
View File
@@ -26,7 +26,7 @@ import type { Theme } from "../modes/theme/theme";
import planModeSubagentPrompt from "../prompts/system/plan-mode-subagent.md" with { type: "text" };
import taskDescriptionTemplate from "../prompts/tools/task.md" with { type: "text" };
import taskSummaryTemplate from "../prompts/tools/task-summary.md" with { type: "text" };
import { formatDuration } from "../tools/render-utils";
import { formatBytes, formatDuration } from "../tools/render-utils";
// Import review tools for side effects (registers subagent tool handlers)
import "../tools/review";
import { discoverAgents, getAgent } from "./discovery";
@@ -55,13 +55,6 @@ import {
type WorktreeBaseline,
} from "./worktree";
/** Format byte count for display */
function formatBytes(bytes: number): string {
if (bytes < 1024) return `${bytes}B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}K`;
return `${(bytes / (1024 * 1024)).toFixed(1)}M`;
}
function createUsageTotals(): Usage {
return {
input: 0,
+3 -4
View File
@@ -12,6 +12,7 @@ import type { RenderResultOptions } from "../extensibility/custom-tools/types";
import { truncateToVisualLines } from "../modes/components/visual-truncate";
import type { Theme } from "../modes/theme/theme";
import bashDescription from "../prompts/tools/bash.md" with { type: "text" };
import { allocateOutputArtifact, DEFAULT_MAX_BYTES, TailBuffer } from "../session/streaming-output";
import { renderStatusLine } from "../tui";
import { CachedOutputBlock } from "../tui/output-block";
import type { ToolSession } from ".";
@@ -20,12 +21,10 @@ import { checkBashInterception } from "./bash-interceptor";
import { applyHeadTail } from "./bash-normalize";
import { expandInternalUrls } from "./bash-skill-urls";
import type { OutputMeta } from "./output-meta";
import { allocateOutputArtifact, createTailBuffer } from "./output-utils";
import { resolveToCwd } from "./path-utils";
import { formatBytes, replaceTabs, wrapBrackets } from "./render-utils";
import { ToolAbortError, ToolError } from "./tool-errors";
import { toolResult } from "./tool-result";
import { DEFAULT_MAX_BYTES } from "./truncate";
export const BASH_DEFAULT_PREVIEW_LINES = 10;
@@ -114,12 +113,12 @@ export class BashTool implements AgentTool<typeof bashSchema, BashToolDetails> {
const timeoutMs = timeoutSec * 1000;
// Track output for streaming updates (tail only)
const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES);
const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES);
// Set up artifacts environment and allocation
const artifactsDir = this.session.getArtifactsDir?.();
const extraEnv = artifactsDir ? { ARTIFACTS: artifactsDir } : undefined;
const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "bash");
const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "bash");
const usePty =
this.session.settings.get("bash.virtualTerminal") === "on" &&
+3 -4
View File
@@ -10,6 +10,7 @@ import { renderPromptTemplate } from "../config/prompt-templates";
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
import { type Theme, theme } from "../modes/theme/theme";
import fetchDescription from "../prompts/tools/fetch.md" with { type: "text" };
import { allocateOutputArtifact, DEFAULT_MAX_BYTES, truncateHead } from "../session/streaming-output";
import { renderStatusLine } from "../tui";
import { CachedOutputBlock } from "../tui/output-block";
import { ensureTool } from "../utils/tools-manager";
@@ -20,11 +21,9 @@ import { convertWithMarkitdown, fetchBinary } from "../web/scrapers/utils";
import type { ToolSession } from ".";
import { applyListLimit } from "./list-limit";
import type { OutputMeta } from "./output-meta";
import { allocateOutputArtifact } from "./output-utils";
import { formatExpandHint } from "./render-utils";
import { ToolAbortError } from "./tool-errors";
import { toolResult } from "./tool-result";
import { DEFAULT_MAX_BYTES, truncateHead } from "./truncate";
// =============================================================================
// Types and Constants
@@ -900,10 +899,10 @@ export class FetchTool implements AgentTool<typeof fetchSchema, FetchToolDetails
};
if (needsArtifact) {
const { artifactPath, artifactId: allocatedId } = await allocateOutputArtifact(this.session, "fetch");
const { path: artifactPath, id } = await allocateOutputArtifact(this.session, "fetch");
if (artifactPath) {
await Bun.write(artifactPath, buildOutput(result.content));
artifactId = allocatedId;
artifactId = id;
}
}
+1 -1
View File
@@ -11,6 +11,7 @@ import { renderPromptTemplate } from "../config/prompt-templates";
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
import type { Theme } from "../modes/theme/theme";
import findDescription from "../prompts/tools/find.md" with { type: "text" };
import { type TruncationResult, truncateHead } from "../session/streaming-output";
import {
Ellipsis,
Hasher,
@@ -27,7 +28,6 @@ import { resolveToCwd } from "./path-utils";
import { formatCount, formatEmptyMessage, formatErrorMessage, PREVIEW_LIMITS } from "./render-utils";
import { ToolAbortError, ToolError, throwIfAborted } from "./tool-errors";
import { toolResult } from "./tool-result";
import { type TruncationResult, truncateHead } from "./truncate";
const findSchema = Type.Object({
pattern: Type.String({ description: "Glob pattern, e.g. '*.ts', 'src/**/*.json', 'lib/*.tsx'" }),
+1 -1
View File
@@ -11,6 +11,7 @@ import type { RenderResultOptions } from "../extensibility/custom-tools/types";
import type { Theme } from "../modes/theme/theme";
import { computeLineHash } from "../patch/hashline";
import grepDescription from "../prompts/tools/grep.md" with { type: "text" };
import { DEFAULT_MAX_COLUMN, type TruncationResult, truncateHead } from "../session/streaming-output";
import { Ellipsis, Hasher, type RenderCache, renderStatusLine, renderTreeList, truncateToWidth } from "../tui";
import { resolveFileDisplayMode } from "../utils/file-display-mode";
import type { ToolSession } from ".";
@@ -19,7 +20,6 @@ import { resolveToCwd } from "./path-utils";
import { formatCount, formatEmptyMessage, formatErrorMessage, PREVIEW_LIMITS } from "./render-utils";
import { ToolError } from "./tool-errors";
import { toolResult } from "./tool-result";
import { DEFAULT_MAX_COLUMN, type TruncationResult, truncateHead } from "./truncate";
const grepSchema = Type.Object({
pattern: Type.String({ description: "Regex pattern to search for" }),
+4 -28
View File
@@ -48,23 +48,9 @@ export {
warmupLspServers,
} from "../lsp";
export { EditTool, type EditToolDetails } from "../patch";
export * from "../session/streaming-output";
export { BUNDLED_AGENTS, TaskTool } from "../task";
export {
companySearchTools,
exaSearchTools,
getSearchTools,
type SearchProvider,
type SearchResponse,
SearchTool,
type SearchToolsOptions,
setPreferredSearchProvider,
webSearchCodeContextTool,
webSearchCompanyTool,
webSearchCrawlTool,
webSearchCustomTool,
webSearchDeepTool,
webSearchLinkedinTool,
} from "../web/search";
export * from "../web/search";
export { AskTool, type AskToolDetails } from "./ask";
export { BashTool, type BashToolDetails, type BashToolInput, type BashToolOptions } from "./bash";
export { BrowserTool, type BrowserToolDetails } from "./browser";
@@ -81,16 +67,6 @@ export { reportFindingTool, type SubmitReviewDetails } from "./review";
export { loadSshTool, type SSHToolDetails, SshTool } from "./ssh";
export { SubmitResultTool } from "./submit-result";
export { type TodoItem, TodoWriteTool, type TodoWriteToolDetails } from "./todo-write";
export {
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
formatSize,
type TruncationOptions,
type TruncationResult,
truncateHead,
truncateLine,
truncateTail,
} from "./truncate";
export { WriteTool, type WriteToolDetails, type WriteToolInput } from "./write";
/** Tool type (AgentTool from pi-ai) */
@@ -132,8 +108,8 @@ export interface ToolSession {
getSessionFile: () => string | null;
/** Get session ID */
getSessionId?: () => string | null;
/** Cached artifact manager (allocated per ToolSession) */
artifactManager?: ArtifactManager;
/** Get artifact manager (allocated per ToolSession) */
getArtifactManager?: () => ArtifactManager | null;
/** Get artifacts directory for artifact:// URLs and $ARTIFACTS env var */
getArtifactsDir?: () => string | null;
/** Get session spawns */
@@ -12,10 +12,9 @@ import type {
AgentToolUpdateCallback,
} from "@oh-my-pi/pi-agent-core";
import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai";
import type { OutputSummary } from "../session/streaming-output";
import type { OutputSummary, TruncationResult } from "../session/streaming-output";
import { formatBytes } from "./render-utils";
import { renderError } from "./tool-errors";
import type { TruncationResult } from "./truncate";
import { formatSize } from "./truncate";
/**
* Truncation metadata for the output notice.
@@ -336,7 +335,7 @@ export function formatOutputNotice(meta: OutputMeta | undefined): string {
if (t.truncatedBy === "bytes") {
const maxBytes = t.maxBytes ?? t.outputBytes;
notice += ` (${formatSize(maxBytes)} limit)`;
notice += ` (${formatBytes(maxBytes)} limit)`;
}
if (t.nextOffset != null) {
@@ -1,63 +0,0 @@
import { ArtifactManager } from "../session/artifacts";
import type { ToolSession } from ".";
export interface TailBuffer {
append(chunk: string): void;
text(): string;
bytes(): number;
}
export function createTailBuffer(maxBytes: number): TailBuffer {
let buffer = "";
let bufferBytes = 0;
const append = (text: string) => {
if (!text) return;
const chunkBytes = Buffer.byteLength(text, "utf-8");
buffer += text;
bufferBytes += chunkBytes;
if (bufferBytes > maxBytes) {
const buf = Buffer.from(buffer, "utf-8");
let start = Math.max(0, buf.length - maxBytes);
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
start++;
}
buffer = buf.subarray(start).toString("utf-8");
bufferBytes = Buffer.byteLength(buffer, "utf-8");
}
};
return {
append,
text: () => buffer,
bytes: () => bufferBytes,
};
}
export function getArtifactManager(session: ToolSession): ArtifactManager | null {
if (session.artifactManager) {
return session.artifactManager;
}
const sessionFile = session.getSessionFile();
if (!sessionFile) {
return null;
}
const manager = new ArtifactManager(sessionFile);
session.artifactManager = manager;
return manager;
}
export async function allocateOutputArtifact(
session: ToolSession,
toolType: string,
): Promise<{ artifactPath?: string; artifactId?: string }> {
const manager = getArtifactManager(session);
if (!manager) return {};
try {
const allocation = await manager.allocatePath(toolType);
return { artifactPath: allocation.path, artifactId: allocation.id };
} catch {
return {};
}
}
+9 -5
View File
@@ -13,16 +13,20 @@ import type { PreludeHelper, PythonStatusEvent } from "../ipy/kernel";
import { truncateToVisualLines } from "../modes/components/visual-truncate";
import type { Theme } from "../modes/theme/theme";
import pythonDescription from "../prompts/tools/python.md" with { type: "text" };
import { OutputSink, type OutputSummary } from "../session/streaming-output";
import {
allocateOutputArtifact,
DEFAULT_MAX_BYTES,
OutputSink,
type OutputSummary,
TailBuffer,
} from "../session/streaming-output";
import { getTreeBranch, getTreeContinuePrefix, renderCodeCell } from "../tui";
import type { ToolSession } from ".";
import type { OutputMeta } from "./output-meta";
import { allocateOutputArtifact, createTailBuffer } from "./output-utils";
import { resolveToCwd } from "./path-utils";
import { replaceTabs, shortenPath, ToolUIKit, truncateToWidth } from "./render-utils";
import { ToolAbortError, ToolError } from "./tool-errors";
import { toolResult } from "./tool-result";
import { DEFAULT_MAX_BYTES } from "./truncate";
export const PYTHON_DEFAULT_PREVIEW_LINES = 10;
@@ -207,7 +211,7 @@ export class PythonTool implements AgentTool<typeof pythonSchema> {
throw new ToolError(`Working directory is not a directory: ${commandCwd}`);
}
const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES * 2);
const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES * 2);
const jsonOutputs: unknown[] = [];
const images: ImageContent[] = [];
const statusEvents: PythonStatusEvent[] = [];
@@ -255,7 +259,7 @@ export class PythonTool implements AgentTool<typeof pythonSchema> {
const sessionFile = this.session.getSessionFile?.() ?? undefined;
const artifactsDir = this.session.getArtifactsDir?.() ?? undefined;
const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "python");
const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "python");
outputSink = new OutputSink({
artifactPath,
artifactId,
+25 -35
View File
@@ -14,6 +14,13 @@ import { getLanguageFromPath, type Theme } from "../modes/theme/theme";
import { computeLineHash } from "../patch/hashline";
import readDescription from "../prompts/tools/read.md" with { type: "text" };
import type { ToolSession } from "../sdk";
import {
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
type TruncationResult,
truncateHead,
truncateHeadBytes,
} from "../session/streaming-output";
import { renderCodeCell, renderStatusLine } from "../tui";
import { CachedOutputBlock } from "../tui/output-block";
import { resolveFileDisplayMode } from "../utils/file-display-mode";
@@ -23,17 +30,9 @@ import { ensureTool } from "../utils/tools-manager";
import { applyListLimit } from "./list-limit";
import type { OutputMeta } from "./output-meta";
import { resolveReadPath, resolveToCwd } from "./path-utils";
import { formatAge, shortenPath, wrapBrackets } from "./render-utils";
import { formatAge, formatBytes, shortenPath, wrapBrackets } from "./render-utils";
import { ToolAbortError, ToolError, throwIfAborted } from "./tool-errors";
import { toolResult } from "./tool-result";
import {
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
formatSize,
type TruncationResult,
truncateHead,
truncateStringToBytesFromStart,
} from "./truncate";
// Document types convertible via markitdown
const CONVERTIBLE_EXTENSIONS = new Set([".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".rtf", ".epub"]);
@@ -211,17 +210,8 @@ async function streamLinesFromFile(
let firstLinePreview: { text: string; bytes: number } | undefined;
if (firstLinePreviewBytes > 0) {
const buf = Buffer.concat(firstLinePreviewChunks, firstLinePreviewBytes);
let end = Math.min(buf.length, maxBytes);
while (end > 0 && (buf[end] & 0xc0) === 0x80) {
end--;
}
if (end > 0) {
const text = buf.slice(0, end).toString("utf-8");
firstLinePreview = { text, bytes: Buffer.byteLength(text, "utf-8") };
} else {
firstLinePreview = { text: "", bytes: 0 };
}
const { text, bytes } = truncateHeadBytes(Buffer.concat(firstLinePreviewChunks, firstLinePreviewBytes), maxBytes);
firstLinePreview = { text, bytes };
}
return {
@@ -623,8 +613,8 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
if (mimeType) {
if (fileSize > MAX_IMAGE_SIZE) {
const sizeStr = formatSize(fileSize);
const maxStr = formatSize(MAX_IMAGE_SIZE);
const sizeStr = formatBytes(fileSize);
const maxStr = formatBytes(MAX_IMAGE_SIZE);
throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`);
} else {
// Read as image (binary)
@@ -633,8 +623,8 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
// Check actual buffer size after reading to prevent OOM during serialization
if (buffer.byteLength > MAX_IMAGE_SIZE) {
const sizeStr = formatSize(buffer.byteLength);
const maxStr = formatSize(MAX_IMAGE_SIZE);
const sizeStr = formatBytes(buffer.byteLength);
const maxStr = formatBytes(MAX_IMAGE_SIZE);
throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`);
} else {
const base64 = new Uint8Array(buffer).toBase64();
@@ -788,16 +778,16 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
const snippet = firstLinePreview ?? { text: "", bytes: 0 };
if (shouldAddHashLines) {
outputText = `[Line ${startLineDisplay} is ${formatSize(
outputText = `[Line ${startLineDisplay} is ${formatBytes(
firstLineBytes,
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
} else {
outputText = formatText(snippet.text, startLineDisplay);
}
if (snippet.text.length === 0) {
outputText = `[Line ${startLineDisplay} is ${formatSize(
outputText = `[Line ${startLineDisplay} is ${formatBytes(
firstLineBytes,
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
}
details = { truncation };
sourcePath = absolutePath;
@@ -946,19 +936,19 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
if (truncation.firstLineExceedsLimit) {
const firstLine = allLines[startLine] ?? "";
const firstLineBytes = Buffer.byteLength(firstLine, "utf-8");
const snippet = truncateStringToBytesFromStart(firstLine, DEFAULT_MAX_BYTES);
const snippet = truncateHeadBytes(firstLine, DEFAULT_MAX_BYTES);
if (shouldAddHashLines) {
outputText = `[Line ${startLineDisplay} is ${formatSize(
outputText = `[Line ${startLineDisplay} is ${formatBytes(
firstLineBytes,
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
} else {
outputText = formatText(snippet.text, startLineDisplay);
}
if (snippet.text.length === 0) {
outputText = `[Line ${startLineDisplay} is ${formatSize(
outputText = `[Line ${startLineDisplay} is ${formatBytes(
firstLineBytes,
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
}
details = { truncation };
truncationInfo = {
@@ -1116,12 +1106,12 @@ export const readToolRenderer = {
if (truncation) {
let warning: string;
if (fallback?.firstLineExceedsLimit) {
warning = `First line exceeds ${formatSize(fallback.maxBytes ?? DEFAULT_MAX_BYTES)} limit`;
warning = `First line exceeds ${formatBytes(fallback.maxBytes ?? DEFAULT_MAX_BYTES)} limit`;
} else if (truncation.truncatedBy === "lines") {
warning = `Truncated: ${truncation.outputLines} of ${truncation.totalLines} lines (${DEFAULT_MAX_LINES} line limit)`;
} else {
const maxBytes = fallback?.maxBytes ?? DEFAULT_MAX_BYTES;
warning = `Truncated: ${truncation.outputLines} lines (${formatSize(maxBytes)} limit)`;
warning = `Truncated: ${truncation.outputLines} lines (${formatBytes(maxBytes)} limit)`;
}
if (truncation.artifactId) {
warning += `. Full output: artifact://${truncation.artifactId}`;
+3 -4
View File
@@ -9,6 +9,7 @@ import { loadCapability } from "../discovery";
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
import type { Theme } from "../modes/theme/theme";
import sshDescriptionBase from "../prompts/tools/ssh.md" with { type: "text" };
import { allocateOutputArtifact, DEFAULT_MAX_BYTES, TailBuffer } from "../session/streaming-output";
import type { SSHHostInfo } from "../ssh/connection-manager";
import { ensureHostInfo, getHostInfoForHost } from "../ssh/connection-manager";
import { executeSSH } from "../ssh/ssh-executor";
@@ -16,11 +17,9 @@ import { renderStatusLine } from "../tui";
import { CachedOutputBlock } from "../tui/output-block";
import type { ToolSession } from ".";
import type { OutputMeta } from "./output-meta";
import { allocateOutputArtifact, createTailBuffer } from "./output-utils";
import { formatBytes, wrapBrackets } from "./render-utils";
import { ToolError } from "./tool-errors";
import { toolResult } from "./tool-result";
import { DEFAULT_MAX_BYTES } from "./truncate";
const sshSchema = Type.Object({
host: Type.String({ description: "Host name from managed SSH config or discovered ssh.json files" }),
@@ -159,8 +158,8 @@ export class SshTool implements AgentTool<typeof sshSchema, SSHToolDetails> {
const timeoutSec = Math.max(1, Math.min(3600, rawTimeout));
const timeoutMs = timeoutSec * 1000;
const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES);
const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "ssh");
const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES);
const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "ssh");
const result = await executeSSH(hostConfig, remoteCommand, {
timeout: timeoutMs,
@@ -1,9 +1,8 @@
import type { AgentToolResult } from "@oh-my-pi/pi-agent-core";
import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai";
import type { OutputSummary } from "../session/streaming-output";
import type { OutputSummary, TruncationResult } from "../session/streaming-output";
import type { OutputMeta, TruncationOptions, TruncationSummaryOptions, TruncationTextOptions } from "./output-meta";
import { outputMeta } from "./output-meta";
import type { TruncationResult } from "./truncate";
type ToolContent = Array<TextContent | ImageContent>;
-385
View File
@@ -1,385 +0,0 @@
/**
* Shared truncation utilities for tool outputs.
*
* Truncation is based on two independent limits - whichever is hit first wins:
* - Line limit (default: 4000 lines)
* - Byte limit (default: 50KB)
*
* Never returns partial lines (except bash tail truncation edge case
* and the read tool's long-line snippet fallback).
*/
export const DEFAULT_MAX_LINES = 3000;
export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB
export const DEFAULT_MAX_COLUMN = 1024; // Max chars per grep match line
export interface TruncationResult {
/** The truncated content */
content: string;
/** Whether truncation occurred */
truncated: boolean;
/** Which limit was hit: "lines", "bytes", or null if not truncated */
truncatedBy: "lines" | "bytes" | null;
/** Total number of lines in the original content */
totalLines: number;
/** Total number of bytes in the original content */
totalBytes: number;
/** Number of complete lines in the truncated output */
outputLines: number;
/** Number of bytes in the truncated output */
outputBytes: number;
/** Whether the last line was partially truncated (only for tail truncation edge case) */
lastLinePartial: boolean;
/** Whether the first line exceeded the byte limit (for head truncation) */
firstLineExceedsLimit: boolean;
/** The max lines limit that was applied */
maxLines: number;
/** The max bytes limit that was applied */
maxBytes: number;
}
export interface TruncationOptions {
/** Maximum number of lines (default: 2000) */
maxLines?: number;
/** Maximum number of bytes (default: 50KB) */
maxBytes?: number;
}
/**
* Format bytes as human-readable size.
*/
export function formatSize(bytes: number): string {
if (bytes < 1024) {
return `${bytes}B`;
} else if (bytes < 1024 * 1024) {
return `${(bytes / 1024).toFixed(1)}KB`;
} else if (bytes < 1024 * 1024 * 1024) {
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
} else {
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
}
}
/**
* Truncate content from the head (keep first N lines/bytes).
* Suitable for file reads where you want to see the beginning.
*
* Never returns partial lines. If first line exceeds byte limit,
* returns empty content with firstLineExceedsLimit=true.
*/
export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult {
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
const totalBytes = Buffer.byteLength(content, "utf-8");
const lines = content.split("\n");
const totalLines = lines.length;
// Check if no truncation needed
if (totalLines <= maxLines && totalBytes <= maxBytes) {
return {
content,
truncated: false,
truncatedBy: null,
totalLines,
totalBytes,
outputLines: totalLines,
outputBytes: totalBytes,
lastLinePartial: false,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
// Check if first line alone exceeds byte limit
const firstLineBytes = Buffer.byteLength(lines[0], "utf-8");
if (firstLineBytes > maxBytes) {
return {
content: "",
truncated: true,
truncatedBy: "bytes",
totalLines,
totalBytes,
outputLines: 0,
outputBytes: 0,
lastLinePartial: false,
firstLineExceedsLimit: true,
maxLines,
maxBytes,
};
}
// Collect complete lines that fit
const outputLinesArr: string[] = [];
let outputBytesCount = 0;
let truncatedBy: "lines" | "bytes" = "lines";
for (let i = 0; i < lines.length && i < maxLines; i++) {
const line = lines[i];
const lineBytes = Buffer.byteLength(line, "utf-8") + (i > 0 ? 1 : 0); // +1 for newline
if (outputBytesCount + lineBytes > maxBytes) {
truncatedBy = "bytes";
break;
}
outputLinesArr.push(line);
outputBytesCount += lineBytes;
}
// If we exited due to line limit
if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
truncatedBy = "lines";
}
const outputContent = outputLinesArr.join("\n");
const finalOutputBytes = Buffer.byteLength(outputContent, "utf-8");
return {
content: outputContent,
truncated: true,
truncatedBy,
totalLines,
totalBytes,
outputLines: outputLinesArr.length,
outputBytes: finalOutputBytes,
lastLinePartial: false,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
/**
* Truncate content from the tail (keep last N lines/bytes).
* Suitable for bash output where you want to see the end (errors, final results).
*
* May return partial first line if the last line of original content exceeds byte limit.
*/
export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult {
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
const totalBytes = Buffer.byteLength(content, "utf-8");
const lines = content.split("\n");
const totalLines = lines.length;
// Check if no truncation needed
if (totalLines <= maxLines && totalBytes <= maxBytes) {
return {
content,
truncated: false,
truncatedBy: null,
totalLines,
totalBytes,
outputLines: totalLines,
outputBytes: totalBytes,
lastLinePartial: false,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
// Work backwards from the end
const outputLinesArr: string[] = [];
let outputBytesCount = 0;
let truncatedBy: "lines" | "bytes" = "lines";
let lastLinePartial = false;
for (let i = lines.length - 1; i >= 0 && outputLinesArr.length < maxLines; i--) {
const line = lines[i];
const lineBytes = Buffer.byteLength(line, "utf-8") + (outputLinesArr.length > 0 ? 1 : 0); // +1 for newline
if (outputBytesCount + lineBytes > maxBytes) {
truncatedBy = "bytes";
// Edge case: if we haven't added ANY lines yet and this line exceeds maxBytes,
// take the end of the line (partial)
if (outputLinesArr.length === 0) {
const truncatedLine = truncateStringToBytesFromEnd(line, maxBytes);
outputLinesArr.unshift(truncatedLine);
outputBytesCount = Buffer.byteLength(truncatedLine, "utf-8");
lastLinePartial = true;
}
break;
}
outputLinesArr.unshift(line);
outputBytesCount += lineBytes;
}
// If we exited due to line limit
if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
truncatedBy = "lines";
}
const outputContent = outputLinesArr.join("\n");
const finalOutputBytes = Buffer.byteLength(outputContent, "utf-8");
return {
content: outputContent,
truncated: true,
truncatedBy,
totalLines,
totalBytes,
outputLines: outputLinesArr.length,
outputBytes: finalOutputBytes,
lastLinePartial,
firstLineExceedsLimit: false,
maxLines,
maxBytes,
};
}
/**
* Truncate a string to fit within a byte limit (from the end).
* Handles multi-byte UTF-8 characters correctly.
*/
function truncateStringToBytesFromEnd(str: string, maxBytes: number): string {
const buf = Buffer.from(str, "utf-8");
if (buf.length <= maxBytes) {
return str;
}
// Start from the end, skip maxBytes back
let start = buf.length - maxBytes;
// Find a valid UTF-8 boundary (start of a character)
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
start++;
}
return buf.slice(start).toString("utf-8");
}
/**
* Truncate a string to fit within a byte limit (from the start).
* Handles multi-byte UTF-8 characters correctly.
*/
export function truncateStringToBytesFromStart(str: string, maxBytes: number): { text: string; bytes: number } {
const buf = Buffer.from(str, "utf-8");
if (buf.length <= maxBytes) {
return { text: str, bytes: buf.length };
}
let end = maxBytes;
// Find a valid UTF-8 boundary (start of a character)
while (end > 0 && (buf[end] & 0xc0) === 0x80) {
end--;
}
if (end <= 0) {
return { text: "", bytes: 0 };
}
const text = buf.slice(0, end).toString("utf-8");
return { text, bytes: Buffer.byteLength(text, "utf-8") };
}
/**
* Truncate a single line to max characters, adding [truncated] suffix.
* Used for grep match lines.
*/
export function truncateLine(
line: string,
maxChars: number = DEFAULT_MAX_COLUMN,
): { text: string; wasTruncated: boolean } {
if (line.length <= maxChars) {
return { text: line, wasTruncated: false };
}
return { text: `${line.slice(0, maxChars)}…`, wasTruncated: true };
}
// =============================================================================
// Truncation notice formatting
// =============================================================================
export interface TailTruncationNoticeOptions {
/** Path to full output file (e.g., from bash/python executor) */
fullOutputPath?: string;
/** Original content for computing last line size when lastLinePartial */
originalContent?: string;
/** Additional suffix to append inside the brackets */
suffix?: string;
}
/**
* Format a truncation notice for tail-truncated output (bash, python, ssh).
* Returns empty string if not truncated.
*
* Examples:
* - "[Showing last 50KB of line 1000 (line is 2.1MB). Full output: /tmp/out.txt]"
* - "[Showing lines 500-1000 of 1000. Full output: /tmp/out.txt]"
* - "[Showing lines 500-1000 of 1000 (50KB limit). Full output: /tmp/out.txt]"
*/
export function formatTailTruncationNotice(
truncation: TruncationResult,
options: TailTruncationNoticeOptions = {},
): string {
if (!truncation.truncated) {
return "";
}
const { fullOutputPath, originalContent, suffix = "" } = options;
const startLine = truncation.totalLines - truncation.outputLines + 1;
const endLine = truncation.totalLines;
const fullOutputPart = fullOutputPath ? `. Full output: ${fullOutputPath}` : "";
let notice: string;
if (truncation.lastLinePartial) {
let lastLineSizePart = "";
if (originalContent) {
const lastLine = originalContent.split("\n").pop() || "";
lastLineSizePart = ` (line is ${formatSize(Buffer.byteLength(lastLine, "utf-8"))})`;
}
notice = `[Showing last ${formatSize(truncation.outputBytes)} of line ${endLine}${lastLineSizePart}${fullOutputPart}${suffix}]`;
} else if (truncation.truncatedBy === "lines") {
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}${fullOutputPart}${suffix}]`;
} else {
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatSize(truncation.maxBytes)} limit)${fullOutputPart}${suffix}]`;
}
return `\n\n${notice}`;
}
export interface HeadTruncationNoticeOptions {
/** 1-indexed start line number (default: 1) */
startLine?: number;
/** Total lines in the original file (for "of N" display) */
totalFileLines?: number;
}
/**
* Format a truncation notice for head-truncated output (read tool).
* Returns empty string if not truncated.
*
* Examples:
* - "[Showing lines 1-2000 of 5000. Use offset=2001 to continue]"
* - "[Showing lines 100-2099 of 5000 (50KB limit). Use offset=2100 to continue]"
*/
export function formatHeadTruncationNotice(
truncation: TruncationResult,
options: HeadTruncationNoticeOptions = {},
): string {
if (!truncation.truncated) {
return "";
}
const startLineDisplay = options.startLine ?? 1;
const totalFileLines = options.totalFileLines ?? truncation.totalLines;
const endLineDisplay = startLineDisplay + truncation.outputLines - 1;
const nextOffset = endLineDisplay + 1;
let notice: string;
if (truncation.truncatedBy === "lines") {
notice = `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`;
} else {
notice = `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines} (${formatSize(truncation.maxBytes)} limit). Use offset=${nextOffset} to continue]`;
}
return `\n\n${notice}`;
}
@@ -11,9 +11,9 @@ import type { AgentMessage } from "@oh-my-pi/pi-agent-core";
import { glob } from "@oh-my-pi/pi-natives";
import { formatHashLines } from "../patch/hashline";
import type { FileMentionMessage } from "../session/messages";
import { DEFAULT_MAX_BYTES, truncateHead, truncateHeadBytes } from "../session/streaming-output";
import { resolveReadPath } from "../tools/path-utils";
import { formatAge } from "../tools/render-utils";
import { DEFAULT_MAX_BYTES, formatSize, truncateHead, truncateStringToBytesFromStart } from "../tools/truncate";
import { formatAge, formatBytes } from "../tools/render-utils";
import { fuzzyMatch } from "./fuzzy";
import { formatDimensionNote, resizeImage } from "./image-resize";
import { detectSupportedImageMimeTypeFromFile } from "./mime";
@@ -162,15 +162,15 @@ function buildTextOutput(textContent: string): { output: string; lineCount: numb
if (truncation.firstLineExceedsLimit) {
const firstLine = allLines[0] ?? "";
const firstLineBytes = Buffer.byteLength(firstLine, "utf-8");
const snippet = truncateStringToBytesFromStart(firstLine, DEFAULT_MAX_BYTES);
const snippet = truncateHeadBytes(firstLine, DEFAULT_MAX_BYTES);
let outputText = snippet.text;
if (outputText.length > 0) {
outputText += `\n\n[Line 1 is ${formatSize(firstLineBytes)}, exceeds ${formatSize(
outputText += `\n\n[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes(
DEFAULT_MAX_BYTES,
)} limit. Showing first ${formatSize(snippet.bytes)} of the line.]`;
)} limit. Showing first ${formatBytes(snippet.bytes)} of the line.]`;
} else {
outputText = `[Line 1 is ${formatSize(firstLineBytes)}, exceeds ${formatSize(
outputText = `[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes(
DEFAULT_MAX_BYTES,
)} limit. Unable to display a valid UTF-8 snippet.]`;
}
@@ -187,7 +187,7 @@ function buildTextOutput(textContent: string): { output: string; lineCount: numb
if (truncation.truncatedBy === "lines") {
outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`;
} else {
outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines} (${formatSize(
outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines} (${formatBytes(
DEFAULT_MAX_BYTES,
)} limit). Use offset=${nextOffset} to continue]`;
}
@@ -247,7 +247,7 @@ async function buildDirectoryListing(absolutePath: string): Promise<{ output: st
notices.push(`${DEFAULT_DIR_LIMIT} entries limit reached. Use limit=${DEFAULT_DIR_LIMIT * 2} for more`);
}
if (truncation.truncated) {
notices.push(`${formatSize(DEFAULT_MAX_BYTES)} limit reached`);
notices.push(`${formatBytes(DEFAULT_MAX_BYTES)} limit reached`);
}
if (notices.length > 0) {
output += `\n\n[${notices.join(". ")}]`;
@@ -313,7 +313,7 @@ export async function generateFileMentionMessages(
if (stat.size > MAX_AUTO_READ_IMAGE_BYTES) {
files.push({
path: resolvedPath,
content: `(skipped auto-read: too large, ${formatSize(stat.size)})`,
content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`,
byteSize: stat.size,
skippedReason: "tooLarge",
});
@@ -349,7 +349,7 @@ export async function generateFileMentionMessages(
if (stat.size > MAX_AUTO_READ_TEXT_BYTES) {
files.push({
path: resolvedPath,
content: `(skipped auto-read: too large, ${formatSize(stat.size)})`,
content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`,
byteSize: stat.size,
skippedReason: "tooLarge",
});
@@ -1,3 +1,4 @@
import { formatBytes } from "../../tools/render-utils";
import type { RenderResult, SpecialHandler } from "./types";
import { finalizeOutput, formatCount, loadPage } from "./types";
@@ -29,13 +30,6 @@ interface DockerHubTagsResponse {
results?: DockerHubTag[];
}
function formatSize(bytes: number): string {
if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`;
if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`;
if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`;
return `${bytes}B`;
}
/**
* Handle Docker Hub URLs via API
*/
@@ -131,7 +125,7 @@ export const handleDockerHub: SpecialHandler = async (
md += "|-----|------|---------------|--------|\n";
for (const tag of tags) {
const size = tag.full_size ? formatSize(tag.full_size) : "-";
const size = tag.full_size ? formatBytes(tag.full_size) : "-";
const archs =
tag.images
?.map(img => img.architecture)
@@ -1,3 +1,4 @@
import { formatBytes } from "../../tools/render-utils";
import type { RenderResult, SpecialHandler } from "./types";
import { finalizeOutput, loadPage } from "./types";
@@ -100,13 +101,6 @@ function extractTagsFromHtml(html: string, baseRef: string): string[] {
return Array.from(tags);
}
function formatSize(bytes: number): string {
if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`;
if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`;
if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`;
return `${bytes}B`;
}
function buildModelPath(parts: string[]): string {
return parts.map(part => encodeURIComponent(part)).join("/");
}
@@ -227,11 +221,11 @@ export const handleOllama: SpecialHandler = async (
let sizeLine: string | null = null;
if (selectedTag?.size) {
sizeLine = formatSize(selectedTag.size);
sizeLine = formatBytes(selectedTag.size);
} else if (sizes.length > 0) {
const minSize = Math.min(...sizes);
const maxSize = Math.max(...sizes);
sizeLine = minSize === maxSize ? formatSize(minSize) : `${formatSize(minSize)} - ${formatSize(maxSize)}`;
sizeLine = minSize === maxSize ? formatBytes(minSize) : `${formatBytes(minSize)} - ${formatBytes(maxSize)}`;
}
let md = `# ${baseRef}\n\n`;
@@ -13,7 +13,7 @@ export interface ConvertResult {
}
export interface BinaryFetchResult {
buffer: Buffer;
buffer: Uint8Array;
contentType: string;
contentDisposition?: string;
ok: boolean;
@@ -22,7 +22,7 @@ export interface BinaryFetchResult {
}
export async function convertWithMarkitdown(
content: Buffer,
content: Uint8Array,
extensionHint: string,
timeout: number,
userSignal?: AbortSignal,
@@ -68,7 +68,7 @@ export async function convertWithMarkitdown(
}
}
const kEmptyBuffer = Buffer.alloc(0);
const kEmptyBuffer = new Uint8Array(0);
export async function fetchBinary(url: string, timeout: number, userSignal?: AbortSignal): Promise<BinaryFetchResult> {
if (userSignal?.aborted) {
@@ -115,7 +115,7 @@ export async function fetchBinary(url: string, timeout: number, userSignal?: Abo
}
}
const buffer = Buffer.from(await response.arrayBuffer());
const buffer = await response.bytes();
if (buffer.length > MAX_BYTES) {
return {
buffer: kEmptyBuffer,
@@ -4,7 +4,7 @@ import * as os from "node:os";
import * as path from "node:path";
import { _resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
import { executeBash } from "@oh-my-pi/pi-coding-agent/exec/bash-executor";
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate";
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output";
import * as shellSnapshot from "@oh-my-pi/pi-coding-agent/utils/shell-snapshot";
function makeTempDir(): string {
@@ -1,7 +1,7 @@
import { describe, expect, it } from "bun:test";
import { executePythonWithKernel, type PythonKernelExecutor } from "@oh-my-pi/pi-coding-agent/ipy/executor";
import type { KernelExecuteOptions, KernelExecuteResult } from "@oh-my-pi/pi-coding-agent/ipy/kernel";
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate";
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output";
class FakeKernel implements PythonKernelExecutor {
private result: KernelExecuteResult;
@@ -14,7 +14,7 @@ import {
type PreludeHelper,
PythonKernel,
} from "@oh-my-pi/pi-coding-agent/ipy/kernel";
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate";
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output";
import { TempDir } from "@oh-my-pi/pi-utils";
class FakeKernel implements PythonKernelExecutor {
@@ -51,6 +51,15 @@ describe("ModelRegistry runtime provider registration", () => {
const streamSimple: NonNullable<ProviderConfigInput["streamSimple"]> = () =>
({}) as unknown as AssistantMessageEventStream;
test("loads built-in GitLab Duo models and OAuth provider metadata", () => {
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("gitlab-duo", "claude-sonnet-4-5-20250929");
expect(model).toBeDefined();
expect(model?.api).toBe("anthropic-messages");
expect(getOAuthProviders().some(provider => provider.id === "gitlab-duo")).toBe(true);
});
test("validates provider config before mutating custom API state", () => {
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const beforeAnthropicCount = registry.getAll().filter(model => model.provider === "anthropic").length;
@@ -0,0 +1,353 @@
import { afterEach, describe, expect, test } from "bun:test";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { ArtifactManager } from "../src/session/artifacts";
import {
allocateOutputArtifact,
DEFAULT_MAX_BYTES,
DEFAULT_MAX_COLUMN,
DEFAULT_MAX_LINES,
formatHeadTruncationNotice,
formatTailTruncationNotice,
OutputSink,
TailBuffer,
truncateHead,
truncateHeadBytes,
truncateLine,
truncateTail,
truncateTailBytes,
} from "../src/session/streaming-output";
import type { ToolSession } from "../src/tools";
const createdTempDirs: string[] = [];
async function createTempDir(): Promise<string> {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "streaming-output-test-"));
createdTempDirs.push(dir);
return dir;
}
function byteLength(text: string): number {
return Buffer.byteLength(text, "utf-8");
}
function createSession(overrides: Partial<ToolSession> = {}): ToolSession {
return {
cwd: "/",
hasUI: false,
getSessionFile: () => null,
getSessionSpawns: () => null,
settings: {} as ToolSession["settings"],
...overrides,
};
}
afterEach(async () => {
for (const dir of createdTempDirs.splice(0)) {
await fs.rm(dir, { recursive: true, force: true });
}
});
describe("streaming-output exports", () => {
test("exports expected default limits", () => {
expect(DEFAULT_MAX_LINES).toBe(3000);
expect(DEFAULT_MAX_BYTES).toBe(50 * 1024);
expect(DEFAULT_MAX_COLUMN).toBe(1024);
});
});
describe("truncateTailBytes", () => {
test("returns source when already under limit", () => {
const text = "hello";
expect(truncateTailBytes(text, 10)).toEqual({ text: "hello", bytes: 5 });
});
test("truncates from end without breaking UTF-8 boundaries", () => {
const text = "a😀b";
const result = truncateTailBytes(text, 4);
expect(result).toEqual({ text: "b", bytes: 1 });
expect(result.text).not.toContain("\uFFFD");
});
test("accepts Uint8Array input", () => {
const bytes = new TextEncoder().encode("abc😀");
const result = truncateTailBytes(bytes, 4);
expect(result.text).toBe("😀");
expect(result.bytes).toBe(4);
});
});
describe("truncateHeadBytes", () => {
test("returns source when already under limit", () => {
const text = "hello";
expect(truncateHeadBytes(text, 10)).toEqual({ text: "hello", bytes: 5 });
});
test("truncates from start without breaking UTF-8 boundaries", () => {
const text = "a😀b";
const result = truncateHeadBytes(text, 2);
expect(result).toEqual({ text: "a", bytes: 1 });
expect(result.text).not.toContain("\uFFFD");
});
test("returns empty when maxBytes is zero", () => {
const result = truncateHeadBytes("abc", 0);
expect(result).toEqual({ text: "", bytes: 0 });
});
});
describe("truncateHead", () => {
test("returns unmodified content when within limits", () => {
const content = "a\nb";
const result = truncateHead(content, { maxLines: 10, maxBytes: 20 });
expect(result.truncated).toBe(false);
expect(result.content).toBe(content);
expect(result.truncatedBy).toBe(null);
});
test("handles first line exceeding byte limit", () => {
const result = truncateHead("abcdef\nnext", { maxBytes: 3, maxLines: 10 });
expect(result.content).toBe("");
expect(result.truncated).toBe(true);
expect(result.truncatedBy).toBe("bytes");
expect(result.firstLineExceedsLimit).toBe(true);
});
test("includes first line when text fits exact byte budget", () => {
const result = truncateHead("abc\nx", { maxBytes: 3, maxLines: 10 });
expect(result.content).toBe("abc");
expect(result.truncated).toBe(true);
expect(result.truncatedBy).toBe("bytes");
expect(result.firstLineExceedsLimit).toBe(false);
expect(result.outputBytes).toBe(byteLength("abc"));
});
test("truncates by line count", () => {
const result = truncateHead("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
expect(result.content).toBe("l1\nl2");
expect(result.truncatedBy).toBe("lines");
expect(result.outputLines).toBe(2);
});
test("truncates by byte budget using complete lines", () => {
const result = truncateHead("12345\nabc\nz", { maxLines: 10, maxBytes: 7 });
expect(result.content).toBe("12345");
expect(result.truncatedBy).toBe("bytes");
expect(result.lastLinePartial).toBe(false);
expect(result.outputBytes).toBe(byteLength("12345"));
});
});
describe("truncateTail", () => {
test("returns unmodified content when within limits", () => {
const content = "a\nb";
const result = truncateTail(content, { maxLines: 10, maxBytes: 20 });
expect(result.truncated).toBe(false);
expect(result.content).toBe(content);
expect(result.truncatedBy).toBe(null);
});
test("truncates by line count", () => {
const result = truncateTail("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
expect(result.content).toBe("l2\nl3");
expect(result.truncatedBy).toBe("lines");
expect(result.outputLines).toBe(2);
});
test("truncates by byte budget while preserving line boundaries", () => {
const result = truncateTail("aaa\nbbbb\ncc", { maxLines: 10, maxBytes: 6 });
expect(result.content).toBe("cc");
expect(result.truncatedBy).toBe("bytes");
expect(result.lastLinePartial).toBe(false);
});
test("returns partial single line when last line exceeds byte limit", () => {
const result = truncateTail("abcdefghij", { maxLines: 10, maxBytes: 4 });
expect(result.content).toBe("ghij");
expect(result.truncatedBy).toBe("bytes");
expect(result.lastLinePartial).toBe(true);
});
});
describe("truncateLine", () => {
test("does not truncate short lines", () => {
expect(truncateLine("hello", 10)).toEqual({ text: "hello", wasTruncated: false });
});
test("truncates long lines with ellipsis", () => {
expect(truncateLine("abcdefgh", 5)).toEqual({ text: "abcde…", wasTruncated: true });
});
});
describe("TailBuffer", () => {
test("keeps trailing bytes under budget", () => {
const tail = new TailBuffer(5);
tail.append("abc");
tail.append("def");
expect(tail.text()).toBe("bcdef");
expect(tail.bytes()).toBe(5);
});
test("handles multibyte data and empty appends", () => {
const tail = new TailBuffer(4);
tail.append("");
tail.append("😀");
tail.append("x");
expect(tail.text()).toBe("x");
expect(tail.bytes()).toBe(1);
});
});
describe("OutputSink", () => {
test("tracks totals and adds notice in dump", async () => {
const sink = new OutputSink();
await sink.push("hello\nworld");
const dumped = await sink.dump("notice");
expect(dumped.output).toBe("[notice]\nhello\nworld");
expect(dumped.truncated).toBe(false);
expect(dumped.totalLines).toBe(2);
expect(dumped.totalBytes).toBe(byteLength("hello\nworld"));
expect(dumped.outputLines).toBe(2);
expect(dumped.outputBytes).toBe(byteLength("hello\nworld"));
});
test("counts lines correctly when chunks contain no newlines", async () => {
const sink = new OutputSink();
await sink.push("abc");
await sink.push("def");
const dumped = await sink.dump();
expect(dumped.totalLines).toBe(1);
expect(dumped.outputLines).toBe(1);
});
test("counts all newline boundaries across chunk splits", async () => {
const sink = new OutputSink();
await sink.push("a\n");
await sink.push("b\n\n");
await sink.push("c");
const dumped = await sink.dump();
expect(dumped.output).toBe("a\nb\n\nc");
expect(dumped.totalLines).toBe(4);
expect(dumped.outputLines).toBe(4);
});
test("invokes onChunk callback with sanitized text", async () => {
const chunks: string[] = [];
const sink = new OutputSink({ onChunk: chunk => chunks.push(chunk) });
await sink.push("abc");
await sink.push("def");
expect(chunks).toEqual(["abc", "def"]);
});
test("truncates in-memory output when spill threshold is exceeded", async () => {
const sink = new OutputSink({ spillThreshold: 5 });
await sink.push("abc");
await sink.push("def");
const dumped = await sink.dump();
expect(dumped.truncated).toBe(true);
expect(dumped.output).toBe("bcdef");
expect(dumped.totalBytes).toBe(6);
expect(dumped.outputBytes).toBe(5);
});
test("spills full output to artifact file when artifact path is provided", async () => {
const dir = await createTempDir();
const artifactPath = path.join(dir, "output.log");
const sink = new OutputSink({
artifactPath,
artifactId: "artifact-1",
spillThreshold: 5,
});
await sink.push("abc");
await sink.push("def");
const dumped = await sink.dump();
const artifactText = await Bun.file(artifactPath).text();
expect(dumped.truncated).toBe(true);
expect(dumped.artifactId).toBe("artifact-1");
expect(artifactText).toBe("abcdef");
expect(dumped.output).toBe("bcdef");
});
test("createInput decodes streamed UTF-8 chunks correctly", async () => {
const sink = new OutputSink();
const writer = sink.createInput().getWriter();
const bytes = new TextEncoder().encode("😀X");
await writer.write(bytes.subarray(0, 2));
await writer.write(bytes.subarray(2));
await writer.close();
const dumped = await sink.dump();
expect(dumped.output).toBe("😀X");
expect(dumped.totalBytes).toBe(byteLength("😀X"));
});
});
describe("allocateOutputArtifact", () => {
test("returns empty object when session has no artifact manager", async () => {
const session = createSession({ getArtifactManager: undefined });
expect(await allocateOutputArtifact(session, "bash")).toEqual({});
});
test("allocates artifact path via session artifact manager", async () => {
const dir = await createTempDir();
const sessionFile = path.join(dir, "session.jsonl");
await Bun.write(sessionFile, "");
const manager = new ArtifactManager(sessionFile);
const session = createSession({ getArtifactManager: () => manager });
const result = await allocateOutputArtifact(session, "bash");
expect(result.id).toBeDefined();
expect(result.path).toContain(`${path.sep}${result.id}.bash.log`);
});
});
describe("truncation notice formatting", () => {
test("formatTailTruncationNotice returns empty string for non-truncated results", () => {
const truncation = truncateTail("a\nb", { maxLines: 10, maxBytes: 50 });
expect(formatTailTruncationNotice(truncation)).toBe("");
});
test("formatTailTruncationNotice supports partial-line notices", () => {
const truncation = truncateTail("abcdefghij", { maxLines: 10, maxBytes: 4 });
const notice = formatTailTruncationNotice(truncation, {
fullOutputPath: "/tmp/full.log",
originalContent: "abcdefghij",
suffix: " [suffix]",
});
expect(notice).toBe("\n\n[Showing last 4B of line 1 (line is 10B). Full output: /tmp/full.log [suffix]]");
});
test("formatTailTruncationNotice supports line-based and byte-based notices", () => {
const lineTruncation = truncateTail("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
expect(formatTailTruncationNotice(lineTruncation)).toBe("\n\n[Showing lines 2-3 of 3]");
const byteTruncation = truncateTail("aaa\nbbbb\ncc", { maxLines: 10, maxBytes: 6 });
expect(formatTailTruncationNotice(byteTruncation)).toBe("\n\n[Showing lines 3-3 of 3 (6B limit)]");
});
test("formatHeadTruncationNotice returns empty string for non-truncated results", () => {
const truncation = truncateHead("a\nb", { maxLines: 10, maxBytes: 50 });
expect(formatHeadTruncationNotice(truncation)).toBe("");
});
test("formatHeadTruncationNotice supports line and byte truncation", () => {
const lineTruncation = truncateHead("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
expect(formatHeadTruncationNotice(lineTruncation)).toBe("\n\n[Showing lines 1-2 of 3. Use offset=3 to continue]");
const byteTruncation = truncateHead("12345\nabc\nz", { maxLines: 10, maxBytes: 7 });
expect(
formatHeadTruncationNotice(byteTruncation, {
startLine: 100,
totalFileLines: 500,
}),
).toBe("\n\n[Showing lines 100-100 of 500 (7B limit). Use offset=101 to continue]");
});
});
+3
View File
@@ -4,6 +4,7 @@ import * as os from "node:os";
import * as path from "node:path";
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
import { EditTool } from "@oh-my-pi/pi-coding-agent/patch";
import { ArtifactManager } from "@oh-my-pi/pi-coding-agent/session/artifacts";
import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools";
import { BashTool } from "@oh-my-pi/pi-coding-agent/tools/bash";
import { FindTool } from "@oh-my-pi/pi-coding-agent/tools/find";
@@ -25,11 +26,13 @@ function getTextOutput(result: any): string {
function createTestToolSession(cwd: string): ToolSession {
const sessionFile = path.join(cwd, "session.jsonl");
const artifactManager = new ArtifactManager(sessionFile);
return {
cwd,
hasUI: false,
getSessionFile: () => sessionFile,
getSessionSpawns: () => "*",
getArtifactManager: () => artifactManager,
settings: Settings.isolated(),
};
}