refactor(coding-agent): consolidated output utilities into streaming-output module
- Consolidated truncation and output utilities from tools/truncate.ts and tools/output-utils.ts into session/streaming-output.ts with improved UTF-8 boundary handling. - Renamed formatSize() to formatBytes() across codebase for consistency and clarity in byte-level formatting. - Refactored OutputSink to use windowed byte truncation instead of full-buffer encoding, improving memory efficiency on large outputs. - Migrated from Buffer to Uint8Array in web scrapers for better cross-platform compatibility and native browser support. - Added getArtifactManager() lazy-initialization method to ToolSession for deferred artifact manager instantiation. - Simplified API surface with wildcard exports from tools and session modules, reducing import complexity.
This commit is contained in:
@@ -1,6 +1,37 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
|
||||
- Exported truncation utilities and streaming output types from `session/streaming-output` module for public use
|
||||
- Added `TailBuffer` class for efficient ring-style buffering with lazy joining and windowed truncation
|
||||
- Added `truncateTailBytes` and `truncateHeadBytes` functions for UTF-8-aware byte-level truncation
|
||||
- Added `formatTailTruncationNotice` and `formatHeadTruncationNotice` functions for consistent truncation notice formatting
|
||||
- Added `allocateOutputArtifact` function to allocate artifact paths for tool output spilling
|
||||
- Added `getArtifactManager()` method to `ToolSession` for lazy artifact manager access
|
||||
|
||||
### Changed
|
||||
|
||||
- Moved truncation logic from `tools/truncate.ts` to `session/streaming-output.ts` for better architectural separation
|
||||
- Renamed `formatSize()` to `formatBytes()` across codebase for consistency
|
||||
- Refactored `OutputSink` to use windowed byte truncation for memory efficiency when spilling to files
|
||||
- Changed `ToolSession.artifactManager` from cached property to `getArtifactManager()` method for lazy initialization
|
||||
- Updated `allocateOutputArtifact` return type to use `{ path, id }` instead of `{ artifactPath, artifactId }`
|
||||
- Optimized newline counting to use native `indexOf` for V8-optimized string scanning
|
||||
- Improved UTF-8 boundary detection in byte truncation with dedicated helper functions
|
||||
|
||||
### Removed
|
||||
|
||||
- Deleted `tools/truncate.ts` module (functionality moved to `session/streaming-output.ts`)
|
||||
- Deleted `tools/output-utils.ts` module (functionality moved to `session/streaming-output.ts`)
|
||||
- Removed `formatSize` export from public API (renamed to `formatBytes`)
|
||||
- Removed individual truncation function exports from `tools/index.ts` (now exported via `session/streaming-output`)
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed UTF-8 boundary handling in byte truncation to prevent invalid character sequences
|
||||
- Fixed memory efficiency in `OutputSink` by using windowed truncation instead of full-buffer encoding
|
||||
- Fixed line counting to handle chunk boundaries correctly across multiple `push()` calls
|
||||
|
||||
## [12.18.1] - 2026-02-21
|
||||
### Added
|
||||
|
||||
@@ -424,7 +424,7 @@ Key meta blocks:
|
||||
`formatOutputNotice(meta)` converts metadata into appended textual notices (for model visibility), including:
|
||||
|
||||
- shown line range and total line count
|
||||
- byte-limit context via `formatSize(...)`
|
||||
- byte-limit context via `formatBytes(...)`
|
||||
- pagination hint (`nextOffset`) for head-truncated output
|
||||
- artifact recovery hint (`Full: artifact://<id>`)
|
||||
- limit and diagnostics notices
|
||||
|
||||
@@ -8,7 +8,7 @@ import { isEnoent } from "@oh-my-pi/pi-utils";
|
||||
import { getProjectDir } from "@oh-my-pi/pi-utils/dirs";
|
||||
import chalk from "chalk";
|
||||
import { resolveReadPath } from "../tools/path-utils";
|
||||
import { formatSize } from "../tools/truncate";
|
||||
import { formatBytes } from "../tools/render-utils";
|
||||
import { formatDimensionNote, resizeImage } from "../utils/image-resize";
|
||||
import { detectSupportedImageMimeTypeFromFile } from "../utils/mime";
|
||||
|
||||
@@ -47,9 +47,9 @@ export async function processFileArguments(fileArgs: string[], options?: Process
|
||||
const maxBytes = mimeType ? MAX_CLI_IMAGE_BYTES : MAX_CLI_TEXT_BYTES;
|
||||
if (stat.size > maxBytes) {
|
||||
console.error(
|
||||
chalk.yellow(`Warning: Skipping file contents (too large: ${formatSize(stat.size)}): ${absolutePath}`),
|
||||
chalk.yellow(`Warning: Skipping file contents (too large: ${formatBytes(stat.size)}): ${absolutePath}`),
|
||||
);
|
||||
text += `<file name="${absolutePath}">(skipped: too large, ${formatSize(stat.size)})</file>\n`;
|
||||
text += `<file name="${absolutePath}">(skipped: too large, ${formatBytes(stat.size)})</file>\n`;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import { getSessionsDir } from "@oh-my-pi/pi-utils/dirs";
|
||||
import { DynamicBorder } from "../modes/components/dynamic-border";
|
||||
import { getSelectListTheme, getSymbolTheme, theme } from "../modes/theme/theme";
|
||||
import type { InteractiveModeContext } from "../modes/types";
|
||||
import { formatBytes } from "../tools/render-utils";
|
||||
import { openPath } from "../utils/open";
|
||||
import { DebugLogViewerComponent } from "./log-viewer";
|
||||
import { generateHeapSnapshotData, type ProfilerSession, startCpuProfile } from "./profiler";
|
||||
@@ -417,12 +418,6 @@ export class DebugSelectorComponent extends Container {
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Show the debug selector.
|
||||
*/
|
||||
|
||||
@@ -243,27 +243,4 @@ export {
|
||||
export { runSubprocess } from "./task/executor";
|
||||
export type { AgentDefinition, AgentProgress, AgentSource, SingleResult, TaskParams } from "./task/types";
|
||||
// Tools (detail types and utilities)
|
||||
export {
|
||||
type BashToolDetails,
|
||||
type BashToolInput,
|
||||
type BrowserToolDetails,
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
type FindOperations,
|
||||
type FindToolDetails,
|
||||
type FindToolInput,
|
||||
type FindToolOptions,
|
||||
formatSize,
|
||||
type GrepToolDetails,
|
||||
type GrepToolInput,
|
||||
type PythonToolDetails,
|
||||
type ReadToolDetails,
|
||||
type ReadToolInput,
|
||||
type TruncationOptions,
|
||||
type TruncationResult,
|
||||
truncateHead,
|
||||
truncateLine,
|
||||
truncateTail,
|
||||
type WriteToolDetails,
|
||||
type WriteToolInput,
|
||||
} from "./tools";
|
||||
export * from "./tools";
|
||||
|
||||
@@ -6,7 +6,7 @@ import { sanitizeText } from "@oh-my-pi/pi-natives";
|
||||
import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui";
|
||||
import { getSymbolTheme, theme } from "../../modes/theme/theme";
|
||||
import type { TruncationMeta } from "../../tools/output-meta";
|
||||
import { formatSize } from "../../tools/truncate";
|
||||
import { formatBytes } from "../../tools/render-utils";
|
||||
import { DynamicBorder } from "./dynamic-border";
|
||||
import { truncateToVisualLines } from "./visual-truncate";
|
||||
|
||||
@@ -177,7 +177,7 @@ export class BashExecutionComponent extends Container {
|
||||
);
|
||||
} else {
|
||||
warnings.push(
|
||||
`Truncated: ${this.#truncation.outputLines} lines shown (${formatSize(this.#truncation.outputBytes)} limit)`,
|
||||
`Truncated: ${this.#truncation.outputLines} lines shown (${formatBytes(this.#truncation.outputBytes)} limit)`,
|
||||
);
|
||||
}
|
||||
statusParts.push(theme.fg("warning", warnings.join(". ")));
|
||||
|
||||
@@ -7,7 +7,7 @@ import { sanitizeText } from "@oh-my-pi/pi-natives";
|
||||
import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui";
|
||||
import { getSymbolTheme, highlightCode, theme } from "../../modes/theme/theme";
|
||||
import type { TruncationMeta } from "../../tools/output-meta";
|
||||
import { formatSize } from "../../tools/truncate";
|
||||
import { formatBytes } from "../../tools/render-utils";
|
||||
import { DynamicBorder } from "./dynamic-border";
|
||||
import { truncateToVisualLines } from "./visual-truncate";
|
||||
|
||||
@@ -161,7 +161,7 @@ export class PythonExecutionComponent extends Container {
|
||||
);
|
||||
} else {
|
||||
warnings.push(
|
||||
`Truncated: ${this.#truncation.outputLines} lines shown (${formatSize(this.#truncation.outputBytes)} limit)`,
|
||||
`Truncated: ${this.#truncation.outputLines} lines shown (${formatBytes(this.#truncation.outputBytes)} limit)`,
|
||||
);
|
||||
}
|
||||
statusParts.push(theme.fg("warning", warnings.join(". ")));
|
||||
|
||||
@@ -17,7 +17,7 @@ import { theme } from "../../modes/theme/theme";
|
||||
import type { CompactionQueuedMessage, InteractiveModeContext } from "../../modes/types";
|
||||
import { type CustomMessage, SKILL_PROMPT_MESSAGE_TYPE, type SkillPromptDetails } from "../../session/messages";
|
||||
import type { SessionContext } from "../../session/session-manager";
|
||||
import { formatSize } from "../../tools/truncate";
|
||||
import { formatBytes } from "../../tools/render-utils";
|
||||
|
||||
type TextBlock = { type: "text"; text: string };
|
||||
|
||||
@@ -130,7 +130,7 @@ export class UiHelpers {
|
||||
for (const file of message.files) {
|
||||
let suffix: string;
|
||||
if (file.skippedReason === "tooLarge") {
|
||||
const size = typeof file.byteSize === "number" ? formatSize(file.byteSize) : "unknown size";
|
||||
const size = typeof file.byteSize === "number" ? formatBytes(file.byteSize) : "unknown size";
|
||||
suffix = `(skipped: ${size})`;
|
||||
} else {
|
||||
suffix = file.image
|
||||
|
||||
@@ -13,6 +13,7 @@ import { loadPromptTemplates as loadPromptTemplatesInternal, type PromptTemplate
|
||||
import { Settings, type SkillsSettings } from "./config/settings";
|
||||
import { CursorExecHandlers } from "./cursor";
|
||||
import "./discovery";
|
||||
import { ArtifactManager } from "@oh-my-pi/pi-coding-agent/session/artifacts";
|
||||
import { initializeWithSettings } from "./discovery";
|
||||
import { TtsrManager } from "./export/ttsr";
|
||||
import {
|
||||
@@ -715,6 +716,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
||||
|
||||
const enableLsp = options.enableLsp ?? true;
|
||||
|
||||
let artifactManager: ArtifactManager | null = null;
|
||||
const toolSession: ToolSession = {
|
||||
cwd,
|
||||
hasUI: options.hasUI ?? false,
|
||||
@@ -742,6 +744,18 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
||||
},
|
||||
getPlanModeState: () => session.getPlanModeState(),
|
||||
getCompactContext: () => session.formatCompactContext(),
|
||||
getArtifactManager: () => {
|
||||
if (artifactManager) {
|
||||
return artifactManager;
|
||||
}
|
||||
const sessionFile = sessionManager.getSessionFile();
|
||||
if (!sessionFile) {
|
||||
return null;
|
||||
}
|
||||
const manager = new ArtifactManager(sessionFile);
|
||||
artifactManager = manager;
|
||||
return manager;
|
||||
},
|
||||
settings,
|
||||
authStorage,
|
||||
modelRegistry,
|
||||
|
||||
@@ -1,5 +1,18 @@
|
||||
import { sanitizeText } from "@oh-my-pi/pi-natives";
|
||||
import { DEFAULT_MAX_BYTES } from "../tools/truncate";
|
||||
import type { ToolSession } from "../tools";
|
||||
import { formatBytes } from "../tools/render-utils";
|
||||
|
||||
// =============================================================================
|
||||
// Constants
|
||||
// =============================================================================
|
||||
|
||||
export const DEFAULT_MAX_LINES = 3000;
|
||||
export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB
|
||||
export const DEFAULT_MAX_COLUMN = 1024; // Max chars per grep match line
|
||||
|
||||
// =============================================================================
|
||||
// Interfaces
|
||||
// =============================================================================
|
||||
|
||||
export interface OutputSummary {
|
||||
output: string;
|
||||
@@ -19,40 +32,438 @@ export interface OutputSinkOptions {
|
||||
onChunk?: (chunk: string) => void;
|
||||
}
|
||||
|
||||
export interface TruncationResult {
|
||||
content: string;
|
||||
truncated: boolean;
|
||||
truncatedBy: "lines" | "bytes" | null;
|
||||
totalLines: number;
|
||||
totalBytes: number;
|
||||
outputLines: number;
|
||||
outputBytes: number;
|
||||
lastLinePartial: boolean;
|
||||
firstLineExceedsLimit: boolean;
|
||||
maxLines: number;
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
export interface TruncationOptions {
|
||||
/** Maximum number of lines (default: 3000) */
|
||||
maxLines?: number;
|
||||
/** Maximum number of bytes (default: 50KB) */
|
||||
maxBytes?: number;
|
||||
}
|
||||
|
||||
/** Result from byte-level truncation helpers. */
|
||||
export interface ByteTruncationResult {
|
||||
text: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface TailTruncationNoticeOptions {
|
||||
fullOutputPath?: string;
|
||||
originalContent?: string;
|
||||
suffix?: string;
|
||||
}
|
||||
|
||||
export interface HeadTruncationNoticeOptions {
|
||||
startLine?: number;
|
||||
totalFileLines?: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Low-level byte utilities
|
||||
//
|
||||
// Both use a "windowed encoding" strategy for strings: instead of encoding the
|
||||
// entire input with Buffer.from(), we encode a window of at most `maxBytes`
|
||||
// code units from the relevant end. Every JS code unit produces ≥1 UTF-8 byte,
|
||||
// so this window is guaranteed to contain ≥ maxBytes encoded bytes (unless the
|
||||
// input is shorter), avoiding O(N) encoding cost on large strings.
|
||||
// =============================================================================
|
||||
|
||||
/** Advance past UTF-8 continuation bytes (10xxxxxx) to a leading byte. */
|
||||
function findUtf8BoundaryForward(buf: Buffer, pos: number): number {
|
||||
while (pos < buf.length && (buf[pos] & 0xc0) === 0x80) pos++;
|
||||
return pos;
|
||||
}
|
||||
|
||||
/** Retreat past UTF-8 continuation bytes to land on a leading byte. */
|
||||
function findUtf8BoundaryBackward(buf: Buffer, pos: number): number {
|
||||
while (pos > 0 && (buf[pos] & 0xc0) === 0x80) pos--;
|
||||
return pos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate a string/buffer to fit within a byte limit, keeping the tail.
|
||||
* Handles multi-byte UTF-8 boundaries correctly.
|
||||
*/
|
||||
export function truncateTailBytes(data: string | Uint8Array, maxBytes: number): ByteTruncationResult {
|
||||
if (typeof data === "string") {
|
||||
const len = Buffer.byteLength(data, "utf-8");
|
||||
if (len <= maxBytes) return { text: data, bytes: len };
|
||||
|
||||
// Windowed: encode only the last `maxBytes` code units (≥ maxBytes bytes).
|
||||
const window = data.substring(Math.max(0, data.length - maxBytes));
|
||||
const buf = Buffer.from(window, "utf-8");
|
||||
const start = findUtf8BoundaryForward(buf, buf.length - maxBytes);
|
||||
const slice = buf.subarray(start);
|
||||
return { text: slice.toString("utf-8"), bytes: slice.length };
|
||||
}
|
||||
|
||||
// Uint8Array / Buffer path
|
||||
if (data.length <= maxBytes) return { text: Buffer.from(data).toString("utf-8"), bytes: data.length };
|
||||
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
||||
const start = findUtf8BoundaryForward(buf, buf.length - maxBytes);
|
||||
const slice = buf.subarray(start);
|
||||
return { text: slice.toString("utf-8"), bytes: slice.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate a string/buffer to fit within a byte limit, keeping the head.
|
||||
* Handles multi-byte UTF-8 boundaries correctly.
|
||||
*/
|
||||
export function truncateHeadBytes(data: string | Uint8Array, maxBytes: number): ByteTruncationResult {
|
||||
if (typeof data === "string") {
|
||||
const len = Buffer.byteLength(data, "utf-8");
|
||||
if (len <= maxBytes) return { text: data, bytes: len };
|
||||
|
||||
// Windowed: encode only the first `maxBytes` code units.
|
||||
const window = data.substring(0, maxBytes);
|
||||
const buf = Buffer.from(window, "utf-8");
|
||||
const end = findUtf8BoundaryBackward(buf, maxBytes);
|
||||
if (end <= 0) return { text: "", bytes: 0 };
|
||||
const slice = buf.subarray(0, end);
|
||||
return { text: slice.toString("utf-8"), bytes: slice.length };
|
||||
}
|
||||
|
||||
if (data.length <= maxBytes) return { text: Buffer.from(data).toString("utf-8"), bytes: data.length };
|
||||
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
||||
const end = findUtf8BoundaryBackward(buf, maxBytes);
|
||||
if (end <= 0) return { text: "", bytes: 0 };
|
||||
const slice = buf.subarray(0, end);
|
||||
return { text: slice.toString("utf-8"), bytes: slice.length };
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Line-level utilities
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Count newline characters. Uses indexOf for V8-optimized native string scanning.
|
||||
*/
|
||||
function countNewlines(text: string): number {
|
||||
let count = 0;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if (text.charCodeAt(i) === 10) count += 1;
|
||||
let pos = text.indexOf("\n", 0);
|
||||
while (pos !== -1) {
|
||||
count++;
|
||||
pos = text.indexOf("\n", pos + 1);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function countLines(text: string): number {
|
||||
if (text.length === 0) return 0;
|
||||
return countNewlines(text) + 1;
|
||||
/**
|
||||
* Truncate a single line to max characters, appending '…' if truncated.
|
||||
*/
|
||||
export function truncateLine(
|
||||
line: string,
|
||||
maxChars: number = DEFAULT_MAX_COLUMN,
|
||||
): { text: string; wasTruncated: boolean } {
|
||||
if (line.length <= maxChars) return { text: line, wasTruncated: false };
|
||||
return { text: `${line.slice(0, maxChars)}…`, wasTruncated: true };
|
||||
}
|
||||
|
||||
function truncateStringToBytesFromEnd(text: string, maxBytes: number): { text: string; bytes: number } {
|
||||
const buf = Buffer.from(text, "utf-8");
|
||||
if (buf.length <= maxBytes) {
|
||||
return { text, bytes: buf.length };
|
||||
}
|
||||
// =============================================================================
|
||||
// Content truncation (line + byte aware)
|
||||
//
|
||||
// Both truncateHead and truncateTail encode the content to a Buffer once and
|
||||
// collect newline byte-offsets in a single forward pass, avoiding split("\n")
|
||||
// which would allocate N intermediate strings for N lines.
|
||||
// =============================================================================
|
||||
|
||||
let start = buf.length - maxBytes;
|
||||
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
|
||||
start++;
|
||||
}
|
||||
|
||||
const sliced = buf.subarray(start).toString("utf-8");
|
||||
return { text: sliced, bytes: Buffer.byteLength(sliced, "utf-8") };
|
||||
/** Shared helper to build a no-truncation result. */
|
||||
function noTruncResult(
|
||||
content: string,
|
||||
totalLines: number,
|
||||
totalBytes: number,
|
||||
maxLines: number,
|
||||
maxBytes: number,
|
||||
): TruncationResult {
|
||||
return {
|
||||
content,
|
||||
truncated: false,
|
||||
truncatedBy: null,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: totalLines,
|
||||
outputBytes: totalBytes,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Line-buffered output sink with file spill support.
|
||||
* Collect byte-offsets of every 0x0a in a Buffer. Avoids re-scanning.
|
||||
*
|
||||
* Uses a single string buffer with line position tracking.
|
||||
* When memory limit exceeded, spills ~half to file in one batch operation.
|
||||
* For a buffer with lines [L0 \n L1 \n L2]:
|
||||
* nlOffsets = [offset_of_first_\n, offset_of_second_\n]
|
||||
* Line k starts at (k === 0 ? 0 : nlOffsets[k-1] + 1)
|
||||
* Line k ends at (k < nlOffsets.length ? nlOffsets[k] - 1 : buf.length - 1)
|
||||
*/
|
||||
function collectNewlineOffsets(buf: Buffer): number[] {
|
||||
const offsets: number[] = [];
|
||||
let pos = Bun.indexOfLine(buf, 0);
|
||||
while (pos !== -1) {
|
||||
offsets.push(pos);
|
||||
pos = Bun.indexOfLine(buf, pos + 1);
|
||||
}
|
||||
return offsets;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate content from the head (keep first N lines/bytes).
|
||||
* Suitable for file reads where you want to see the beginning.
|
||||
*
|
||||
* Uses indexOfLine to scan forward incrementally — stops as soon as the
|
||||
* line or byte limit is hit without ever collecting all newline positions.
|
||||
*
|
||||
* Never returns partial lines. If the first line exceeds the byte limit,
|
||||
* returns empty content with firstLineExceedsLimit=true.
|
||||
*/
|
||||
export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult {
|
||||
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
const totalBytes = Buffer.byteLength(content, "utf-8");
|
||||
|
||||
// Fast path: if under byte limit, only need a string-level newline count
|
||||
if (totalBytes <= maxBytes) {
|
||||
const totalLines = countNewlines(content) + 1;
|
||||
if (totalLines <= maxLines) {
|
||||
return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes);
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: encode once, scan with native indexOfLine
|
||||
const totalLines = countNewlines(content) + 1;
|
||||
if (totalLines <= maxLines && totalBytes <= maxBytes) {
|
||||
return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes);
|
||||
}
|
||||
|
||||
// Forward scan: include complete lines that fit within both limits.
|
||||
// Uses indexOfLine incrementally — no offset array needed.
|
||||
let includedLines = 0;
|
||||
let cutByte = 0;
|
||||
let truncatedBy: "lines" | "bytes" = "lines";
|
||||
let scanPos = 0;
|
||||
|
||||
const buf = Buffer.from(content, "utf-8");
|
||||
while (includedLines < maxLines) {
|
||||
const nlPos = Bun.indexOfLine(buf, scanPos);
|
||||
// Byte position right after this line's trailing \n, or end-of-buffer
|
||||
const lineEnd = nlPos === -1 ? buf.length : nlPos + 1;
|
||||
// Output strips a trailing \n from the final slice; enforce limit on displayed bytes
|
||||
const outputEnd = nlPos === -1 ? lineEnd : lineEnd - 1;
|
||||
|
||||
if (outputEnd > maxBytes) {
|
||||
truncatedBy = "bytes";
|
||||
if (includedLines === 0) {
|
||||
return {
|
||||
content: "",
|
||||
truncated: true,
|
||||
truncatedBy: "bytes",
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: 0,
|
||||
outputBytes: 0,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: true,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
cutByte = lineEnd;
|
||||
includedLines++;
|
||||
|
||||
if (nlPos === -1) break; // no more lines
|
||||
scanPos = nlPos + 1;
|
||||
}
|
||||
|
||||
if (includedLines >= maxLines && cutByte <= maxBytes) {
|
||||
truncatedBy = "lines";
|
||||
}
|
||||
|
||||
// Strip trailing \n so output matches join("\n") semantics
|
||||
const sliceEnd = cutByte > 0 && buf[cutByte - 1] === 0x0a ? cutByte - 1 : cutByte;
|
||||
const outputContent = buf.subarray(0, sliceEnd).toString("utf-8");
|
||||
|
||||
return {
|
||||
content: outputContent,
|
||||
truncated: true,
|
||||
truncatedBy,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: includedLines,
|
||||
outputBytes: sliceEnd,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate content from the tail (keep last N lines/bytes).
|
||||
* Suitable for bash output where you want to see the end (errors, final results).
|
||||
*
|
||||
* May return a partial first line if the last line exceeds the byte limit.
|
||||
*/
|
||||
export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult {
|
||||
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
const totalBytes = Buffer.byteLength(content, "utf-8");
|
||||
const totalLines = countNewlines(content) + 1;
|
||||
|
||||
if (totalLines <= maxLines && totalBytes <= maxBytes) {
|
||||
return noTruncResult(content, totalLines, totalBytes, maxLines, maxBytes);
|
||||
}
|
||||
|
||||
const buf = Buffer.from(content, "utf-8");
|
||||
const nlOffsets = collectNewlineOffsets(buf);
|
||||
const lineCount = nlOffsets.length + 1;
|
||||
|
||||
// Walk backward, accumulating complete lines that fit.
|
||||
let includedLines = 0;
|
||||
let startByte = buf.length;
|
||||
let truncatedBy: "lines" | "bytes" = "lines";
|
||||
|
||||
for (let lineIdx = lineCount - 1; lineIdx >= 0 && includedLines < maxLines; lineIdx--) {
|
||||
const lineStart = lineIdx === 0 ? 0 : nlOffsets[lineIdx - 1] + 1;
|
||||
const spanBytes = buf.length - lineStart;
|
||||
|
||||
if (spanBytes > maxBytes) {
|
||||
truncatedBy = "bytes";
|
||||
if (includedLines === 0) {
|
||||
// Last line alone exceeds byte limit — take its tail
|
||||
const tailResult = truncateTailBytes(buf, maxBytes);
|
||||
return {
|
||||
content: tailResult.text,
|
||||
truncated: true,
|
||||
truncatedBy: "bytes",
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: 1,
|
||||
outputBytes: tailResult.bytes,
|
||||
lastLinePartial: true,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
startByte = lineStart;
|
||||
includedLines++;
|
||||
}
|
||||
|
||||
if (includedLines >= maxLines && buf.length - startByte <= maxBytes) {
|
||||
truncatedBy = "lines";
|
||||
}
|
||||
|
||||
const outputBytes = buf.length - startByte;
|
||||
const outputContent = buf.subarray(startByte).toString("utf-8");
|
||||
|
||||
return {
|
||||
content: outputContent,
|
||||
truncated: true,
|
||||
truncatedBy,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: includedLines,
|
||||
outputBytes,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// TailBuffer — ring-style tail buffer with lazy joining
|
||||
//
|
||||
// Uses windowed truncateTailBytes for trimming, so only a suffix of the
|
||||
// accumulated string is ever encoded — not the entire buffer.
|
||||
// =============================================================================
|
||||
|
||||
const MAX_PENDING = 10;
|
||||
|
||||
export class TailBuffer {
|
||||
#pending: string[] = [];
|
||||
#pos = 0; // tracked byte count (approximate after trims)
|
||||
|
||||
constructor(readonly maxBytes: number) {}
|
||||
|
||||
append(text: string): void {
|
||||
if (!text) return;
|
||||
const n = Buffer.byteLength(text, "utf-8");
|
||||
this.#pos += n;
|
||||
|
||||
if (this.#pending.length > 0) {
|
||||
this.#pending.push(text);
|
||||
if (this.#pending.length > MAX_PENDING) this.#compact();
|
||||
// Trim when we exceed 2× budget to amortise cost
|
||||
if (this.#pos > this.maxBytes * 2) this.#trim();
|
||||
} else {
|
||||
this.#pending[0] = text;
|
||||
this.#pending.length = 1;
|
||||
}
|
||||
}
|
||||
|
||||
text(): string {
|
||||
return this.#trim();
|
||||
}
|
||||
|
||||
bytes(): number {
|
||||
return this.#pos;
|
||||
}
|
||||
|
||||
// -- private ---------------------------------------------------------------
|
||||
|
||||
#compact(): void {
|
||||
this.#pending[0] = this.#pending.join("");
|
||||
this.#pending.length = 1;
|
||||
}
|
||||
|
||||
#flush(): string {
|
||||
if (this.#pending.length === 0) return "";
|
||||
if (this.#pending.length > 1) this.#compact();
|
||||
return this.#pending[0];
|
||||
}
|
||||
|
||||
/** Trim the buffer to maxBytes using windowed tail truncation. */
|
||||
#trim(): string {
|
||||
if (this.#pos <= this.maxBytes) return this.#flush();
|
||||
|
||||
const joined = this.#flush();
|
||||
const { text, bytes } = truncateTailBytes(joined, this.maxBytes);
|
||||
this.#pos = bytes;
|
||||
this.#pending[0] = text;
|
||||
this.#pending.length = 1;
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// OutputSink — line-buffered output with file spill support
|
||||
//
|
||||
// Uses a string buffer with byte tracking. When the spill threshold is
|
||||
// exceeded, all data is written to a file sink and the in-memory buffer is
|
||||
// trimmed to a tail window via windowed truncateTailBytes.
|
||||
// =============================================================================
|
||||
|
||||
export class OutputSink {
|
||||
#buffer = "";
|
||||
#bufferBytes = 0;
|
||||
@@ -60,11 +471,13 @@ export class OutputSink {
|
||||
#totalBytes = 0;
|
||||
#sawData = false;
|
||||
#truncated = false;
|
||||
|
||||
#file?: {
|
||||
path: string;
|
||||
artifactId?: string;
|
||||
sink: Bun.FileSink;
|
||||
};
|
||||
|
||||
readonly #artifactPath?: string;
|
||||
readonly #artifactId?: string;
|
||||
readonly #spillThreshold: number;
|
||||
@@ -72,66 +485,44 @@ export class OutputSink {
|
||||
|
||||
constructor(options?: OutputSinkOptions) {
|
||||
const { artifactPath, artifactId, spillThreshold = DEFAULT_MAX_BYTES, onChunk } = options ?? {};
|
||||
|
||||
this.#artifactPath = artifactPath;
|
||||
this.#artifactId = artifactId;
|
||||
this.#spillThreshold = spillThreshold;
|
||||
this.#onChunk = onChunk;
|
||||
}
|
||||
|
||||
async #pushSanitized(data: string): Promise<void> {
|
||||
this.#onChunk?.(data);
|
||||
|
||||
const dataBytes = Buffer.byteLength(data, "utf-8");
|
||||
this.#totalBytes += dataBytes;
|
||||
if (data.length > 0) {
|
||||
this.#sawData = true;
|
||||
this.#totalLines += countNewlines(data);
|
||||
}
|
||||
|
||||
const bufferOverflow = this.#bufferBytes + dataBytes > this.#spillThreshold;
|
||||
const overflow = this.#file || bufferOverflow;
|
||||
const sink = overflow ? await this.#fileSink() : null;
|
||||
|
||||
this.#buffer += data;
|
||||
this.#bufferBytes += dataBytes;
|
||||
await sink?.write(data);
|
||||
|
||||
if (bufferOverflow) {
|
||||
this.#truncated = true;
|
||||
const trimmed = truncateStringToBytesFromEnd(this.#buffer, this.#spillThreshold);
|
||||
this.#buffer = trimmed.text;
|
||||
this.#bufferBytes = trimmed.bytes;
|
||||
}
|
||||
if (this.#file) {
|
||||
this.#truncated = true;
|
||||
}
|
||||
}
|
||||
|
||||
async #fileSink(): Promise<Bun.FileSink | null> {
|
||||
if (!this.#artifactPath) return null;
|
||||
if (!this.#file) {
|
||||
try {
|
||||
this.#file = {
|
||||
path: this.#artifactPath,
|
||||
artifactId: this.#artifactId,
|
||||
sink: Bun.file(this.#artifactPath).writer(),
|
||||
};
|
||||
await this.#file.sink.write(this.#buffer);
|
||||
} catch {
|
||||
try {
|
||||
await this.#file?.sink?.end();
|
||||
} catch {}
|
||||
this.#file = undefined;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return this.#file.sink;
|
||||
}
|
||||
|
||||
async push(chunk: string): Promise<void> {
|
||||
chunk = sanitizeText(chunk);
|
||||
await this.#pushSanitized(chunk);
|
||||
this.#onChunk?.(chunk);
|
||||
|
||||
const dataBytes = Buffer.byteLength(chunk, "utf-8");
|
||||
this.#totalBytes += dataBytes;
|
||||
|
||||
if (chunk.length > 0) {
|
||||
this.#sawData = true;
|
||||
this.#totalLines += countNewlines(chunk);
|
||||
}
|
||||
|
||||
const willOverflow = this.#bufferBytes + dataBytes > this.#spillThreshold;
|
||||
|
||||
// Write to file if already spilling or about to overflow
|
||||
if (this.#file != null || willOverflow) {
|
||||
const sink = await this.#ensureFileSink();
|
||||
await sink?.write(chunk);
|
||||
}
|
||||
|
||||
this.#buffer += chunk;
|
||||
this.#bufferBytes += dataBytes;
|
||||
|
||||
// Keep only a tail window in memory when overflowing
|
||||
if (willOverflow) {
|
||||
this.#truncated = true;
|
||||
const { text, bytes } = truncateTailBytes(this.#buffer, this.#spillThreshold);
|
||||
this.#buffer = text;
|
||||
this.#bufferBytes = bytes;
|
||||
}
|
||||
|
||||
if (this.#file) this.#truncated = true;
|
||||
}
|
||||
|
||||
createInput(): WritableStream<Uint8Array | string> {
|
||||
@@ -139,14 +530,9 @@ export class OutputSink {
|
||||
const finalize = async () => {
|
||||
await this.push(dec.decode());
|
||||
};
|
||||
|
||||
return new WritableStream({
|
||||
write: async chunk => {
|
||||
if (typeof chunk === "string") {
|
||||
await this.push(chunk);
|
||||
} else {
|
||||
await this.push(dec.decode(chunk, { stream: true }));
|
||||
}
|
||||
await this.push(typeof chunk === "string" ? chunk : dec.decode(chunk, { stream: true }));
|
||||
},
|
||||
close: finalize,
|
||||
abort: finalize,
|
||||
@@ -155,23 +541,120 @@ export class OutputSink {
|
||||
|
||||
async dump(notice?: string): Promise<OutputSummary> {
|
||||
const noticeLine = notice ? `[${notice}]\n` : "";
|
||||
const outputLines = countLines(this.#buffer);
|
||||
const outputBytes = this.#bufferBytes;
|
||||
const outputLines = this.#buffer.length > 0 ? countNewlines(this.#buffer) + 1 : 0;
|
||||
const totalLines = this.#sawData ? this.#totalLines + 1 : 0;
|
||||
const totalBytes = this.#totalBytes;
|
||||
|
||||
if (this.#file) {
|
||||
await this.#file.sink.end();
|
||||
}
|
||||
if (this.#file) await this.#file.sink.end();
|
||||
|
||||
return {
|
||||
output: `${noticeLine}${this.#buffer}`,
|
||||
truncated: this.#truncated,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
totalBytes: this.#totalBytes,
|
||||
outputLines,
|
||||
outputBytes,
|
||||
outputBytes: this.#bufferBytes,
|
||||
artifactId: this.#file?.artifactId,
|
||||
};
|
||||
}
|
||||
|
||||
// -- private ---------------------------------------------------------------
|
||||
|
||||
async #ensureFileSink(): Promise<Bun.FileSink | null> {
|
||||
if (!this.#artifactPath) return null;
|
||||
if (this.#file) return this.#file.sink;
|
||||
|
||||
try {
|
||||
const sink = Bun.file(this.#artifactPath).writer();
|
||||
this.#file = { path: this.#artifactPath, artifactId: this.#artifactId, sink };
|
||||
// Flush existing buffer to file BEFORE it gets trimmed
|
||||
await sink.write(this.#buffer);
|
||||
return sink;
|
||||
} catch {
|
||||
try {
|
||||
await this.#file?.sink?.end();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
this.#file = undefined;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Session helpers
|
||||
// =============================================================================
|
||||
|
||||
const kEmpty = Object.freeze({} as { id?: string; path?: string });
|
||||
|
||||
/** Allocate a new artifact path and ID without writing content. */
|
||||
export async function allocateOutputArtifact(session: ToolSession, toolType: string) {
|
||||
const manager = session.getArtifactManager?.();
|
||||
if (!manager) return kEmpty;
|
||||
|
||||
try {
|
||||
return await manager.allocatePath(toolType);
|
||||
} catch {
|
||||
return kEmpty;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Truncation notice formatting
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Format a truncation notice for tail-truncated output (bash, python, ssh).
|
||||
* Returns empty string if not truncated.
|
||||
*/
|
||||
export function formatTailTruncationNotice(
|
||||
truncation: TruncationResult,
|
||||
options: TailTruncationNoticeOptions = {},
|
||||
): string {
|
||||
if (!truncation.truncated) return "";
|
||||
|
||||
const { fullOutputPath, originalContent, suffix = "" } = options;
|
||||
const startLine = truncation.totalLines - truncation.outputLines + 1;
|
||||
const endLine = truncation.totalLines;
|
||||
const fullOutputPart = fullOutputPath ? `. Full output: ${fullOutputPath}` : "";
|
||||
|
||||
let notice: string;
|
||||
if (truncation.lastLinePartial) {
|
||||
let lastLineSizePart = "";
|
||||
if (originalContent) {
|
||||
const lastNl = originalContent.lastIndexOf("\n");
|
||||
const lastLine = lastNl === -1 ? originalContent : originalContent.substring(lastNl + 1);
|
||||
lastLineSizePart = ` (line is ${formatBytes(Buffer.byteLength(lastLine, "utf-8"))})`;
|
||||
}
|
||||
notice = `[Showing last ${formatBytes(truncation.outputBytes)} of line ${endLine}${lastLineSizePart}${fullOutputPart}${suffix}]`;
|
||||
} else if (truncation.truncatedBy === "lines") {
|
||||
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}${fullOutputPart}${suffix}]`;
|
||||
} else {
|
||||
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatBytes(truncation.maxBytes)} limit)${fullOutputPart}${suffix}]`;
|
||||
}
|
||||
|
||||
return `\n\n${notice}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a truncation notice for head-truncated output (read tool).
|
||||
* Returns empty string if not truncated.
|
||||
*/
|
||||
export function formatHeadTruncationNotice(
|
||||
truncation: TruncationResult,
|
||||
options: HeadTruncationNoticeOptions = {},
|
||||
): string {
|
||||
if (!truncation.truncated) return "";
|
||||
|
||||
const startLineDisplay = options.startLine ?? 1;
|
||||
const totalFileLines = options.totalFileLines ?? truncation.totalLines;
|
||||
const endLineDisplay = startLineDisplay + truncation.outputLines - 1;
|
||||
const nextOffset = endLineDisplay + 1;
|
||||
|
||||
const notice =
|
||||
truncation.truncatedBy === "lines"
|
||||
? `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`
|
||||
: `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines} (${formatBytes(truncation.maxBytes)} limit). Use offset=${nextOffset} to continue]`;
|
||||
|
||||
return `\n\n${notice}`;
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import type { Theme } from "../modes/theme/theme";
|
||||
import planModeSubagentPrompt from "../prompts/system/plan-mode-subagent.md" with { type: "text" };
|
||||
import taskDescriptionTemplate from "../prompts/tools/task.md" with { type: "text" };
|
||||
import taskSummaryTemplate from "../prompts/tools/task-summary.md" with { type: "text" };
|
||||
import { formatDuration } from "../tools/render-utils";
|
||||
import { formatBytes, formatDuration } from "../tools/render-utils";
|
||||
// Import review tools for side effects (registers subagent tool handlers)
|
||||
import "../tools/review";
|
||||
import { discoverAgents, getAgent } from "./discovery";
|
||||
@@ -55,13 +55,6 @@ import {
|
||||
type WorktreeBaseline,
|
||||
} from "./worktree";
|
||||
|
||||
/** Format byte count for display */
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}K`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)}M`;
|
||||
}
|
||||
|
||||
function createUsageTotals(): Usage {
|
||||
return {
|
||||
input: 0,
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { RenderResultOptions } from "../extensibility/custom-tools/types";
|
||||
import { truncateToVisualLines } from "../modes/components/visual-truncate";
|
||||
import type { Theme } from "../modes/theme/theme";
|
||||
import bashDescription from "../prompts/tools/bash.md" with { type: "text" };
|
||||
import { allocateOutputArtifact, DEFAULT_MAX_BYTES, TailBuffer } from "../session/streaming-output";
|
||||
import { renderStatusLine } from "../tui";
|
||||
import { CachedOutputBlock } from "../tui/output-block";
|
||||
import type { ToolSession } from ".";
|
||||
@@ -20,12 +21,10 @@ import { checkBashInterception } from "./bash-interceptor";
|
||||
import { applyHeadTail } from "./bash-normalize";
|
||||
import { expandInternalUrls } from "./bash-skill-urls";
|
||||
import type { OutputMeta } from "./output-meta";
|
||||
import { allocateOutputArtifact, createTailBuffer } from "./output-utils";
|
||||
import { resolveToCwd } from "./path-utils";
|
||||
import { formatBytes, replaceTabs, wrapBrackets } from "./render-utils";
|
||||
import { ToolAbortError, ToolError } from "./tool-errors";
|
||||
import { toolResult } from "./tool-result";
|
||||
import { DEFAULT_MAX_BYTES } from "./truncate";
|
||||
|
||||
export const BASH_DEFAULT_PREVIEW_LINES = 10;
|
||||
|
||||
@@ -114,12 +113,12 @@ export class BashTool implements AgentTool<typeof bashSchema, BashToolDetails> {
|
||||
const timeoutMs = timeoutSec * 1000;
|
||||
|
||||
// Track output for streaming updates (tail only)
|
||||
const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES);
|
||||
const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES);
|
||||
|
||||
// Set up artifacts environment and allocation
|
||||
const artifactsDir = this.session.getArtifactsDir?.();
|
||||
const extraEnv = artifactsDir ? { ARTIFACTS: artifactsDir } : undefined;
|
||||
const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "bash");
|
||||
const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "bash");
|
||||
|
||||
const usePty =
|
||||
this.session.settings.get("bash.virtualTerminal") === "on" &&
|
||||
|
||||
@@ -10,6 +10,7 @@ import { renderPromptTemplate } from "../config/prompt-templates";
|
||||
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
|
||||
import { type Theme, theme } from "../modes/theme/theme";
|
||||
import fetchDescription from "../prompts/tools/fetch.md" with { type: "text" };
|
||||
import { allocateOutputArtifact, DEFAULT_MAX_BYTES, truncateHead } from "../session/streaming-output";
|
||||
import { renderStatusLine } from "../tui";
|
||||
import { CachedOutputBlock } from "../tui/output-block";
|
||||
import { ensureTool } from "../utils/tools-manager";
|
||||
@@ -20,11 +21,9 @@ import { convertWithMarkitdown, fetchBinary } from "../web/scrapers/utils";
|
||||
import type { ToolSession } from ".";
|
||||
import { applyListLimit } from "./list-limit";
|
||||
import type { OutputMeta } from "./output-meta";
|
||||
import { allocateOutputArtifact } from "./output-utils";
|
||||
import { formatExpandHint } from "./render-utils";
|
||||
import { ToolAbortError } from "./tool-errors";
|
||||
import { toolResult } from "./tool-result";
|
||||
import { DEFAULT_MAX_BYTES, truncateHead } from "./truncate";
|
||||
|
||||
// =============================================================================
|
||||
// Types and Constants
|
||||
@@ -900,10 +899,10 @@ export class FetchTool implements AgentTool<typeof fetchSchema, FetchToolDetails
|
||||
};
|
||||
|
||||
if (needsArtifact) {
|
||||
const { artifactPath, artifactId: allocatedId } = await allocateOutputArtifact(this.session, "fetch");
|
||||
const { path: artifactPath, id } = await allocateOutputArtifact(this.session, "fetch");
|
||||
if (artifactPath) {
|
||||
await Bun.write(artifactPath, buildOutput(result.content));
|
||||
artifactId = allocatedId;
|
||||
artifactId = id;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import { renderPromptTemplate } from "../config/prompt-templates";
|
||||
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
|
||||
import type { Theme } from "../modes/theme/theme";
|
||||
import findDescription from "../prompts/tools/find.md" with { type: "text" };
|
||||
import { type TruncationResult, truncateHead } from "../session/streaming-output";
|
||||
import {
|
||||
Ellipsis,
|
||||
Hasher,
|
||||
@@ -27,7 +28,6 @@ import { resolveToCwd } from "./path-utils";
|
||||
import { formatCount, formatEmptyMessage, formatErrorMessage, PREVIEW_LIMITS } from "./render-utils";
|
||||
import { ToolAbortError, ToolError, throwIfAborted } from "./tool-errors";
|
||||
import { toolResult } from "./tool-result";
|
||||
import { type TruncationResult, truncateHead } from "./truncate";
|
||||
|
||||
const findSchema = Type.Object({
|
||||
pattern: Type.String({ description: "Glob pattern, e.g. '*.ts', 'src/**/*.json', 'lib/*.tsx'" }),
|
||||
|
||||
@@ -11,6 +11,7 @@ import type { RenderResultOptions } from "../extensibility/custom-tools/types";
|
||||
import type { Theme } from "../modes/theme/theme";
|
||||
import { computeLineHash } from "../patch/hashline";
|
||||
import grepDescription from "../prompts/tools/grep.md" with { type: "text" };
|
||||
import { DEFAULT_MAX_COLUMN, type TruncationResult, truncateHead } from "../session/streaming-output";
|
||||
import { Ellipsis, Hasher, type RenderCache, renderStatusLine, renderTreeList, truncateToWidth } from "../tui";
|
||||
import { resolveFileDisplayMode } from "../utils/file-display-mode";
|
||||
import type { ToolSession } from ".";
|
||||
@@ -19,7 +20,6 @@ import { resolveToCwd } from "./path-utils";
|
||||
import { formatCount, formatEmptyMessage, formatErrorMessage, PREVIEW_LIMITS } from "./render-utils";
|
||||
import { ToolError } from "./tool-errors";
|
||||
import { toolResult } from "./tool-result";
|
||||
import { DEFAULT_MAX_COLUMN, type TruncationResult, truncateHead } from "./truncate";
|
||||
|
||||
const grepSchema = Type.Object({
|
||||
pattern: Type.String({ description: "Regex pattern to search for" }),
|
||||
|
||||
@@ -48,23 +48,9 @@ export {
|
||||
warmupLspServers,
|
||||
} from "../lsp";
|
||||
export { EditTool, type EditToolDetails } from "../patch";
|
||||
export * from "../session/streaming-output";
|
||||
export { BUNDLED_AGENTS, TaskTool } from "../task";
|
||||
export {
|
||||
companySearchTools,
|
||||
exaSearchTools,
|
||||
getSearchTools,
|
||||
type SearchProvider,
|
||||
type SearchResponse,
|
||||
SearchTool,
|
||||
type SearchToolsOptions,
|
||||
setPreferredSearchProvider,
|
||||
webSearchCodeContextTool,
|
||||
webSearchCompanyTool,
|
||||
webSearchCrawlTool,
|
||||
webSearchCustomTool,
|
||||
webSearchDeepTool,
|
||||
webSearchLinkedinTool,
|
||||
} from "../web/search";
|
||||
export * from "../web/search";
|
||||
export { AskTool, type AskToolDetails } from "./ask";
|
||||
export { BashTool, type BashToolDetails, type BashToolInput, type BashToolOptions } from "./bash";
|
||||
export { BrowserTool, type BrowserToolDetails } from "./browser";
|
||||
@@ -81,16 +67,6 @@ export { reportFindingTool, type SubmitReviewDetails } from "./review";
|
||||
export { loadSshTool, type SSHToolDetails, SshTool } from "./ssh";
|
||||
export { SubmitResultTool } from "./submit-result";
|
||||
export { type TodoItem, TodoWriteTool, type TodoWriteToolDetails } from "./todo-write";
|
||||
export {
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
formatSize,
|
||||
type TruncationOptions,
|
||||
type TruncationResult,
|
||||
truncateHead,
|
||||
truncateLine,
|
||||
truncateTail,
|
||||
} from "./truncate";
|
||||
export { WriteTool, type WriteToolDetails, type WriteToolInput } from "./write";
|
||||
|
||||
/** Tool type (AgentTool from pi-ai) */
|
||||
@@ -132,8 +108,8 @@ export interface ToolSession {
|
||||
getSessionFile: () => string | null;
|
||||
/** Get session ID */
|
||||
getSessionId?: () => string | null;
|
||||
/** Cached artifact manager (allocated per ToolSession) */
|
||||
artifactManager?: ArtifactManager;
|
||||
/** Get artifact manager (allocated per ToolSession) */
|
||||
getArtifactManager?: () => ArtifactManager | null;
|
||||
/** Get artifacts directory for artifact:// URLs and $ARTIFACTS env var */
|
||||
getArtifactsDir?: () => string | null;
|
||||
/** Get session spawns */
|
||||
|
||||
@@ -12,10 +12,9 @@ import type {
|
||||
AgentToolUpdateCallback,
|
||||
} from "@oh-my-pi/pi-agent-core";
|
||||
import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai";
|
||||
import type { OutputSummary } from "../session/streaming-output";
|
||||
import type { OutputSummary, TruncationResult } from "../session/streaming-output";
|
||||
import { formatBytes } from "./render-utils";
|
||||
import { renderError } from "./tool-errors";
|
||||
import type { TruncationResult } from "./truncate";
|
||||
import { formatSize } from "./truncate";
|
||||
|
||||
/**
|
||||
* Truncation metadata for the output notice.
|
||||
@@ -336,7 +335,7 @@ export function formatOutputNotice(meta: OutputMeta | undefined): string {
|
||||
|
||||
if (t.truncatedBy === "bytes") {
|
||||
const maxBytes = t.maxBytes ?? t.outputBytes;
|
||||
notice += ` (${formatSize(maxBytes)} limit)`;
|
||||
notice += ` (${formatBytes(maxBytes)} limit)`;
|
||||
}
|
||||
|
||||
if (t.nextOffset != null) {
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
import { ArtifactManager } from "../session/artifacts";
|
||||
import type { ToolSession } from ".";
|
||||
|
||||
export interface TailBuffer {
|
||||
append(chunk: string): void;
|
||||
text(): string;
|
||||
bytes(): number;
|
||||
}
|
||||
|
||||
export function createTailBuffer(maxBytes: number): TailBuffer {
|
||||
let buffer = "";
|
||||
let bufferBytes = 0;
|
||||
|
||||
const append = (text: string) => {
|
||||
if (!text) return;
|
||||
const chunkBytes = Buffer.byteLength(text, "utf-8");
|
||||
buffer += text;
|
||||
bufferBytes += chunkBytes;
|
||||
|
||||
if (bufferBytes > maxBytes) {
|
||||
const buf = Buffer.from(buffer, "utf-8");
|
||||
let start = Math.max(0, buf.length - maxBytes);
|
||||
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
|
||||
start++;
|
||||
}
|
||||
buffer = buf.subarray(start).toString("utf-8");
|
||||
bufferBytes = Buffer.byteLength(buffer, "utf-8");
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
append,
|
||||
text: () => buffer,
|
||||
bytes: () => bufferBytes,
|
||||
};
|
||||
}
|
||||
|
||||
export function getArtifactManager(session: ToolSession): ArtifactManager | null {
|
||||
if (session.artifactManager) {
|
||||
return session.artifactManager;
|
||||
}
|
||||
const sessionFile = session.getSessionFile();
|
||||
if (!sessionFile) {
|
||||
return null;
|
||||
}
|
||||
const manager = new ArtifactManager(sessionFile);
|
||||
session.artifactManager = manager;
|
||||
return manager;
|
||||
}
|
||||
|
||||
export async function allocateOutputArtifact(
|
||||
session: ToolSession,
|
||||
toolType: string,
|
||||
): Promise<{ artifactPath?: string; artifactId?: string }> {
|
||||
const manager = getArtifactManager(session);
|
||||
if (!manager) return {};
|
||||
try {
|
||||
const allocation = await manager.allocatePath(toolType);
|
||||
return { artifactPath: allocation.path, artifactId: allocation.id };
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
@@ -13,16 +13,20 @@ import type { PreludeHelper, PythonStatusEvent } from "../ipy/kernel";
|
||||
import { truncateToVisualLines } from "../modes/components/visual-truncate";
|
||||
import type { Theme } from "../modes/theme/theme";
|
||||
import pythonDescription from "../prompts/tools/python.md" with { type: "text" };
|
||||
import { OutputSink, type OutputSummary } from "../session/streaming-output";
|
||||
import {
|
||||
allocateOutputArtifact,
|
||||
DEFAULT_MAX_BYTES,
|
||||
OutputSink,
|
||||
type OutputSummary,
|
||||
TailBuffer,
|
||||
} from "../session/streaming-output";
|
||||
import { getTreeBranch, getTreeContinuePrefix, renderCodeCell } from "../tui";
|
||||
import type { ToolSession } from ".";
|
||||
import type { OutputMeta } from "./output-meta";
|
||||
import { allocateOutputArtifact, createTailBuffer } from "./output-utils";
|
||||
import { resolveToCwd } from "./path-utils";
|
||||
import { replaceTabs, shortenPath, ToolUIKit, truncateToWidth } from "./render-utils";
|
||||
import { ToolAbortError, ToolError } from "./tool-errors";
|
||||
import { toolResult } from "./tool-result";
|
||||
import { DEFAULT_MAX_BYTES } from "./truncate";
|
||||
|
||||
export const PYTHON_DEFAULT_PREVIEW_LINES = 10;
|
||||
|
||||
@@ -207,7 +211,7 @@ export class PythonTool implements AgentTool<typeof pythonSchema> {
|
||||
throw new ToolError(`Working directory is not a directory: ${commandCwd}`);
|
||||
}
|
||||
|
||||
const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES * 2);
|
||||
const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES * 2);
|
||||
const jsonOutputs: unknown[] = [];
|
||||
const images: ImageContent[] = [];
|
||||
const statusEvents: PythonStatusEvent[] = [];
|
||||
@@ -255,7 +259,7 @@ export class PythonTool implements AgentTool<typeof pythonSchema> {
|
||||
|
||||
const sessionFile = this.session.getSessionFile?.() ?? undefined;
|
||||
const artifactsDir = this.session.getArtifactsDir?.() ?? undefined;
|
||||
const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "python");
|
||||
const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "python");
|
||||
outputSink = new OutputSink({
|
||||
artifactPath,
|
||||
artifactId,
|
||||
|
||||
@@ -14,6 +14,13 @@ import { getLanguageFromPath, type Theme } from "../modes/theme/theme";
|
||||
import { computeLineHash } from "../patch/hashline";
|
||||
import readDescription from "../prompts/tools/read.md" with { type: "text" };
|
||||
import type { ToolSession } from "../sdk";
|
||||
import {
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
type TruncationResult,
|
||||
truncateHead,
|
||||
truncateHeadBytes,
|
||||
} from "../session/streaming-output";
|
||||
import { renderCodeCell, renderStatusLine } from "../tui";
|
||||
import { CachedOutputBlock } from "../tui/output-block";
|
||||
import { resolveFileDisplayMode } from "../utils/file-display-mode";
|
||||
@@ -23,17 +30,9 @@ import { ensureTool } from "../utils/tools-manager";
|
||||
import { applyListLimit } from "./list-limit";
|
||||
import type { OutputMeta } from "./output-meta";
|
||||
import { resolveReadPath, resolveToCwd } from "./path-utils";
|
||||
import { formatAge, shortenPath, wrapBrackets } from "./render-utils";
|
||||
import { formatAge, formatBytes, shortenPath, wrapBrackets } from "./render-utils";
|
||||
import { ToolAbortError, ToolError, throwIfAborted } from "./tool-errors";
|
||||
import { toolResult } from "./tool-result";
|
||||
import {
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
formatSize,
|
||||
type TruncationResult,
|
||||
truncateHead,
|
||||
truncateStringToBytesFromStart,
|
||||
} from "./truncate";
|
||||
|
||||
// Document types convertible via markitdown
|
||||
const CONVERTIBLE_EXTENSIONS = new Set([".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".rtf", ".epub"]);
|
||||
@@ -211,17 +210,8 @@ async function streamLinesFromFile(
|
||||
|
||||
let firstLinePreview: { text: string; bytes: number } | undefined;
|
||||
if (firstLinePreviewBytes > 0) {
|
||||
const buf = Buffer.concat(firstLinePreviewChunks, firstLinePreviewBytes);
|
||||
let end = Math.min(buf.length, maxBytes);
|
||||
while (end > 0 && (buf[end] & 0xc0) === 0x80) {
|
||||
end--;
|
||||
}
|
||||
if (end > 0) {
|
||||
const text = buf.slice(0, end).toString("utf-8");
|
||||
firstLinePreview = { text, bytes: Buffer.byteLength(text, "utf-8") };
|
||||
} else {
|
||||
firstLinePreview = { text: "", bytes: 0 };
|
||||
}
|
||||
const { text, bytes } = truncateHeadBytes(Buffer.concat(firstLinePreviewChunks, firstLinePreviewBytes), maxBytes);
|
||||
firstLinePreview = { text, bytes };
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -623,8 +613,8 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
||||
|
||||
if (mimeType) {
|
||||
if (fileSize > MAX_IMAGE_SIZE) {
|
||||
const sizeStr = formatSize(fileSize);
|
||||
const maxStr = formatSize(MAX_IMAGE_SIZE);
|
||||
const sizeStr = formatBytes(fileSize);
|
||||
const maxStr = formatBytes(MAX_IMAGE_SIZE);
|
||||
throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`);
|
||||
} else {
|
||||
// Read as image (binary)
|
||||
@@ -633,8 +623,8 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
||||
|
||||
// Check actual buffer size after reading to prevent OOM during serialization
|
||||
if (buffer.byteLength > MAX_IMAGE_SIZE) {
|
||||
const sizeStr = formatSize(buffer.byteLength);
|
||||
const maxStr = formatSize(MAX_IMAGE_SIZE);
|
||||
const sizeStr = formatBytes(buffer.byteLength);
|
||||
const maxStr = formatBytes(MAX_IMAGE_SIZE);
|
||||
throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`);
|
||||
} else {
|
||||
const base64 = new Uint8Array(buffer).toBase64();
|
||||
@@ -788,16 +778,16 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
||||
const snippet = firstLinePreview ?? { text: "", bytes: 0 };
|
||||
|
||||
if (shouldAddHashLines) {
|
||||
outputText = `[Line ${startLineDisplay} is ${formatSize(
|
||||
outputText = `[Line ${startLineDisplay} is ${formatBytes(
|
||||
firstLineBytes,
|
||||
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
|
||||
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
|
||||
} else {
|
||||
outputText = formatText(snippet.text, startLineDisplay);
|
||||
}
|
||||
if (snippet.text.length === 0) {
|
||||
outputText = `[Line ${startLineDisplay} is ${formatSize(
|
||||
outputText = `[Line ${startLineDisplay} is ${formatBytes(
|
||||
firstLineBytes,
|
||||
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
|
||||
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
|
||||
}
|
||||
details = { truncation };
|
||||
sourcePath = absolutePath;
|
||||
@@ -946,19 +936,19 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
||||
if (truncation.firstLineExceedsLimit) {
|
||||
const firstLine = allLines[startLine] ?? "";
|
||||
const firstLineBytes = Buffer.byteLength(firstLine, "utf-8");
|
||||
const snippet = truncateStringToBytesFromStart(firstLine, DEFAULT_MAX_BYTES);
|
||||
const snippet = truncateHeadBytes(firstLine, DEFAULT_MAX_BYTES);
|
||||
|
||||
if (shouldAddHashLines) {
|
||||
outputText = `[Line ${startLineDisplay} is ${formatSize(
|
||||
outputText = `[Line ${startLineDisplay} is ${formatBytes(
|
||||
firstLineBytes,
|
||||
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
|
||||
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`;
|
||||
} else {
|
||||
outputText = formatText(snippet.text, startLineDisplay);
|
||||
}
|
||||
if (snippet.text.length === 0) {
|
||||
outputText = `[Line ${startLineDisplay} is ${formatSize(
|
||||
outputText = `[Line ${startLineDisplay} is ${formatBytes(
|
||||
firstLineBytes,
|
||||
)}, exceeds ${formatSize(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
|
||||
)}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Unable to display a valid UTF-8 snippet.]`;
|
||||
}
|
||||
details = { truncation };
|
||||
truncationInfo = {
|
||||
@@ -1116,12 +1106,12 @@ export const readToolRenderer = {
|
||||
if (truncation) {
|
||||
let warning: string;
|
||||
if (fallback?.firstLineExceedsLimit) {
|
||||
warning = `First line exceeds ${formatSize(fallback.maxBytes ?? DEFAULT_MAX_BYTES)} limit`;
|
||||
warning = `First line exceeds ${formatBytes(fallback.maxBytes ?? DEFAULT_MAX_BYTES)} limit`;
|
||||
} else if (truncation.truncatedBy === "lines") {
|
||||
warning = `Truncated: ${truncation.outputLines} of ${truncation.totalLines} lines (${DEFAULT_MAX_LINES} line limit)`;
|
||||
} else {
|
||||
const maxBytes = fallback?.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
warning = `Truncated: ${truncation.outputLines} lines (${formatSize(maxBytes)} limit)`;
|
||||
warning = `Truncated: ${truncation.outputLines} lines (${formatBytes(maxBytes)} limit)`;
|
||||
}
|
||||
if (truncation.artifactId) {
|
||||
warning += `. Full output: artifact://${truncation.artifactId}`;
|
||||
|
||||
@@ -9,6 +9,7 @@ import { loadCapability } from "../discovery";
|
||||
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
|
||||
import type { Theme } from "../modes/theme/theme";
|
||||
import sshDescriptionBase from "../prompts/tools/ssh.md" with { type: "text" };
|
||||
import { allocateOutputArtifact, DEFAULT_MAX_BYTES, TailBuffer } from "../session/streaming-output";
|
||||
import type { SSHHostInfo } from "../ssh/connection-manager";
|
||||
import { ensureHostInfo, getHostInfoForHost } from "../ssh/connection-manager";
|
||||
import { executeSSH } from "../ssh/ssh-executor";
|
||||
@@ -16,11 +17,9 @@ import { renderStatusLine } from "../tui";
|
||||
import { CachedOutputBlock } from "../tui/output-block";
|
||||
import type { ToolSession } from ".";
|
||||
import type { OutputMeta } from "./output-meta";
|
||||
import { allocateOutputArtifact, createTailBuffer } from "./output-utils";
|
||||
import { formatBytes, wrapBrackets } from "./render-utils";
|
||||
import { ToolError } from "./tool-errors";
|
||||
import { toolResult } from "./tool-result";
|
||||
import { DEFAULT_MAX_BYTES } from "./truncate";
|
||||
|
||||
const sshSchema = Type.Object({
|
||||
host: Type.String({ description: "Host name from managed SSH config or discovered ssh.json files" }),
|
||||
@@ -159,8 +158,8 @@ export class SshTool implements AgentTool<typeof sshSchema, SSHToolDetails> {
|
||||
const timeoutSec = Math.max(1, Math.min(3600, rawTimeout));
|
||||
const timeoutMs = timeoutSec * 1000;
|
||||
|
||||
const tailBuffer = createTailBuffer(DEFAULT_MAX_BYTES);
|
||||
const { artifactPath, artifactId } = await allocateOutputArtifact(this.session, "ssh");
|
||||
const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES);
|
||||
const { path: artifactPath, id: artifactId } = await allocateOutputArtifact(this.session, "ssh");
|
||||
|
||||
const result = await executeSSH(hostConfig, remoteCommand, {
|
||||
timeout: timeoutMs,
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import type { AgentToolResult } from "@oh-my-pi/pi-agent-core";
|
||||
import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai";
|
||||
import type { OutputSummary } from "../session/streaming-output";
|
||||
import type { OutputSummary, TruncationResult } from "../session/streaming-output";
|
||||
import type { OutputMeta, TruncationOptions, TruncationSummaryOptions, TruncationTextOptions } from "./output-meta";
|
||||
import { outputMeta } from "./output-meta";
|
||||
import type { TruncationResult } from "./truncate";
|
||||
|
||||
type ToolContent = Array<TextContent | ImageContent>;
|
||||
|
||||
|
||||
@@ -1,385 +0,0 @@
|
||||
/**
|
||||
* Shared truncation utilities for tool outputs.
|
||||
*
|
||||
* Truncation is based on two independent limits - whichever is hit first wins:
|
||||
* - Line limit (default: 4000 lines)
|
||||
* - Byte limit (default: 50KB)
|
||||
*
|
||||
* Never returns partial lines (except bash tail truncation edge case
|
||||
* and the read tool's long-line snippet fallback).
|
||||
*/
|
||||
|
||||
export const DEFAULT_MAX_LINES = 3000;
|
||||
export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB
|
||||
export const DEFAULT_MAX_COLUMN = 1024; // Max chars per grep match line
|
||||
|
||||
export interface TruncationResult {
|
||||
/** The truncated content */
|
||||
content: string;
|
||||
/** Whether truncation occurred */
|
||||
truncated: boolean;
|
||||
/** Which limit was hit: "lines", "bytes", or null if not truncated */
|
||||
truncatedBy: "lines" | "bytes" | null;
|
||||
/** Total number of lines in the original content */
|
||||
totalLines: number;
|
||||
/** Total number of bytes in the original content */
|
||||
totalBytes: number;
|
||||
/** Number of complete lines in the truncated output */
|
||||
outputLines: number;
|
||||
/** Number of bytes in the truncated output */
|
||||
outputBytes: number;
|
||||
/** Whether the last line was partially truncated (only for tail truncation edge case) */
|
||||
lastLinePartial: boolean;
|
||||
/** Whether the first line exceeded the byte limit (for head truncation) */
|
||||
firstLineExceedsLimit: boolean;
|
||||
/** The max lines limit that was applied */
|
||||
maxLines: number;
|
||||
/** The max bytes limit that was applied */
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
export interface TruncationOptions {
|
||||
/** Maximum number of lines (default: 2000) */
|
||||
maxLines?: number;
|
||||
/** Maximum number of bytes (default: 50KB) */
|
||||
maxBytes?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes as human-readable size.
|
||||
*/
|
||||
export function formatSize(bytes: number): string {
|
||||
if (bytes < 1024) {
|
||||
return `${bytes}B`;
|
||||
} else if (bytes < 1024 * 1024) {
|
||||
return `${(bytes / 1024).toFixed(1)}KB`;
|
||||
} else if (bytes < 1024 * 1024 * 1024) {
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
||||
} else {
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate content from the head (keep first N lines/bytes).
|
||||
* Suitable for file reads where you want to see the beginning.
|
||||
*
|
||||
* Never returns partial lines. If first line exceeds byte limit,
|
||||
* returns empty content with firstLineExceedsLimit=true.
|
||||
*/
|
||||
export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult {
|
||||
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
|
||||
const totalBytes = Buffer.byteLength(content, "utf-8");
|
||||
const lines = content.split("\n");
|
||||
const totalLines = lines.length;
|
||||
|
||||
// Check if no truncation needed
|
||||
if (totalLines <= maxLines && totalBytes <= maxBytes) {
|
||||
return {
|
||||
content,
|
||||
truncated: false,
|
||||
truncatedBy: null,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: totalLines,
|
||||
outputBytes: totalBytes,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
// Check if first line alone exceeds byte limit
|
||||
const firstLineBytes = Buffer.byteLength(lines[0], "utf-8");
|
||||
if (firstLineBytes > maxBytes) {
|
||||
return {
|
||||
content: "",
|
||||
truncated: true,
|
||||
truncatedBy: "bytes",
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: 0,
|
||||
outputBytes: 0,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: true,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
// Collect complete lines that fit
|
||||
const outputLinesArr: string[] = [];
|
||||
let outputBytesCount = 0;
|
||||
let truncatedBy: "lines" | "bytes" = "lines";
|
||||
|
||||
for (let i = 0; i < lines.length && i < maxLines; i++) {
|
||||
const line = lines[i];
|
||||
const lineBytes = Buffer.byteLength(line, "utf-8") + (i > 0 ? 1 : 0); // +1 for newline
|
||||
|
||||
if (outputBytesCount + lineBytes > maxBytes) {
|
||||
truncatedBy = "bytes";
|
||||
break;
|
||||
}
|
||||
|
||||
outputLinesArr.push(line);
|
||||
outputBytesCount += lineBytes;
|
||||
}
|
||||
|
||||
// If we exited due to line limit
|
||||
if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
|
||||
truncatedBy = "lines";
|
||||
}
|
||||
|
||||
const outputContent = outputLinesArr.join("\n");
|
||||
const finalOutputBytes = Buffer.byteLength(outputContent, "utf-8");
|
||||
|
||||
return {
|
||||
content: outputContent,
|
||||
truncated: true,
|
||||
truncatedBy,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: outputLinesArr.length,
|
||||
outputBytes: finalOutputBytes,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate content from the tail (keep last N lines/bytes).
|
||||
* Suitable for bash output where you want to see the end (errors, final results).
|
||||
*
|
||||
* May return partial first line if the last line of original content exceeds byte limit.
|
||||
*/
|
||||
export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult {
|
||||
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
|
||||
const totalBytes = Buffer.byteLength(content, "utf-8");
|
||||
const lines = content.split("\n");
|
||||
const totalLines = lines.length;
|
||||
|
||||
// Check if no truncation needed
|
||||
if (totalLines <= maxLines && totalBytes <= maxBytes) {
|
||||
return {
|
||||
content,
|
||||
truncated: false,
|
||||
truncatedBy: null,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: totalLines,
|
||||
outputBytes: totalBytes,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
// Work backwards from the end
|
||||
const outputLinesArr: string[] = [];
|
||||
let outputBytesCount = 0;
|
||||
let truncatedBy: "lines" | "bytes" = "lines";
|
||||
let lastLinePartial = false;
|
||||
|
||||
for (let i = lines.length - 1; i >= 0 && outputLinesArr.length < maxLines; i--) {
|
||||
const line = lines[i];
|
||||
const lineBytes = Buffer.byteLength(line, "utf-8") + (outputLinesArr.length > 0 ? 1 : 0); // +1 for newline
|
||||
|
||||
if (outputBytesCount + lineBytes > maxBytes) {
|
||||
truncatedBy = "bytes";
|
||||
// Edge case: if we haven't added ANY lines yet and this line exceeds maxBytes,
|
||||
// take the end of the line (partial)
|
||||
if (outputLinesArr.length === 0) {
|
||||
const truncatedLine = truncateStringToBytesFromEnd(line, maxBytes);
|
||||
outputLinesArr.unshift(truncatedLine);
|
||||
outputBytesCount = Buffer.byteLength(truncatedLine, "utf-8");
|
||||
lastLinePartial = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
outputLinesArr.unshift(line);
|
||||
outputBytesCount += lineBytes;
|
||||
}
|
||||
|
||||
// If we exited due to line limit
|
||||
if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
|
||||
truncatedBy = "lines";
|
||||
}
|
||||
|
||||
const outputContent = outputLinesArr.join("\n");
|
||||
const finalOutputBytes = Buffer.byteLength(outputContent, "utf-8");
|
||||
|
||||
return {
|
||||
content: outputContent,
|
||||
truncated: true,
|
||||
truncatedBy,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: outputLinesArr.length,
|
||||
outputBytes: finalOutputBytes,
|
||||
lastLinePartial,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate a string to fit within a byte limit (from the end).
|
||||
* Handles multi-byte UTF-8 characters correctly.
|
||||
*/
|
||||
function truncateStringToBytesFromEnd(str: string, maxBytes: number): string {
|
||||
const buf = Buffer.from(str, "utf-8");
|
||||
if (buf.length <= maxBytes) {
|
||||
return str;
|
||||
}
|
||||
|
||||
// Start from the end, skip maxBytes back
|
||||
let start = buf.length - maxBytes;
|
||||
|
||||
// Find a valid UTF-8 boundary (start of a character)
|
||||
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
|
||||
start++;
|
||||
}
|
||||
|
||||
return buf.slice(start).toString("utf-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate a string to fit within a byte limit (from the start).
|
||||
* Handles multi-byte UTF-8 characters correctly.
|
||||
*/
|
||||
export function truncateStringToBytesFromStart(str: string, maxBytes: number): { text: string; bytes: number } {
|
||||
const buf = Buffer.from(str, "utf-8");
|
||||
if (buf.length <= maxBytes) {
|
||||
return { text: str, bytes: buf.length };
|
||||
}
|
||||
|
||||
let end = maxBytes;
|
||||
|
||||
// Find a valid UTF-8 boundary (start of a character)
|
||||
while (end > 0 && (buf[end] & 0xc0) === 0x80) {
|
||||
end--;
|
||||
}
|
||||
|
||||
if (end <= 0) {
|
||||
return { text: "", bytes: 0 };
|
||||
}
|
||||
|
||||
const text = buf.slice(0, end).toString("utf-8");
|
||||
return { text, bytes: Buffer.byteLength(text, "utf-8") };
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate a single line to max characters, adding [truncated] suffix.
|
||||
* Used for grep match lines.
|
||||
*/
|
||||
export function truncateLine(
|
||||
line: string,
|
||||
maxChars: number = DEFAULT_MAX_COLUMN,
|
||||
): { text: string; wasTruncated: boolean } {
|
||||
if (line.length <= maxChars) {
|
||||
return { text: line, wasTruncated: false };
|
||||
}
|
||||
return { text: `${line.slice(0, maxChars)}…`, wasTruncated: true };
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Truncation notice formatting
|
||||
// =============================================================================
|
||||
|
||||
export interface TailTruncationNoticeOptions {
|
||||
/** Path to full output file (e.g., from bash/python executor) */
|
||||
fullOutputPath?: string;
|
||||
/** Original content for computing last line size when lastLinePartial */
|
||||
originalContent?: string;
|
||||
/** Additional suffix to append inside the brackets */
|
||||
suffix?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a truncation notice for tail-truncated output (bash, python, ssh).
|
||||
* Returns empty string if not truncated.
|
||||
*
|
||||
* Examples:
|
||||
* - "[Showing last 50KB of line 1000 (line is 2.1MB). Full output: /tmp/out.txt]"
|
||||
* - "[Showing lines 500-1000 of 1000. Full output: /tmp/out.txt]"
|
||||
* - "[Showing lines 500-1000 of 1000 (50KB limit). Full output: /tmp/out.txt]"
|
||||
*/
|
||||
export function formatTailTruncationNotice(
|
||||
truncation: TruncationResult,
|
||||
options: TailTruncationNoticeOptions = {},
|
||||
): string {
|
||||
if (!truncation.truncated) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const { fullOutputPath, originalContent, suffix = "" } = options;
|
||||
const startLine = truncation.totalLines - truncation.outputLines + 1;
|
||||
const endLine = truncation.totalLines;
|
||||
const fullOutputPart = fullOutputPath ? `. Full output: ${fullOutputPath}` : "";
|
||||
|
||||
let notice: string;
|
||||
|
||||
if (truncation.lastLinePartial) {
|
||||
let lastLineSizePart = "";
|
||||
if (originalContent) {
|
||||
const lastLine = originalContent.split("\n").pop() || "";
|
||||
lastLineSizePart = ` (line is ${formatSize(Buffer.byteLength(lastLine, "utf-8"))})`;
|
||||
}
|
||||
notice = `[Showing last ${formatSize(truncation.outputBytes)} of line ${endLine}${lastLineSizePart}${fullOutputPart}${suffix}]`;
|
||||
} else if (truncation.truncatedBy === "lines") {
|
||||
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}${fullOutputPart}${suffix}]`;
|
||||
} else {
|
||||
notice = `[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatSize(truncation.maxBytes)} limit)${fullOutputPart}${suffix}]`;
|
||||
}
|
||||
|
||||
return `\n\n${notice}`;
|
||||
}
|
||||
|
||||
export interface HeadTruncationNoticeOptions {
|
||||
/** 1-indexed start line number (default: 1) */
|
||||
startLine?: number;
|
||||
/** Total lines in the original file (for "of N" display) */
|
||||
totalFileLines?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a truncation notice for head-truncated output (read tool).
|
||||
* Returns empty string if not truncated.
|
||||
*
|
||||
* Examples:
|
||||
* - "[Showing lines 1-2000 of 5000. Use offset=2001 to continue]"
|
||||
* - "[Showing lines 100-2099 of 5000 (50KB limit). Use offset=2100 to continue]"
|
||||
*/
|
||||
export function formatHeadTruncationNotice(
|
||||
truncation: TruncationResult,
|
||||
options: HeadTruncationNoticeOptions = {},
|
||||
): string {
|
||||
if (!truncation.truncated) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const startLineDisplay = options.startLine ?? 1;
|
||||
const totalFileLines = options.totalFileLines ?? truncation.totalLines;
|
||||
const endLineDisplay = startLineDisplay + truncation.outputLines - 1;
|
||||
const nextOffset = endLineDisplay + 1;
|
||||
|
||||
let notice: string;
|
||||
|
||||
if (truncation.truncatedBy === "lines") {
|
||||
notice = `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`;
|
||||
} else {
|
||||
notice = `[Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines} (${formatSize(truncation.maxBytes)} limit). Use offset=${nextOffset} to continue]`;
|
||||
}
|
||||
|
||||
return `\n\n${notice}`;
|
||||
}
|
||||
@@ -11,9 +11,9 @@ import type { AgentMessage } from "@oh-my-pi/pi-agent-core";
|
||||
import { glob } from "@oh-my-pi/pi-natives";
|
||||
import { formatHashLines } from "../patch/hashline";
|
||||
import type { FileMentionMessage } from "../session/messages";
|
||||
import { DEFAULT_MAX_BYTES, truncateHead, truncateHeadBytes } from "../session/streaming-output";
|
||||
import { resolveReadPath } from "../tools/path-utils";
|
||||
import { formatAge } from "../tools/render-utils";
|
||||
import { DEFAULT_MAX_BYTES, formatSize, truncateHead, truncateStringToBytesFromStart } from "../tools/truncate";
|
||||
import { formatAge, formatBytes } from "../tools/render-utils";
|
||||
import { fuzzyMatch } from "./fuzzy";
|
||||
import { formatDimensionNote, resizeImage } from "./image-resize";
|
||||
import { detectSupportedImageMimeTypeFromFile } from "./mime";
|
||||
@@ -162,15 +162,15 @@ function buildTextOutput(textContent: string): { output: string; lineCount: numb
|
||||
if (truncation.firstLineExceedsLimit) {
|
||||
const firstLine = allLines[0] ?? "";
|
||||
const firstLineBytes = Buffer.byteLength(firstLine, "utf-8");
|
||||
const snippet = truncateStringToBytesFromStart(firstLine, DEFAULT_MAX_BYTES);
|
||||
const snippet = truncateHeadBytes(firstLine, DEFAULT_MAX_BYTES);
|
||||
let outputText = snippet.text;
|
||||
|
||||
if (outputText.length > 0) {
|
||||
outputText += `\n\n[Line 1 is ${formatSize(firstLineBytes)}, exceeds ${formatSize(
|
||||
outputText += `\n\n[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes(
|
||||
DEFAULT_MAX_BYTES,
|
||||
)} limit. Showing first ${formatSize(snippet.bytes)} of the line.]`;
|
||||
)} limit. Showing first ${formatBytes(snippet.bytes)} of the line.]`;
|
||||
} else {
|
||||
outputText = `[Line 1 is ${formatSize(firstLineBytes)}, exceeds ${formatSize(
|
||||
outputText = `[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes(
|
||||
DEFAULT_MAX_BYTES,
|
||||
)} limit. Unable to display a valid UTF-8 snippet.]`;
|
||||
}
|
||||
@@ -187,7 +187,7 @@ function buildTextOutput(textContent: string): { output: string; lineCount: numb
|
||||
if (truncation.truncatedBy === "lines") {
|
||||
outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`;
|
||||
} else {
|
||||
outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines} (${formatSize(
|
||||
outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines} (${formatBytes(
|
||||
DEFAULT_MAX_BYTES,
|
||||
)} limit). Use offset=${nextOffset} to continue]`;
|
||||
}
|
||||
@@ -247,7 +247,7 @@ async function buildDirectoryListing(absolutePath: string): Promise<{ output: st
|
||||
notices.push(`${DEFAULT_DIR_LIMIT} entries limit reached. Use limit=${DEFAULT_DIR_LIMIT * 2} for more`);
|
||||
}
|
||||
if (truncation.truncated) {
|
||||
notices.push(`${formatSize(DEFAULT_MAX_BYTES)} limit reached`);
|
||||
notices.push(`${formatBytes(DEFAULT_MAX_BYTES)} limit reached`);
|
||||
}
|
||||
if (notices.length > 0) {
|
||||
output += `\n\n[${notices.join(". ")}]`;
|
||||
@@ -313,7 +313,7 @@ export async function generateFileMentionMessages(
|
||||
if (stat.size > MAX_AUTO_READ_IMAGE_BYTES) {
|
||||
files.push({
|
||||
path: resolvedPath,
|
||||
content: `(skipped auto-read: too large, ${formatSize(stat.size)})`,
|
||||
content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`,
|
||||
byteSize: stat.size,
|
||||
skippedReason: "tooLarge",
|
||||
});
|
||||
@@ -349,7 +349,7 @@ export async function generateFileMentionMessages(
|
||||
if (stat.size > MAX_AUTO_READ_TEXT_BYTES) {
|
||||
files.push({
|
||||
path: resolvedPath,
|
||||
content: `(skipped auto-read: too large, ${formatSize(stat.size)})`,
|
||||
content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`,
|
||||
byteSize: stat.size,
|
||||
skippedReason: "tooLarge",
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { formatBytes } from "../../tools/render-utils";
|
||||
import type { RenderResult, SpecialHandler } from "./types";
|
||||
import { finalizeOutput, formatCount, loadPage } from "./types";
|
||||
|
||||
@@ -29,13 +30,6 @@ interface DockerHubTagsResponse {
|
||||
results?: DockerHubTag[];
|
||||
}
|
||||
|
||||
function formatSize(bytes: number): string {
|
||||
if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`;
|
||||
if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`;
|
||||
if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`;
|
||||
return `${bytes}B`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Docker Hub URLs via API
|
||||
*/
|
||||
@@ -131,7 +125,7 @@ export const handleDockerHub: SpecialHandler = async (
|
||||
md += "|-----|------|---------------|--------|\n";
|
||||
|
||||
for (const tag of tags) {
|
||||
const size = tag.full_size ? formatSize(tag.full_size) : "-";
|
||||
const size = tag.full_size ? formatBytes(tag.full_size) : "-";
|
||||
const archs =
|
||||
tag.images
|
||||
?.map(img => img.architecture)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { formatBytes } from "../../tools/render-utils";
|
||||
import type { RenderResult, SpecialHandler } from "./types";
|
||||
import { finalizeOutput, loadPage } from "./types";
|
||||
|
||||
@@ -100,13 +101,6 @@ function extractTagsFromHtml(html: string, baseRef: string): string[] {
|
||||
return Array.from(tags);
|
||||
}
|
||||
|
||||
function formatSize(bytes: number): string {
|
||||
if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`;
|
||||
if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`;
|
||||
if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`;
|
||||
return `${bytes}B`;
|
||||
}
|
||||
|
||||
function buildModelPath(parts: string[]): string {
|
||||
return parts.map(part => encodeURIComponent(part)).join("/");
|
||||
}
|
||||
@@ -227,11 +221,11 @@ export const handleOllama: SpecialHandler = async (
|
||||
let sizeLine: string | null = null;
|
||||
|
||||
if (selectedTag?.size) {
|
||||
sizeLine = formatSize(selectedTag.size);
|
||||
sizeLine = formatBytes(selectedTag.size);
|
||||
} else if (sizes.length > 0) {
|
||||
const minSize = Math.min(...sizes);
|
||||
const maxSize = Math.max(...sizes);
|
||||
sizeLine = minSize === maxSize ? formatSize(minSize) : `${formatSize(minSize)} - ${formatSize(maxSize)}`;
|
||||
sizeLine = minSize === maxSize ? formatBytes(minSize) : `${formatBytes(minSize)} - ${formatBytes(maxSize)}`;
|
||||
}
|
||||
|
||||
let md = `# ${baseRef}\n\n`;
|
||||
|
||||
@@ -13,7 +13,7 @@ export interface ConvertResult {
|
||||
}
|
||||
|
||||
export interface BinaryFetchResult {
|
||||
buffer: Buffer;
|
||||
buffer: Uint8Array;
|
||||
contentType: string;
|
||||
contentDisposition?: string;
|
||||
ok: boolean;
|
||||
@@ -22,7 +22,7 @@ export interface BinaryFetchResult {
|
||||
}
|
||||
|
||||
export async function convertWithMarkitdown(
|
||||
content: Buffer,
|
||||
content: Uint8Array,
|
||||
extensionHint: string,
|
||||
timeout: number,
|
||||
userSignal?: AbortSignal,
|
||||
@@ -68,7 +68,7 @@ export async function convertWithMarkitdown(
|
||||
}
|
||||
}
|
||||
|
||||
const kEmptyBuffer = Buffer.alloc(0);
|
||||
const kEmptyBuffer = new Uint8Array(0);
|
||||
|
||||
export async function fetchBinary(url: string, timeout: number, userSignal?: AbortSignal): Promise<BinaryFetchResult> {
|
||||
if (userSignal?.aborted) {
|
||||
@@ -115,7 +115,7 @@ export async function fetchBinary(url: string, timeout: number, userSignal?: Abo
|
||||
}
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const buffer = await response.bytes();
|
||||
if (buffer.length > MAX_BYTES) {
|
||||
return {
|
||||
buffer: kEmptyBuffer,
|
||||
|
||||
@@ -4,7 +4,7 @@ import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { _resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
||||
import { executeBash } from "@oh-my-pi/pi-coding-agent/exec/bash-executor";
|
||||
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate";
|
||||
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output";
|
||||
import * as shellSnapshot from "@oh-my-pi/pi-coding-agent/utils/shell-snapshot";
|
||||
|
||||
function makeTempDir(): string {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { executePythonWithKernel, type PythonKernelExecutor } from "@oh-my-pi/pi-coding-agent/ipy/executor";
|
||||
import type { KernelExecuteOptions, KernelExecuteResult } from "@oh-my-pi/pi-coding-agent/ipy/kernel";
|
||||
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate";
|
||||
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output";
|
||||
|
||||
class FakeKernel implements PythonKernelExecutor {
|
||||
private result: KernelExecuteResult;
|
||||
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
type PreludeHelper,
|
||||
PythonKernel,
|
||||
} from "@oh-my-pi/pi-coding-agent/ipy/kernel";
|
||||
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/tools/truncate";
|
||||
import { DEFAULT_MAX_BYTES } from "@oh-my-pi/pi-coding-agent/session/streaming-output";
|
||||
import { TempDir } from "@oh-my-pi/pi-utils";
|
||||
|
||||
class FakeKernel implements PythonKernelExecutor {
|
||||
|
||||
@@ -51,6 +51,15 @@ describe("ModelRegistry runtime provider registration", () => {
|
||||
const streamSimple: NonNullable<ProviderConfigInput["streamSimple"]> = () =>
|
||||
({}) as unknown as AssistantMessageEventStream;
|
||||
|
||||
test("loads built-in GitLab Duo models and OAuth provider metadata", () => {
|
||||
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
||||
const model = registry.find("gitlab-duo", "claude-sonnet-4-5-20250929");
|
||||
|
||||
expect(model).toBeDefined();
|
||||
expect(model?.api).toBe("anthropic-messages");
|
||||
expect(getOAuthProviders().some(provider => provider.id === "gitlab-duo")).toBe(true);
|
||||
});
|
||||
|
||||
test("validates provider config before mutating custom API state", () => {
|
||||
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
||||
const beforeAnthropicCount = registry.getAll().filter(model => model.provider === "anthropic").length;
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { ArtifactManager } from "../src/session/artifacts";
|
||||
import {
|
||||
allocateOutputArtifact,
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_COLUMN,
|
||||
DEFAULT_MAX_LINES,
|
||||
formatHeadTruncationNotice,
|
||||
formatTailTruncationNotice,
|
||||
OutputSink,
|
||||
TailBuffer,
|
||||
truncateHead,
|
||||
truncateHeadBytes,
|
||||
truncateLine,
|
||||
truncateTail,
|
||||
truncateTailBytes,
|
||||
} from "../src/session/streaming-output";
|
||||
import type { ToolSession } from "../src/tools";
|
||||
|
||||
const createdTempDirs: string[] = [];
|
||||
|
||||
async function createTempDir(): Promise<string> {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "streaming-output-test-"));
|
||||
createdTempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function byteLength(text: string): number {
|
||||
return Buffer.byteLength(text, "utf-8");
|
||||
}
|
||||
|
||||
function createSession(overrides: Partial<ToolSession> = {}): ToolSession {
|
||||
return {
|
||||
cwd: "/",
|
||||
hasUI: false,
|
||||
getSessionFile: () => null,
|
||||
getSessionSpawns: () => null,
|
||||
settings: {} as ToolSession["settings"],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const dir of createdTempDirs.splice(0)) {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("streaming-output exports", () => {
|
||||
test("exports expected default limits", () => {
|
||||
expect(DEFAULT_MAX_LINES).toBe(3000);
|
||||
expect(DEFAULT_MAX_BYTES).toBe(50 * 1024);
|
||||
expect(DEFAULT_MAX_COLUMN).toBe(1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe("truncateTailBytes", () => {
|
||||
test("returns source when already under limit", () => {
|
||||
const text = "hello";
|
||||
expect(truncateTailBytes(text, 10)).toEqual({ text: "hello", bytes: 5 });
|
||||
});
|
||||
|
||||
test("truncates from end without breaking UTF-8 boundaries", () => {
|
||||
const text = "a😀b";
|
||||
const result = truncateTailBytes(text, 4);
|
||||
expect(result).toEqual({ text: "b", bytes: 1 });
|
||||
expect(result.text).not.toContain("\uFFFD");
|
||||
});
|
||||
|
||||
test("accepts Uint8Array input", () => {
|
||||
const bytes = new TextEncoder().encode("abc😀");
|
||||
const result = truncateTailBytes(bytes, 4);
|
||||
expect(result.text).toBe("😀");
|
||||
expect(result.bytes).toBe(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("truncateHeadBytes", () => {
|
||||
test("returns source when already under limit", () => {
|
||||
const text = "hello";
|
||||
expect(truncateHeadBytes(text, 10)).toEqual({ text: "hello", bytes: 5 });
|
||||
});
|
||||
|
||||
test("truncates from start without breaking UTF-8 boundaries", () => {
|
||||
const text = "a😀b";
|
||||
const result = truncateHeadBytes(text, 2);
|
||||
expect(result).toEqual({ text: "a", bytes: 1 });
|
||||
expect(result.text).not.toContain("\uFFFD");
|
||||
});
|
||||
|
||||
test("returns empty when maxBytes is zero", () => {
|
||||
const result = truncateHeadBytes("abc", 0);
|
||||
expect(result).toEqual({ text: "", bytes: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("truncateHead", () => {
|
||||
test("returns unmodified content when within limits", () => {
|
||||
const content = "a\nb";
|
||||
const result = truncateHead(content, { maxLines: 10, maxBytes: 20 });
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.content).toBe(content);
|
||||
expect(result.truncatedBy).toBe(null);
|
||||
});
|
||||
|
||||
test("handles first line exceeding byte limit", () => {
|
||||
const result = truncateHead("abcdef\nnext", { maxBytes: 3, maxLines: 10 });
|
||||
expect(result.content).toBe("");
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.firstLineExceedsLimit).toBe(true);
|
||||
});
|
||||
|
||||
|
||||
test("includes first line when text fits exact byte budget", () => {
|
||||
const result = truncateHead("abc\nx", { maxBytes: 3, maxLines: 10 });
|
||||
expect(result.content).toBe("abc");
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.firstLineExceedsLimit).toBe(false);
|
||||
expect(result.outputBytes).toBe(byteLength("abc"));
|
||||
});
|
||||
test("truncates by line count", () => {
|
||||
const result = truncateHead("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
|
||||
expect(result.content).toBe("l1\nl2");
|
||||
expect(result.truncatedBy).toBe("lines");
|
||||
expect(result.outputLines).toBe(2);
|
||||
});
|
||||
|
||||
test("truncates by byte budget using complete lines", () => {
|
||||
const result = truncateHead("12345\nabc\nz", { maxLines: 10, maxBytes: 7 });
|
||||
expect(result.content).toBe("12345");
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.lastLinePartial).toBe(false);
|
||||
expect(result.outputBytes).toBe(byteLength("12345"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("truncateTail", () => {
|
||||
test("returns unmodified content when within limits", () => {
|
||||
const content = "a\nb";
|
||||
const result = truncateTail(content, { maxLines: 10, maxBytes: 20 });
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.content).toBe(content);
|
||||
expect(result.truncatedBy).toBe(null);
|
||||
});
|
||||
|
||||
test("truncates by line count", () => {
|
||||
const result = truncateTail("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
|
||||
expect(result.content).toBe("l2\nl3");
|
||||
expect(result.truncatedBy).toBe("lines");
|
||||
expect(result.outputLines).toBe(2);
|
||||
});
|
||||
|
||||
test("truncates by byte budget while preserving line boundaries", () => {
|
||||
const result = truncateTail("aaa\nbbbb\ncc", { maxLines: 10, maxBytes: 6 });
|
||||
expect(result.content).toBe("cc");
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.lastLinePartial).toBe(false);
|
||||
});
|
||||
|
||||
test("returns partial single line when last line exceeds byte limit", () => {
|
||||
const result = truncateTail("abcdefghij", { maxLines: 10, maxBytes: 4 });
|
||||
expect(result.content).toBe("ghij");
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.lastLinePartial).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("truncateLine", () => {
|
||||
test("does not truncate short lines", () => {
|
||||
expect(truncateLine("hello", 10)).toEqual({ text: "hello", wasTruncated: false });
|
||||
});
|
||||
|
||||
test("truncates long lines with ellipsis", () => {
|
||||
expect(truncateLine("abcdefgh", 5)).toEqual({ text: "abcde…", wasTruncated: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("TailBuffer", () => {
|
||||
test("keeps trailing bytes under budget", () => {
|
||||
const tail = new TailBuffer(5);
|
||||
tail.append("abc");
|
||||
tail.append("def");
|
||||
expect(tail.text()).toBe("bcdef");
|
||||
expect(tail.bytes()).toBe(5);
|
||||
});
|
||||
|
||||
test("handles multibyte data and empty appends", () => {
|
||||
const tail = new TailBuffer(4);
|
||||
tail.append("");
|
||||
tail.append("😀");
|
||||
tail.append("x");
|
||||
expect(tail.text()).toBe("x");
|
||||
expect(tail.bytes()).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("OutputSink", () => {
|
||||
test("tracks totals and adds notice in dump", async () => {
|
||||
const sink = new OutputSink();
|
||||
await sink.push("hello\nworld");
|
||||
const dumped = await sink.dump("notice");
|
||||
|
||||
expect(dumped.output).toBe("[notice]\nhello\nworld");
|
||||
expect(dumped.truncated).toBe(false);
|
||||
expect(dumped.totalLines).toBe(2);
|
||||
expect(dumped.totalBytes).toBe(byteLength("hello\nworld"));
|
||||
expect(dumped.outputLines).toBe(2);
|
||||
expect(dumped.outputBytes).toBe(byteLength("hello\nworld"));
|
||||
});
|
||||
|
||||
|
||||
test("counts lines correctly when chunks contain no newlines", async () => {
|
||||
const sink = new OutputSink();
|
||||
await sink.push("abc");
|
||||
await sink.push("def");
|
||||
const dumped = await sink.dump();
|
||||
|
||||
expect(dumped.totalLines).toBe(1);
|
||||
expect(dumped.outputLines).toBe(1);
|
||||
});
|
||||
|
||||
test("counts all newline boundaries across chunk splits", async () => {
|
||||
const sink = new OutputSink();
|
||||
await sink.push("a\n");
|
||||
await sink.push("b\n\n");
|
||||
await sink.push("c");
|
||||
const dumped = await sink.dump();
|
||||
|
||||
expect(dumped.output).toBe("a\nb\n\nc");
|
||||
expect(dumped.totalLines).toBe(4);
|
||||
expect(dumped.outputLines).toBe(4);
|
||||
});
|
||||
test("invokes onChunk callback with sanitized text", async () => {
|
||||
const chunks: string[] = [];
|
||||
const sink = new OutputSink({ onChunk: chunk => chunks.push(chunk) });
|
||||
await sink.push("abc");
|
||||
await sink.push("def");
|
||||
expect(chunks).toEqual(["abc", "def"]);
|
||||
});
|
||||
|
||||
test("truncates in-memory output when spill threshold is exceeded", async () => {
|
||||
const sink = new OutputSink({ spillThreshold: 5 });
|
||||
await sink.push("abc");
|
||||
await sink.push("def");
|
||||
|
||||
const dumped = await sink.dump();
|
||||
expect(dumped.truncated).toBe(true);
|
||||
expect(dumped.output).toBe("bcdef");
|
||||
expect(dumped.totalBytes).toBe(6);
|
||||
expect(dumped.outputBytes).toBe(5);
|
||||
});
|
||||
|
||||
test("spills full output to artifact file when artifact path is provided", async () => {
|
||||
const dir = await createTempDir();
|
||||
const artifactPath = path.join(dir, "output.log");
|
||||
const sink = new OutputSink({
|
||||
artifactPath,
|
||||
artifactId: "artifact-1",
|
||||
spillThreshold: 5,
|
||||
});
|
||||
|
||||
await sink.push("abc");
|
||||
await sink.push("def");
|
||||
const dumped = await sink.dump();
|
||||
const artifactText = await Bun.file(artifactPath).text();
|
||||
|
||||
expect(dumped.truncated).toBe(true);
|
||||
expect(dumped.artifactId).toBe("artifact-1");
|
||||
expect(artifactText).toBe("abcdef");
|
||||
expect(dumped.output).toBe("bcdef");
|
||||
});
|
||||
|
||||
test("createInput decodes streamed UTF-8 chunks correctly", async () => {
|
||||
const sink = new OutputSink();
|
||||
const writer = sink.createInput().getWriter();
|
||||
const bytes = new TextEncoder().encode("😀X");
|
||||
|
||||
await writer.write(bytes.subarray(0, 2));
|
||||
await writer.write(bytes.subarray(2));
|
||||
await writer.close();
|
||||
|
||||
const dumped = await sink.dump();
|
||||
expect(dumped.output).toBe("😀X");
|
||||
expect(dumped.totalBytes).toBe(byteLength("😀X"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("allocateOutputArtifact", () => {
|
||||
test("returns empty object when session has no artifact manager", async () => {
|
||||
const session = createSession({ getArtifactManager: undefined });
|
||||
expect(await allocateOutputArtifact(session, "bash")).toEqual({});
|
||||
});
|
||||
|
||||
test("allocates artifact path via session artifact manager", async () => {
|
||||
const dir = await createTempDir();
|
||||
const sessionFile = path.join(dir, "session.jsonl");
|
||||
await Bun.write(sessionFile, "");
|
||||
const manager = new ArtifactManager(sessionFile);
|
||||
const session = createSession({ getArtifactManager: () => manager });
|
||||
|
||||
const result = await allocateOutputArtifact(session, "bash");
|
||||
expect(result.id).toBeDefined();
|
||||
expect(result.path).toContain(`${path.sep}${result.id}.bash.log`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("truncation notice formatting", () => {
|
||||
test("formatTailTruncationNotice returns empty string for non-truncated results", () => {
|
||||
const truncation = truncateTail("a\nb", { maxLines: 10, maxBytes: 50 });
|
||||
expect(formatTailTruncationNotice(truncation)).toBe("");
|
||||
});
|
||||
|
||||
test("formatTailTruncationNotice supports partial-line notices", () => {
|
||||
const truncation = truncateTail("abcdefghij", { maxLines: 10, maxBytes: 4 });
|
||||
const notice = formatTailTruncationNotice(truncation, {
|
||||
fullOutputPath: "/tmp/full.log",
|
||||
originalContent: "abcdefghij",
|
||||
suffix: " [suffix]",
|
||||
});
|
||||
expect(notice).toBe("\n\n[Showing last 4B of line 1 (line is 10B). Full output: /tmp/full.log [suffix]]");
|
||||
});
|
||||
|
||||
test("formatTailTruncationNotice supports line-based and byte-based notices", () => {
|
||||
const lineTruncation = truncateTail("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
|
||||
expect(formatTailTruncationNotice(lineTruncation)).toBe("\n\n[Showing lines 2-3 of 3]");
|
||||
|
||||
const byteTruncation = truncateTail("aaa\nbbbb\ncc", { maxLines: 10, maxBytes: 6 });
|
||||
expect(formatTailTruncationNotice(byteTruncation)).toBe("\n\n[Showing lines 3-3 of 3 (6B limit)]");
|
||||
});
|
||||
|
||||
test("formatHeadTruncationNotice returns empty string for non-truncated results", () => {
|
||||
const truncation = truncateHead("a\nb", { maxLines: 10, maxBytes: 50 });
|
||||
expect(formatHeadTruncationNotice(truncation)).toBe("");
|
||||
});
|
||||
|
||||
test("formatHeadTruncationNotice supports line and byte truncation", () => {
|
||||
const lineTruncation = truncateHead("l1\nl2\nl3", { maxLines: 2, maxBytes: 100 });
|
||||
expect(formatHeadTruncationNotice(lineTruncation)).toBe("\n\n[Showing lines 1-2 of 3. Use offset=3 to continue]");
|
||||
|
||||
const byteTruncation = truncateHead("12345\nabc\nz", { maxLines: 10, maxBytes: 7 });
|
||||
expect(
|
||||
formatHeadTruncationNotice(byteTruncation, {
|
||||
startLine: 100,
|
||||
totalFileLines: 500,
|
||||
}),
|
||||
).toBe("\n\n[Showing lines 100-100 of 500 (7B limit). Use offset=101 to continue]");
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
||||
import { EditTool } from "@oh-my-pi/pi-coding-agent/patch";
|
||||
import { ArtifactManager } from "@oh-my-pi/pi-coding-agent/session/artifacts";
|
||||
import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools";
|
||||
import { BashTool } from "@oh-my-pi/pi-coding-agent/tools/bash";
|
||||
import { FindTool } from "@oh-my-pi/pi-coding-agent/tools/find";
|
||||
@@ -25,11 +26,13 @@ function getTextOutput(result: any): string {
|
||||
|
||||
function createTestToolSession(cwd: string): ToolSession {
|
||||
const sessionFile = path.join(cwd, "session.jsonl");
|
||||
const artifactManager = new ArtifactManager(sessionFile);
|
||||
return {
|
||||
cwd,
|
||||
hasUI: false,
|
||||
getSessionFile: () => sessionFile,
|
||||
getSessionSpawns: () => "*",
|
||||
getArtifactManager: () => artifactManager,
|
||||
settings: Settings.isolated(),
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user