From 2e33e436b4025e7f030d3d24048065ff22d14f21 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 14 Feb 2026 03:21:32 +0100 Subject: [PATCH] feat: switched to native text sanitization, removed Bun.stripANSI - Added `sanitizeText` function to pi-natives that strips ANSI escape sequences, removes control characters and lone surrogates, and normalizes line endings. - Moved `sanitizeText` function from `@oh-my-pi/pi-utils` to `@oh-my-pi/pi-natives` for better code organization and native performance. - Added line length clamping (4000 characters) to bash and Python execution output to prevent excessively long lines. - Replaced internal `#normalizeOutput` methods with `sanitizeText` utility function in bash and Python execution components. - Fixed bash interactive tool to gracefully handle malformed output chunks by normalizing them with `sanitizeText`. - Simplified documentation by removing WASM terminology from package descriptions and comments. --- AGENTS.md | 40 ++----- crates/pi-natives/src/text.rs | 102 ++++++++++++++++++ packages/coding-agent/CHANGELOG.md | 5 +- packages/coding-agent/src/lsp/render.ts | 2 +- .../src/modes/components/bash-execution.ts | 25 +++-- .../src/modes/components/python-execution.ts | 23 ++-- .../src/modes/components/tool-execution.ts | 3 +- .../src/session/streaming-output.ts | 2 +- .../src/tools/bash-interactive.ts | 16 +-- .../test/tools/python-renderer.test.ts | 5 +- packages/natives/CHANGELOG.md | 3 + packages/natives/src/index.ts | 1 + packages/natives/src/native.ts | 2 + packages/natives/src/text/index.ts | 2 +- packages/natives/src/text/types.ts | 5 + packages/natives/test/native.test.ts | 29 ++++- packages/tui/bench/sanitize.ts | 83 ++++++++++++++ packages/utils/src/stream.ts | 50 --------- packages/utils/test/stream.test.ts | 27 +---- 19 files changed, 290 insertions(+), 135 deletions(-) create mode 100644 packages/tui/bench/sanitize.ts diff --git a/AGENTS.md b/AGENTS.md index 2650e7b86..06ecf8ba8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,16 +8,16 @@ This repo contains multiple packages, but **`packages/coding-agent/`** is the pr ### Package Structure -| Package | Description | -| ----------------------- | ------------------------------------------------------ | -| `packages/ai` | Multi-provider LLM client with streaming support | -| `packages/agent` | Agent runtime with tool calling and state management | -| `packages/coding-agent` | Main CLI application (primary focus) | -| `packages/tui` | Terminal UI library with differential rendering | -| `packages/natives` | WASM bindings for native text/image/grep operations | -| `packages/stats` | Local observability dashboard (`omp stats`) | -| `packages/utils` | Shared utilities (logger, streams, temp files) | -| `crates/pi-natives` | Rust WASM crate for performance-critical text/grep ops | +| Package | Description | +| ----------------------- | ---------------------------------------------------- | +| `packages/ai` | Multi-provider LLM client with streaming support | +| `packages/agent` | Agent runtime with tool calling and state management | +| `packages/coding-agent` | Main CLI application (primary focus) | +| `packages/tui` | Terminal UI library with differential rendering | +| `packages/natives` | bindings for native text/image/grep operations | +| `packages/stats` | Local observability dashboard (`omp stats`) | +| `packages/utils` | Shared utilities (logger, streams, temp files) | +| `crates/pi-natives` | Rust crate for performance-critical text/grep ops | ## Code Quality @@ -79,26 +79,6 @@ This repo contains multiple packages, but **`packages/coding-agent/`** is the pr const { promise, resolve, reject } = Promise.withResolvers(); ``` -## WASM Bindgen Resources - -**Use `using` declarations** for wasm-bindgen objects — they implement `Symbol.dispose`: - -```typescript -// BAD: Manual try/finally with .free() -const image = PhotonImage.new_from_byteslice(bytes); -try { - return image.get_bytes(); -} finally { - image.free(); -} - -// GOOD: Automatic cleanup via Symbol.dispose -using image = PhotonImage.new_from_byteslice(bytes); -return image.get_bytes(); -``` - -This applies to all wasm-bindgen generated classes (`CompiledPattern`, `PhotonImage`, etc.). - ## Bun Over Node This project uses Bun. Use Bun APIs where they provide a cleaner alternative; use `node:fs` for operations Bun doesn't cover. diff --git a/crates/pi-natives/src/text.rs b/crates/pi-natives/src/text.rs index 989bec8e2..27e1212dd 100644 --- a/crates/pi-natives/src/text.rs +++ b/crates/pi-natives/src/text.rs @@ -322,6 +322,25 @@ fn ansi_seq_len_u16(data: &[u16], pos: usize) -> Option { } None }, + 0x50 | 0x58 | 0x5e | 0x5f => { + // 'P' DCS, 'X' SOS, '^' PM, '_' APC (terminated by ST) + for (i, &b) in data[pos + 2..].iter().enumerate() { + if b == ESC && data.get(pos + 2 + i + 1) == Some(&0x5c) { + return Some(i + 4); + } + } + None + }, + 0x20..=0x2f => { + // ESC + intermediates + final byte + for (i, b) in data[pos + 2..].iter().enumerate() { + if (0x30..=0x7e).contains(b) { + return Some(i + 3); + } + } + None + }, + 0x40..=0x7e => Some(2), _ => None, } } @@ -1191,6 +1210,89 @@ pub fn extract_segments( }) } +// ============================================================================ +// sanitizeText +// ============================================================================ + +/// Strip ANSI escape sequences, remove control characters / lone surrogates, +/// and normalize line endings. +#[napi(js_name = "sanitizeText")] +pub fn sanitize_text(text: JsString<'_>) -> Result, Utf16String>> { + let original = text; + let text_u16 = text.into_utf16()?; + let data = text_u16.as_slice(); + + let mut did_change = false; + let mut out: Vec = Vec::new(); + let mut last = 0usize; + let mut i = 0usize; + let len = data.len(); + + while i < len { + let u = data[i]; + + // Allow tab + newline; normalize CR by removing it. + if u == 0x09 || u == 0x0a { + i += 1; + continue; + } + + let mut remove_len = if u == ESC + && let Some(seq_len) = ansi_seq_len_u16(data, i) + { + seq_len + } else { + 0usize + }; + + if remove_len == 0 { + // Drop CR to normalize line endings. + if u == 0x0d { + remove_len = 1; + } else if u <= 0x1f || u == 0x7f || (0x80..=0x9f).contains(&u) { + // C0 + DEL + C1 controls. + remove_len = 1; + } else if (0xd800..=0xdbff).contains(&u) { + // High surrogate: keep only if followed by a valid low surrogate. + if i + 1 < len { + let lo = data[i + 1]; + if (0xdc00..=0xdfff).contains(&lo) { + i += 2; + continue; + } + } + remove_len = 1; + } else if (0xdc00..=0xdfff).contains(&u) { + // Lone low surrogate. + remove_len = 1; + } + } + + if remove_len == 0 { + i += 1; + continue; + } + + if !did_change { + did_change = true; + out = Vec::with_capacity(len); + } + if last != i { + out.extend_from_slice(&data[last..i]); + } + i += remove_len; + last = i; + } + + if !did_change { + return Ok(Either::A(original)); + } + if last < len { + out.extend_from_slice(&data[last..]); + } + Ok(Either::B(build_utf16_string(out))) +} + // ============================================================================ // visibleWidth // ============================================================================ diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 46f2d10b1..e80a368a7 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,13 +1,16 @@ # Changelog ## [Unreleased] - ### Changed +- Moved `sanitizeText` function from `@oh-my-pi/pi-utils` to `@oh-my-pi/pi-natives` for better code organization +- Replaced internal `#normalizeOutput` methods with `sanitizeText` utility function in bash and Python execution components +- Added line length clamping (4000 characters) to bash and Python execution output to prevent display of excessively long lines - Modified memory storage to isolate memories by project working directory, preventing cross-project memory contamination ### Fixed +- Fixed bash interactive tool to gracefully handle malformed output chunks by normalizing them before display - Fixed fetch tool incorrectly treating HTML content as plain text or markdown - Fixed output truncation notice displaying incorrect byte limit when maxBytes differs from outputBytes - Fixed Cloudflare returning corrupted bytes when compression is negotiated in web scraper requests diff --git a/packages/coding-agent/src/lsp/render.ts b/packages/coding-agent/src/lsp/render.ts index 4e8fe144d..c45c1b788 100644 --- a/packages/coding-agent/src/lsp/render.ts +++ b/packages/coding-agent/src/lsp/render.ts @@ -282,7 +282,7 @@ function renderHover( } /** - * Syntax highlight code using native WASM highlighter. + * Syntax highlight code using native highlighter. */ function highlightCode(codeText: string, language: string, theme: Theme): string[] { const validLang = language && supportsLanguage(language) ? language : undefined; diff --git a/packages/coding-agent/src/modes/components/bash-execution.ts b/packages/coding-agent/src/modes/components/bash-execution.ts index db31224fd..4a15207a2 100644 --- a/packages/coding-agent/src/modes/components/bash-execution.ts +++ b/packages/coding-agent/src/modes/components/bash-execution.ts @@ -1,6 +1,8 @@ /** * Component for displaying bash command execution with streaming output. */ + +import { sanitizeText } from "@oh-my-pi/pi-natives"; import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui"; import { getSymbolTheme, theme } from "../../modes/theme/theme"; import type { TruncationMeta } from "../../tools/output-meta"; @@ -10,6 +12,7 @@ import { truncateToVisualLines } from "./visual-truncate"; // Preview line limit when not expanded (matches tool execution behavior) const PREVIEW_LINES = 20; +const MAX_DISPLAY_LINE_CHARS = 4000; export class BashExecutionComponent extends Container { #outputLines: string[] = []; @@ -73,13 +76,15 @@ export class BashExecutionComponent extends Container { } appendOutput(chunk: string): void { - const clean = this.#normalizeOutput(chunk); + const clean = sanitizeText(chunk); // Append to output lines - const newLines = clean.split("\n"); + const newLines = clean.split("\n").map(line => this.#clampDisplayLine(line)); if (this.#outputLines.length > 0 && newLines.length > 0) { // Append first chunk to last line (incomplete line continuation) - this.#outputLines[this.#outputLines.length - 1] += newLines[0]; + this.#outputLines[this.#outputLines.length - 1] = this.#clampDisplayLine( + `${this.#outputLines[this.#outputLines.length - 1]}${newLines[0]}`, + ); this.#outputLines.push(...newLines.slice(1)); } else { this.#outputLines.push(...newLines); @@ -184,15 +189,17 @@ export class BashExecutionComponent extends Container { } } - #normalizeOutput(text: string): string { - // Strip ANSI codes and normalize line endings - // Note: binary data is already sanitized in tui-renderer.ts executeBashCommand - return Bun.stripANSI(text).replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + #clampDisplayLine(line: string): string { + if (line.length <= MAX_DISPLAY_LINE_CHARS) { + return line; + } + const omitted = line.length - MAX_DISPLAY_LINE_CHARS; + return `${line.slice(0, MAX_DISPLAY_LINE_CHARS)}… [${omitted} chars omitted]`; } #setOutput(output: string): void { - const clean = this.#normalizeOutput(output); - this.#outputLines = clean ? clean.split("\n") : []; + const clean = sanitizeText(output); + this.#outputLines = clean ? clean.split("\n").map(line => this.#clampDisplayLine(line)) : []; } /** diff --git a/packages/coding-agent/src/modes/components/python-execution.ts b/packages/coding-agent/src/modes/components/python-execution.ts index f2d3a90d3..d9a69d808 100644 --- a/packages/coding-agent/src/modes/components/python-execution.ts +++ b/packages/coding-agent/src/modes/components/python-execution.ts @@ -2,6 +2,8 @@ * Component for displaying user-initiated Python execution with streaming output. * Shares the same kernel session as the agent's Python tool. */ + +import { sanitizeText } from "@oh-my-pi/pi-natives"; import { Container, Loader, Spacer, Text, type TUI } from "@oh-my-pi/pi-tui"; import { getSymbolTheme, highlightCode, theme } from "../../modes/theme/theme"; import type { TruncationMeta } from "../../tools/output-meta"; @@ -10,6 +12,7 @@ import { DynamicBorder } from "./dynamic-border"; import { truncateToVisualLines } from "./visual-truncate"; const PREVIEW_LINES = 20; +const MAX_DISPLAY_LINE_CHARS = 4000; export class PythonExecutionComponent extends Container { #outputLines: string[] = []; @@ -70,11 +73,13 @@ export class PythonExecutionComponent extends Container { } appendOutput(chunk: string): void { - const clean = this.#normalizeOutput(chunk); + const clean = sanitizeText(chunk); - const newLines = clean.split("\n"); + const newLines = clean.split("\n").map(line => this.#clampDisplayLine(line)); if (this.#outputLines.length > 0 && newLines.length > 0) { - this.#outputLines[this.#outputLines.length - 1] += newLines[0]; + this.#outputLines[this.#outputLines.length - 1] = this.#clampDisplayLine( + `${this.#outputLines[this.#outputLines.length - 1]}${newLines[0]}`, + ); this.#outputLines.push(...newLines.slice(1)); } else { this.#outputLines.push(...newLines); @@ -168,13 +173,17 @@ export class PythonExecutionComponent extends Container { } } - #normalizeOutput(text: string): string { - return Bun.stripANSI(text).replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + #clampDisplayLine(line: string): string { + if (line.length <= MAX_DISPLAY_LINE_CHARS) { + return line; + } + const omitted = line.length - MAX_DISPLAY_LINE_CHARS; + return `${line.slice(0, MAX_DISPLAY_LINE_CHARS)}… [${omitted} chars omitted]`; } #setOutput(output: string): void { - const clean = this.#normalizeOutput(output); - this.#outputLines = clean ? clean.split("\n") : []; + const clean = sanitizeText(output); + this.#outputLines = clean ? clean.split("\n").map(line => this.#clampDisplayLine(line)) : []; } getOutput(): string { diff --git a/packages/coding-agent/src/modes/components/tool-execution.ts b/packages/coding-agent/src/modes/components/tool-execution.ts index 5768ed40a..3fa9b6c30 100644 --- a/packages/coding-agent/src/modes/components/tool-execution.ts +++ b/packages/coding-agent/src/modes/components/tool-execution.ts @@ -1,4 +1,5 @@ import type { AgentTool } from "@oh-my-pi/pi-agent-core"; +import { sanitizeText } from "@oh-my-pi/pi-natives"; import { Box, type Component, @@ -12,7 +13,7 @@ import { Text, type TUI, } from "@oh-my-pi/pi-tui"; -import { logger, sanitizeText } from "@oh-my-pi/pi-utils"; +import { logger } from "@oh-my-pi/pi-utils"; import { getProjectDir } from "@oh-my-pi/pi-utils/dirs"; import type { Theme } from "../../modes/theme/theme"; import { theme } from "../../modes/theme/theme"; diff --git a/packages/coding-agent/src/session/streaming-output.ts b/packages/coding-agent/src/session/streaming-output.ts index 62d15f103..0cd01a59a 100644 --- a/packages/coding-agent/src/session/streaming-output.ts +++ b/packages/coding-agent/src/session/streaming-output.ts @@ -1,4 +1,4 @@ -import { sanitizeText } from "@oh-my-pi/pi-utils"; +import { sanitizeText } from "@oh-my-pi/pi-natives"; import { DEFAULT_MAX_BYTES } from "../tools/truncate"; export interface OutputSummary { diff --git a/packages/coding-agent/src/tools/bash-interactive.ts b/packages/coding-agent/src/tools/bash-interactive.ts index 10a7b7a9a..c306a2d51 100644 --- a/packages/coding-agent/src/tools/bash-interactive.ts +++ b/packages/coding-agent/src/tools/bash-interactive.ts @@ -1,5 +1,5 @@ import type { AgentToolContext } from "@oh-my-pi/pi-agent-core"; -import { type PtyRunResult, PtySession } from "@oh-my-pi/pi-natives"; +import { type PtyRunResult, PtySession, sanitizeText } from "@oh-my-pi/pi-natives"; import { type Component, matchesKey, @@ -23,9 +23,8 @@ export interface BashInteractiveResult extends OutputSummary { } function normalizeCaptureChunk(chunk: string): string { - const noAnsi = Bun.stripANSI(chunk); - const normalized = noAnsi.replace(/\r\n/gu, "\n").replace(/\r/gu, "\n"); - return normalized.replace(/[\x00-\x08\x0B-\x1F\x7F]/gu, ""); + const normalized = chunk.replace(/\r\n/gu, "\n").replace(/\r/gu, "\n"); + return sanitizeText(normalized); } const XtermTerminal = xterm.Terminal; @@ -168,7 +167,7 @@ class BashInteractiveOverlayComponent implements Component { const visibleLines: string[] = []; for (let i = 0; i < maxContentRows; i++) { const line = buffer.getLine(viewportY + i)?.translateToString(true) ?? ""; - visibleLines.push(truncateToWidth(replaceTabs(line), innerWidth)); + visibleLines.push(truncateToWidth(replaceTabs(sanitizeText(line)), innerWidth)); } return visibleLines; } @@ -350,7 +349,12 @@ export async function runInteractiveBashPty( }, (err, chunk) => { if (err || !chunk) return; - component.appendOutput(chunk); + try { + component.appendOutput(chunk); + } catch { + const normalizedChunk = normalizeCaptureChunk(chunk); + component.appendOutput(normalizedChunk); + } const normalizedChunk = normalizeCaptureChunk(chunk); pendingChunks = pendingChunks.then(() => sink.push(normalizedChunk)).catch(() => {}); tui.requestRender(); diff --git a/packages/coding-agent/test/tools/python-renderer.test.ts b/packages/coding-agent/test/tools/python-renderer.test.ts index b652f3d2b..2a3ddbbba 100644 --- a/packages/coding-agent/test/tools/python-renderer.test.ts +++ b/packages/coding-agent/test/tools/python-renderer.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "bun:test"; import { getThemeByName } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; import { pythonToolRenderer } from "@oh-my-pi/pi-coding-agent/tools/python"; +import { sanitizeText } from "@oh-my-pi/pi-natives"; describe("pythonToolRenderer", () => { it("renders truncated output when collapsed and full output when expanded", async () => { @@ -31,13 +32,13 @@ describe("pythonToolRenderer", () => { { expanded: false, isPartial: false, renderContext: { previewLines: 2 } }, uiTheme, ); - const collapsedLines = Bun.stripANSI(collapsed.render(80).join("\n")); + const collapsedLines = sanitizeText(collapsed.render(80).join("\n")); expect(collapsedLines).toContain("line 4"); expect(collapsedLines).not.toContain("line 1"); expect(collapsedLines).toContain("more lines"); const expanded = pythonToolRenderer.renderResult(result, { expanded: true, isPartial: false }, uiTheme); - const expandedLines = Bun.stripANSI(expanded.render(80).join("\n")); + const expandedLines = sanitizeText(expanded.render(80).join("\n")); expect(expandedLines).toContain("line 1"); expect(expandedLines).toContain("line 4"); expect(expandedLines).not.toContain("more lines"); diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index a76108d83..0cb5ae30b 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog ## [Unreleased] +### Added + +- Exported `sanitizeText` function to strip ANSI codes, remove binary garbage, and normalize line endings in text output ## [12.1.0] - 2026-02-13 ### Added diff --git a/packages/natives/src/index.ts b/packages/natives/src/index.ts index 6a9a4fbc9..3f0f89e62 100644 --- a/packages/natives/src/index.ts +++ b/packages/natives/src/index.ts @@ -55,6 +55,7 @@ export { type ExtractSegmentsResult, extractSegments, type SliceWithWidthResult, + sanitizeText, sliceWithWidth, truncateToWidth, visibleWidth, diff --git a/packages/natives/src/native.ts b/packages/natives/src/native.ts index dea87c906..5c97053cb 100644 --- a/packages/natives/src/native.ts +++ b/packages/natives/src/native.ts @@ -170,6 +170,8 @@ function validateNative(bindings: NativeBindings, source: string): void { checkFn("supportsLanguage"); checkFn("getSupportedLanguages"); checkFn("truncateToWidth"); + checkFn("sanitizeText"); + checkFn("wrapTextWithAnsi"); checkFn("sliceWithWidth"); checkFn("extractSegments"); diff --git a/packages/natives/src/text/index.ts b/packages/natives/src/text/index.ts index 73d1df1ea..4d1f4c822 100644 --- a/packages/natives/src/text/index.ts +++ b/packages/natives/src/text/index.ts @@ -41,4 +41,4 @@ export function sliceWithWidth(line: string, startCol: number, length: number, s return native.sliceWithWidth(line, startCol, length, strict); } -export const { wrapTextWithAnsi, visibleWidth, extractSegments } = native; +export const { wrapTextWithAnsi, visibleWidth, extractSegments, sanitizeText } = native; diff --git a/packages/natives/src/text/types.ts b/packages/natives/src/text/types.ts index 4501a0324..6e87cecf0 100644 --- a/packages/natives/src/text/types.ts +++ b/packages/natives/src/text/types.ts @@ -42,6 +42,11 @@ declare module "../bindings" { * @param pad Whether to pad the output to `maxWidth`. */ truncateToWidth(text: string, maxWidth: number, ellipsisKind: number, pad: boolean): string; + /** + * Sanitize text output: strip ANSI codes, remove binary garbage, normalize line endings. + */ + sanitizeText(text: string): string; + /** * Wrap text to a visible width, preserving ANSI codes across line breaks. * @param text UTF-16 input text with optional ANSI escapes. diff --git a/packages/natives/test/native.test.ts b/packages/natives/test/native.test.ts index 5e42ea7d3..6116d32ff 100644 --- a/packages/natives/test/native.test.ts +++ b/packages/natives/test/native.test.ts @@ -2,7 +2,16 @@ import { beforeAll, describe, expect, it } from "bun:test"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; -import { FileType, fuzzyFind, type GlobMatch, glob, grep, htmlToMarkdown, invalidateFsScanCache } from "../src/index"; +import { + FileType, + fuzzyFind, + type GlobMatch, + glob, + grep, + htmlToMarkdown, + invalidateFsScanCache, + sanitizeText, +} from "../src/index"; let testDir: string; @@ -215,4 +224,22 @@ describe("pi-natives", () => { // Navigation/footer may or may not be removed depending on preprocessing }); }); + + describe("sanitizeText", () => { + it("should strip ANSI, remove control chars and normalize CR", () => { + const input = "\x1b[31mred\x1b[0m\ra\u0000b\tline\ncarriage\r\u0001\u0085"; + expect(sanitizeText(input)).toBe("redab\tline\ncarriage"); + }); + + it("should remove lone surrogates but keep valid pairs", () => { + expect(sanitizeText(`a\ud800b\udc00c`)).toBe("abc"); + const validPair = "a\u{1f600}b"; + expect(sanitizeText(validPair)).toBe(validPair); + }); + + it("should strip OSC sequences", () => { + const input = "\x1b]0;title\x07hello"; + expect(sanitizeText(input)).toBe("hello"); + }); + }); }); diff --git a/packages/tui/bench/sanitize.ts b/packages/tui/bench/sanitize.ts new file mode 100644 index 000000000..076f4bd28 --- /dev/null +++ b/packages/tui/bench/sanitize.ts @@ -0,0 +1,83 @@ +import { sanitizeText as nativeSanitizeText } from "@oh-my-pi/pi-natives"; + +function sanitizeBinaryOutput(str: string): string { + let out: string[] | undefined; + let last = 0; + + for (let i = 0; i < str.length; ) { + const code = str.codePointAt(i)!; + const width = code > 0xffff ? 2 : 1; + const next = i + width; + + // Allow tab, newline, carriage return. + const isAllowedControl = code === 0x09 || code === 0x0a || code === 0x0d; + if (isAllowedControl) { + i = next; + continue; + } + + // Filter out characters that crash `Bun.stringWidth()` or cause display issues: + // - ASCII control chars (C0) + // - DEL + C1 control block + // - Lone surrogates + const isControl = code <= 0x1f || code === 0x7f || (code >= 0x80 && code <= 0x9f); + const isSurrogate = code >= 0xd800 && code <= 0xdfff; + if (isControl || isSurrogate) { + out ??= []; + if (last !== i) out.push(str.slice(last, i)); + last = next; + } + + i = next; + } + + if (!out) return str; + if (last < str.length) out.push(str.slice(last)); + return out.join(""); +} +function jsSanitizeText(text: string): string { + return sanitizeBinaryOutput(Bun.stripANSI(text)).replaceAll("\r", ""); +} + +const ITERATIONS = 2000; + +const samples = { + plain: "hello world this is a plain ASCII string with some words", + ansi: "\x1b[31mred text\x1b[0m and \x1b[4munderlined content\x1b[24m with emoji 😅😅", + links: "prefix \x1b]8;;https://example.com\x07link\x1b]8;;\x07 suffix", + wide: "日本語のテキストとemoji 🚀✨ mixed with ascii", + wrapped: + "This is a long line that should wrap multiple times when rendered with ANSI \x1b[32mcolors\x1b[0m and tabs\tbetween words.", +}; + +const wrapWidth = 40; + +function bench(name: string, fn: () => void): number { + const start = Bun.nanoseconds(); + for (let i = 0; i < ITERATIONS; i++) { + fn(); + } + const elapsed = (Bun.nanoseconds() - start) / 1e6; + const perOp = (elapsed / ITERATIONS).toFixed(6); + console.log(`${name}: ${elapsed.toFixed(2)}ms total (${perOp}ms/op)`); + return elapsed; +} + +console.log(`Text layout benchmark (${ITERATIONS} iterations)\n`); + +for (const [name, text] of Object.entries(samples)) { + const jsResult = jsSanitizeText(text); + const nativeResult = nativeSanitizeText(text); + if (jsResult !== nativeResult) { + console.log(`MISMATCH ${name}: js="${jsResult}" native="${nativeResult}"`); + } + + bench(`jsSanitizeText/${name}`, () => { + jsSanitizeText(text); + }); + bench(`nativeSanitizeText/${name}`, () => { + nativeSanitizeText(text); + }); +} + + diff --git a/packages/utils/src/stream.ts b/packages/utils/src/stream.ts index ccc02ae54..b19b8f439 100644 --- a/packages/utils/src/stream.ts +++ b/packages/utils/src/stream.ts @@ -1,55 +1,5 @@ import { createAbortableStream } from "./abortable"; -/** - * Sanitize binary output for display/storage. - * Removes characters that crash string-width or cause display issues: - * - Control characters (except tab, newline, carriage return) - * - Lone surrogates - * - Characters with undefined code points - */ -export function sanitizeBinaryOutput(str: string): string { - let out: string[] | undefined; - let last = 0; - - for (let i = 0; i < str.length; ) { - const code = str.codePointAt(i)!; - const width = code > 0xffff ? 2 : 1; - const next = i + width; - - // Allow tab, newline, carriage return. - const isAllowedControl = code === 0x09 || code === 0x0a || code === 0x0d; - if (isAllowedControl) { - i = next; - continue; - } - - // Filter out characters that crash `Bun.stringWidth()` or cause display issues: - // - ASCII control chars (C0) - // - DEL + C1 control block - // - Lone surrogates - const isControl = code <= 0x1f || code === 0x7f || (code >= 0x80 && code <= 0x9f); - const isSurrogate = code >= 0xd800 && code <= 0xdfff; - if (isControl || isSurrogate) { - out ??= []; - if (last !== i) out.push(str.slice(last, i)); - last = next; - } - - i = next; - } - - if (!out) return str; - if (last < str.length) out.push(str.slice(last)); - return out.join(""); -} - -/** - * Sanitize text output: strip ANSI codes, remove binary garbage, normalize line endings. - */ -export function sanitizeText(text: string): string { - return sanitizeBinaryOutput(Bun.stripANSI(text)).replaceAll("\r", ""); -} - const LF = 0x0a; const CR = 0x0d; const decoder = new TextDecoder(); diff --git a/packages/utils/test/stream.test.ts b/packages/utils/test/stream.test.ts index 20fa37154..798f2b62b 100644 --- a/packages/utils/test/stream.test.ts +++ b/packages/utils/test/stream.test.ts @@ -1,12 +1,6 @@ import { describe, expect, it } from "bun:test"; -import { - parseJsonlLenient, - readJsonl, - readLines, - readSseJson, - sanitizeBinaryOutput, - sanitizeText, -} from "../src/stream"; +import { sanitizeText } from "@oh-my-pi/pi-natives"; +import { parseJsonlLenient, readJsonl, readLines, readSseJson } from "../src/stream"; const encoder = new TextEncoder(); @@ -34,23 +28,6 @@ async function collectAsync(iter: AsyncIterable): Promise { return output; } -describe("sanitizeBinaryOutput", () => { - it("removes control characters but keeps tabs/newlines", () => { - const input = "a\u0000b\tline\ncarriage\r\u0001"; - expect(sanitizeBinaryOutput(input)).toBe("ab\tline\ncarriage\r"); - }); - - it("removes lone surrogates", () => { - const input = `a\ud800b\udc00c`; - expect(sanitizeBinaryOutput(input)).toBe("abc"); - }); - - it("removes C1 control characters", () => { - const input = `a\u0085b`; - expect(sanitizeBinaryOutput(input)).toBe("ab"); - }); -}); - describe("sanitizeText", () => { it("strips ANSI and normalizes CR", () => { const input = "\u001b[31mred\u001b[0m\r\n";