diff --git a/docs/tui-core-renderer.md b/docs/tui-core-renderer.md index cb93617cc..aef72e4a4 100644 --- a/docs/tui-core-renderer.md +++ b/docs/tui-core-renderer.md @@ -72,18 +72,30 @@ could be looking at. 1. Compose the frame (`render(width)`), collecting `liveRegionStart` / `commitSafeEnd` from the root children (absolute row indices). -2. Classify: **fullPaint** (first paint, `clearScrollback` session replace, or +2. **Audit the committed prefix** (`findCommittedPrefixResync`, skipped on + geometry frames). Components must never re-layout rows below C, but real + flows violate it (a TTSR rewind truncating a streamed block, an image-cap + demotion shrinking a committed image) and the violation must not become + content loss. The detector samples the prefix *tail* (up to 8 non-blank + rows in the last 24, SGR-stripped): an in-place edit or restyle disturbs + only the touched rows (≤1 mismatch ⇒ aligned ⇒ ignored — stale styling in + history is the accepted artifact), while any insertion/deletion shifts + every row below it including the tail (⇒ re-anchor C at the first changed + row and recommit from there: history keeps the stale copy and gains a + fresh one — **duplication, never loss**). +3. Classify: **fullPaint** (first paint, `clearScrollback` session replace, or geometry change outside a multiplexer — all user gestures) or **update**. -3. Window math as in §1. Two special rules: +4. Window math as in §1. Two special rules: - **Overlays freeze commits** (`C' = C`): composited rows must never enter history; the hidden gap backfills via the chunk after the overlay closes. - - **Shrink into the committed prefix** (`L ≤ C`, only possible without a - seam): re-anchor `W = max(0, L − height)`, reset `C = min(B, W)`, keep the - stale history above (no gesture, no erase). -4. Extract the cursor marker, prepare lines (width fitting), slice the window, - composite overlays **into the window slice only** (screen coordinates — an - overlay never touches the frame or the ledger). -5. Emit: + - **Shrink into the committed prefix** (`L ≤ C`): re-anchor + `W = max(0, L − height)`, reset `C = min(B, W)`, keep the stale history + above (no gesture, no erase). +5. Extract the cursor marker (strip-first: markers never reach the terminal, + the prefix ledger, or the audit), prepare lines (width fitting), slice the + window, composite overlays **into the window slice only** (screen + coordinates — an overlay never touches the frame or the ledger). +6. Emit: | Emitter | Bytes | When | |---|---|---| @@ -133,7 +145,9 @@ engine's required guarantee, not a per-terminal optimization. multiplexers. 2. **NEVER rewrite a committed row.** No emitter may touch frame rows `< C`, and `W ≥ C` always (re-showing a committed row on the grid duplicates it - for a scrolling reader — the historical corruption family). + for a scrolling reader — the historical corruption family). When a + *component* violates immutability, the audit (§2) degrades to duplication — + never silently skip rows, never erase history. 3. **Commits are exactly the chunk.** Any byte shape that scrolls the screen must scroll *only* rows accounted for by `C' − C` — that is what makes scrollback provably `frame[0..C)`. @@ -252,7 +266,10 @@ Kitty images are **transmit-once, place-many** (`kitty-graphics.ts`). exceeded the demoted image's pixels are deleted by id (`a=d,d=I`) and its visible rows re-render as the text fallback through the ordinary window diff — **no destructive replay**. A demoted placement already committed to history -simply loses its pixels (committed rows are immutable). +simply loses its pixels (committed rows are immutable), and the text fallback +is **height-preserving** once a graphic has rendered (reserved rows + fallback +line), so demotion never shrinks the block and never shifts committed content +below it. **Rule:** never re-emit full base64 per frame. Kitty Unicode placeholders are default-on only for kitty/ghostty (`PI_NO_KITTY_PLACEHOLDERS` / diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 5715bba51..e1f386684 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -1,13 +1,19 @@ # Changelog ## [Unreleased] +### Fixed + +- Fixed committed transcript rows silently vanishing when a component re-laid-out content the engine had already scrolled into native history — a TTSR stream rewind truncating a streamed block, or the image budget demoting a committed inline image to its one-line fallback, shifted every row below by the height delta and the engine kept committing from the stale index, skipping that many rows of everything after (missing interruption banners, half-cut images in scrollback). The engine now audits its committed prefix every ordinary frame: an in-place edit or restyle keeps its alignment (stale styling in history remains the accepted artifact), while any shift re-anchors the commit index at the first moved row and recommits from there — history keeps the stale copy and gains a fresh one. Duplication, never loss. The detector (`findCommittedPrefixResync`, exported for the stress harness's shadow ledger) samples the prefix tail SGR-stripped so theme restyles and single-row edits never trigger spurious recommits. +- Fixed budget-demoted inline images shrinking their transcript block: the text fallback is now height-preserving once a graphic has rendered (reserved rows plus the fallback line), so demotion never shifts content below a committed image. +- Fixed stale trailing cells bleeding into committed history on combining-heavy rows: the native width model can over-count Arabic/combining clusters, classifying a short-rendering row as full-width and skipping the trailing erase — the previous occupant's cells then scrolled into scrollback baked into the committed row. Non-ASCII row rewrites now erase the line before writing. + ### Changed - Rewrote the render core around an append-only native-scrollback contract. Committed rows are immutable: rows enter terminal history exactly once, in order, when the component-reported commit boundary (`NativeScrollbackLiveRegion`) marks them final, and the visible window repaints in place with relative moves. The engine no longer probes the terminal's scroll position or guesses whether a destructive rebuild is safe — the entire ED3-risk/defer/checkpoint machinery (viewport probes, eager streaming mode, dirty-scrollback reconciliation, deferred shrink/mutation intents, streaming high-water rebuilds, ConPTY-specific defer paths) is deleted. ED3 (`CSI 3 J`) now fires only on explicit user gestures: session replace, resize outside multiplexers, and `resetDisplay()`. This structurally removes the yank / flash / duplicated-rows / invisible-until-resize failure families tracked across #1610, #1635, #1651, #1682, #1719, #1746, #1799, #1823, #1962, #1974, #2000, #2011, #2154. - A frame that shrinks into its committed prefix re-anchors the visible window at the new tail and restarts commit bookkeeping; previously committed rows stay in history (history is never rewritten without a gesture). - Overlays now composite into the visible window slice only and freeze commits while visible, so overlay pixels can never enter native scrollback and closing an overlay no longer triggers a destructive history rebuild. - Inline-image budget demotion now deletes the demoted image's graphics by id and lets the window diff repaint the text fallback — no more mid-session destructive full replay when the image cap is exceeded. -- The render-stress harness now validates the contract with a shadow commit ledger (an independent reimplementation of the ledger math fed only by observed frames and bytes), asserting scrollback equals the committed prefix row-for-row across randomized op sequences, resizes, overlays, and multiplexer scenarios. +- The render-stress harness now validates the contract with a shadow commit ledger (an independent reimplementation of the ledger math fed only by observed frames and bytes), asserting scrollback equals the committed prefix row-for-row and that tape growth matches physical scroll exactly, across randomized op sequences, resizes, overlays, and multiplexer scenarios. The ghostty-web virtual terminal additionally survives libghostty-vt 0.4's WASM allocator traps via an event-log replay/compaction recovery, and strips non-spacing combining marks on input (a margin-aligned combining cluster deterministically corrupts that engine; mark placement through it was already unverifiable). ### Removed diff --git a/packages/tui/src/components/image.ts b/packages/tui/src/components/image.ts index e4695564d..ac88e7629 100644 --- a/packages/tui/src/components/image.ts +++ b/packages/tui/src/components/image.ts @@ -240,6 +240,10 @@ export class Image implements Component { #cachedLines?: string[]; #cachedWidth?: number; #cachedSuppressed = false; + // Tallest graphic placement this image has rendered. The text fallback + // pads itself to this height so a budget demotion never shrinks the block + // (its rows may already be committed to native scrollback). + #renderedGraphicRows = 0; constructor( base64Data: string, @@ -309,12 +313,11 @@ export class Image implements Component { const moveUp = result.rows > 1 ? `\x1b[${result.rows - 1}A` : ""; lines.push(moveUp + (result.sequence ?? "")); } else { - lines = [ - this.#theme.fallbackColor(imageFallback(this.#mimeType, this.#dimensions, this.#options.filename)), - ]; + lines = this.#fallbackLines(); } + this.#renderedGraphicRows = Math.max(this.#renderedGraphicRows, lines.length); } else { - lines = [this.#theme.fallbackColor(imageFallback(this.#mimeType, this.#dimensions, this.#options.filename))]; + lines = this.#fallbackLines(); } this.#cachedLines = lines; @@ -323,4 +326,25 @@ export class Image implements Component { return lines; } + + /** + * Text fallback, height-preserving once a graphic has rendered: a demoted + * image must keep occupying the rows its placement used, because those + * rows may already be committed to native scrollback — shrinking the block + * would shift everything below it and force the renderer's commit-resync + * (stale band + recommit). Reserved rows stay non-plain so blank-edge + * trimming cannot collapse the block either. + */ + #fallbackLines(): string[] { + const fallback = this.#theme.fallbackColor( + imageFallback(this.#mimeType, this.#dimensions, this.#options.filename), + ); + if (this.#renderedGraphicRows <= 1) return [fallback]; + const lines: string[] = []; + for (let i = 0; i < this.#renderedGraphicRows - 1; i++) { + lines.push(RESERVED_IMAGE_ROW); + } + lines.push(fallback); + return lines; + } } diff --git a/packages/tui/src/tui.ts b/packages/tui/src/tui.ts index f8877aead..9badfc9d8 100644 --- a/packages/tui/src/tui.ts +++ b/packages/tui/src/tui.ts @@ -421,6 +421,73 @@ interface PreparedLine { line: string; } +const SGR_SEQUENCE = /\x1b\[[0-9;:]*m/g; + +/** Compare two rows ignoring SGR styling (theme restyles keep alignment). */ +function rowsEquivalent(a: string, b: string): boolean { + if (a === b) return true; + return a.replace(SGR_SEQUENCE, "") === b.replace(SGR_SEQUENCE, ""); +} + +function isBlankRow(row: string): boolean { + if (row.length === 0) return true; + return row.replace(SGR_SEQUENCE, "").trim().length === 0; +} + +// Tail-alignment sampling bounds: look back through up to LOOKBACK rows of +// the committed prefix to collect SAMPLES non-blank comparisons. +const RESYNC_TAIL_LOOKBACK = 24; +const RESYNC_TAIL_SAMPLES = 8; + +/** + * Decide whether `frame` still aligns with the committed prefix, and where to + * re-anchor the commit index when it does not. Returns the resync row index, + * or -1 when no resync is needed. + * + * The detector exploits the asymmetry between the two mutation classes: an + * in-place edit or restyle of committed rows disturbs only the touched rows + * (alignment below them is intact — the stale copy in history is the + * long-accepted artifact), while any insertion or deletion shifts EVERY row + * below it, including the rows just above the commit boundary. So the prefix + * *tail* is sampled (up to 8 non-blank rows within the last 24, compared + * SGR-stripped so theme changes stay quiet, tolerating one mismatch for a + * legitimate single-row edit): aligned ⇒ no resync; misaligned ⇒ resync at + * the first non-equivalent row, recommitting from there — duplication, never + * loss. Highly repetitive tails (identical filler rows) can mask a shift, in + * which case the skipped rows are content-identical to the committed ones — + * observationally harmless. Exported for the render-stress harness, whose + * shadow commit ledger must mirror the engine's law exactly. + */ +export function findCommittedPrefixResync(frame: readonly string[], prefix: readonly string[]): number { + const committed = prefix.length; + if (committed === 0) return -1; + if (frame.length >= committed) { + let samples = 0; + let mismatches = 0; + const lookback = Math.min(RESYNC_TAIL_LOOKBACK, committed); + for (let j = 1; j <= lookback && samples < RESYNC_TAIL_SAMPLES; j++) { + const row = frame[committed - j]!; + const old = prefix[committed - j]!; + if (row === old) { + if (!isBlankRow(row)) samples++; + continue; + } + if (isBlankRow(row) && isBlankRow(old)) continue; + samples++; + if (!rowsEquivalent(row, old)) mismatches++; + } + // No signal (all-blank tail) or at most one edited row: aligned. + if (samples === 0 || mismatches <= 1) return -1; + } + // Misaligned (or the frame no longer covers the prefix): re-anchor at the + // first row whose content actually changed. + const limit = Math.min(committed, frame.length); + for (let i = 0; i < limit; i++) { + if (!rowsEquivalent(frame[i]!, prefix[i]!)) return i; + } + return limit < committed ? limit : -1; +} + /** * TUI - Main class for managing terminal UI with differential rendering */ @@ -495,6 +562,12 @@ export class TUI extends Container { // rows below the `NativeScrollbackLiveRegion` boundary so they never get // here while they can still change. #committedRows = 0; + // Raw rows mirroring [0, #committedRows) — the engine's claim of what it + // committed, audited each ordinary frame against the current render to + // detect components re-laying-out committed content (see + // #auditCommittedPrefix). Holds references to component-cached strings, so + // the audit is a pointer walk in the common case. + #committedPrefix: string[] = []; // Frame row currently mapped to screen row 0. Monotonic between full // paints: a shrink never re-exposes scrolled-off rows (they cannot be // un-scrolled without rewriting history); live rows repaint at fixed @@ -1574,22 +1647,20 @@ export class TUI extends Container { } /** - * Find and extract cursor position from rendered lines. - * Searches for CURSOR_MARKER, calculates its position, and strips it from - * the output. Markers are internal sentinels and must never reach the - * terminal, so every occurrence is stripped; only a marker at or below - * `viewportTop` becomes a hardware cursor target. + * Strip every CURSOR_MARKER from the rendered lines (markers are internal + * sentinels and must never reach the terminal, the committed prefix, or + * the resync audit) and return the positions of the stripped markers, + * bottom-most first. Callers pick the visible one once the window top is + * known. */ - #extractCursorPosition(lines: string[], viewportTop: number): { row: number; col: number } | null { - let cursor: { row: number; col: number } | null = null; + #extractCursorMarkers(lines: string[]): { row: number; col: number }[] { + const markers: { row: number; col: number }[] = []; for (let row = lines.length - 1; row >= 0; row--) { const line = lines[row]; let markerIndex = line.indexOf(CURSOR_MARKER); if (markerIndex === -1) continue; - if (cursor === null && row >= viewportTop) { - const beforeMarker = line.slice(0, markerIndex); - cursor = { row, col: visibleWidth(beforeMarker) }; - } + const beforeMarker = line.slice(0, markerIndex); + markers.push({ row, col: visibleWidth(beforeMarker) }); let stripped = line; while (markerIndex !== -1) { stripped = stripped.slice(0, markerIndex) + stripped.slice(markerIndex + CURSOR_MARKER.length); @@ -1597,7 +1668,7 @@ export class TUI extends Container { } lines[row] = stripped; } - return cursor; + return markers; } #terminalLine(line: string): string { @@ -1654,6 +1725,10 @@ export class TUI extends Container { this.#imageBudget.beginPass(); const rawFrame = this.render(width); this.#imageBudget.endPass(); + // Strip cursor markers immediately (they are internal sentinels and + // must never reach the terminal, the committed prefix, or the audit); + // the visible marker is chosen after the window top is known. + const cursorMarkers = this.#extractCursorMarkers(rawFrame); const liveRegionStart = this.#nativeScrollbackLiveRegionStart; const commitSafeEnd = this.#nativeScrollbackCommitSafeEnd; @@ -1672,6 +1747,20 @@ export class TUI extends Container { (resizeEventOccurred && this.#previousHeight > 0); const geometryChanged = widthChanged || heightChanged; + // Committed-prefix audit: rows below the commit index are physically in + // terminal history and must never re-layout. When a component violates + // that — a budget-demoted image collapsing to its one-line fallback, a + // TTSR rewind truncating a block whose sealed prefix already committed — + // keeping the old index would silently skip that many rows of + // everything below (content loss). Re-anchor at the divergence instead: + // the stale copy stays in history and rows recommit from there — + // duplication, never loss. Skipped on geometry frames (a rewrap + // legitimately reflows every row; the mux branch re-bases the prefix + // and non-mux geometry replays from scratch). + if (this.#hasEverRendered && !geometryChanged && !this.#clearScrollbackOnNextRender) { + this.#auditCommittedPrefix(rawFrame); + } + // 3. Window and commit math (lengths only; content prepared below). const frameLength = rawFrame.length; let hasVisibleOverlay = false; @@ -1703,14 +1792,15 @@ export class TUI extends Container { } else if (frameLength <= this.#committedRows) { // The frame shrank into (or below) the committed prefix: the app // replaced content it had already let scroll into history without - // requesting a session replace (only possible without a live-region - // seam). History is immutable without a gesture, so the stale - // committed copy stays in scrollback; re-anchor the window at the - // tail and restart commit bookkeeping there so the live grid shows - // the real content instead of a blank pinned window. + // requesting a session replace. History is immutable without a + // gesture, so the stale committed copy stays in scrollback; + // re-anchor the window at the tail and restart commit bookkeeping + // there so the live grid shows the real content instead of a blank + // pinned window. windowTop = Math.max(0, frameLength - height); chunkTo = Math.min(commitBoundary, windowTop); this.#committedRows = chunkTo; + this.#committedPrefix = rawFrame.slice(0, chunkTo); } else { // Re-anchor to the frame tail, floored at the committed boundary: a // shrink (or overlay close) pulls the window back down, but never @@ -1722,23 +1812,36 @@ export class TUI extends Container { // Overlays freeze commits: composited rows must never enter // history, and the hidden gap backfills via the chunk once the // overlay closes. A multiplexer resize also commits nothing — the - // pane keeps its own (old-wrap) history. + // pane keeps its own (old-wrap) history — and re-bases the audit + // prefix at the new width so the accepted wrap drift does not read + // as a violation on the next ordinary frame. chunkTo = hasVisibleOverlay || geometryChanged ? this.#committedRows : Math.max(this.#committedRows, Math.min(commitBoundary, windowTop)); + if (geometryChanged) { + this.#committedPrefix = rawFrame.slice(0, this.#committedRows); + } } - // 5. Extract the hardware-cursor marker (frame coordinates) before - // width fitting, then prepare lines and build the visible window slice. - let cursorPos = this.#extractCursorPosition(rawFrame, windowTop); + // 5. Pick the visible cursor marker (bottom-most at or below the window + // top), prepare lines, and build the visible window slice. + let cursorPos: { row: number; col: number } | null = null; + for (const marker of cursorMarkers) { + if (marker.row >= windowTop) { + cursorPos = marker; + break; + } + } const frame = this.#prepareLines(rawFrame, width, true); let window: string[] = new Array(height); for (let r = 0; r < height; r++) window[r] = frame[windowTop + r] ?? ""; if (hasVisibleOverlay) { window = this.#compositeOverlaysIntoWindow(window, width, height); - const overlayCursor = this.#extractCursorPosition(window, 0); - if (overlayCursor) cursorPos = { row: windowTop + overlayCursor.row, col: overlayCursor.col }; + const overlayMarkers = this.#extractCursorMarkers(window); + if (overlayMarkers.length > 0) { + cursorPos = { row: windowTop + overlayMarkers[0]!.row, col: overlayMarkers[0]!.col }; + } window = this.#prepareLines(window, width, false); } @@ -1776,6 +1879,7 @@ export class TUI extends Container { chunkTo, windowTop, }); + this.#committedPrefix = rawFrame.slice(0, chunkTo); this.#clearScrollbackOnNextRender = false; this.#hasEverRendered = true; if (!firstPaint && frameLength > height) this.#armPostFullPaintSettle(); @@ -1788,6 +1892,29 @@ export class TUI extends Container { prevHardwareCursorRow, forceWindowRewrite: this.#forceViewportRepaintOnNextRender || (geometryChanged && isMultiplexerSession()), }); + for (let i = this.#committedPrefix.length; i < chunkTo; i++) { + this.#committedPrefix.push(rawFrame[i] ?? ""); + } + } + + /** + * Detect committed-prefix violations and re-anchor the commit index at the + * first moved row, so subsequent rows recommit instead of being skipped: + * the stale copy stays in history — duplication, never loss. Pure in-place + * restyles keep their alignment and are left alone (stale styling in + * history was always the accepted artifact). + */ + #auditCommittedPrefix(rawFrame: string[]): void { + const prefix = this.#committedPrefix; + if (prefix.length === 0) return; + const resyncTo = findCommittedPrefixResync(rawFrame, prefix); + if (resyncTo < 0) return; + this.#committedRows = resyncTo; + prefix.length = resyncTo; + if ($flag("PI_DEBUG_REDRAW")) { + const msg = `[${new Date().toISOString()}] commit resync: committed prefix diverged at row ${resyncTo}; recommitting\n`; + fs.appendFileSync(getDebugLogPath(), msg); + } } #prepareLines(lines: string[], width: number, useCache: boolean): string[] { @@ -1966,8 +2093,18 @@ export class TUI extends Container { if (TERMINAL.isImageLine(line)) return ERASE_LINE + line; const terminalLine = this.#terminalLine(line); const asciiWidth = this.#ansiAsciiLineWidth(line, width); - const lineWidth = asciiWidth ?? visibleWidth(line); - return lineWidth >= width ? terminalLine : terminalLine + ERASE_TO_END_OF_LINE; + if (asciiWidth !== undefined) { + // Exact width model: skip the erase only when the row truly fills + // the line (an EL there would eat the last cell via pending-wrap). + return asciiWidth >= width ? terminalLine : terminalLine + ERASE_TO_END_OF_LINE; + } + // Non-ASCII rows: the native measure can over-count combining-heavy + // scripts, so a row it calls "full" may render short and leave stale + // cells from the previous occupant — which would then scroll into + // history baked into the committed row. Erase the line first instead + // (rewrites always start at column 1, so EL-to-end clears the whole + // row); the leading reset keeps BCE on the default background. + return SEGMENT_RESET + ERASE_TO_END_OF_LINE + terminalLine; } /** @@ -2128,7 +2265,7 @@ export class TUI extends Container { #renderAltFrame(width: number, height: number): void { const base: string[] = new Array(Math.max(0, height)).fill(""); let lines = this.#compositeOverlaysIntoWindow(base, width, height); - this.#extractCursorPosition(lines, 0); + this.#extractCursorMarkers(lines); lines = this.#prepareLines(lines, width, false); this.#emitAltFrame(lines, width, height); } diff --git a/packages/tui/test/focus-menu-regression.test.ts b/packages/tui/test/focus-menu-regression.test.ts index c5483ce3b..06b31a33c 100644 --- a/packages/tui/test/focus-menu-regression.test.ts +++ b/packages/tui/test/focus-menu-regression.test.ts @@ -70,8 +70,12 @@ describe("focus-changing menu teardown", () => { tui.requestRender(); await term.waitForRender(); - expect(term.getViewport().map(line => line.trimEnd())).toEqual(["assistant", "prompt", "", "", "", ""]); - expect(term.getCursor()).toEqual({ row: 1, col: 6 }); + // The menu rows were committed to history while the menu was tall; + // closing it resyncs the commit index at the divergence (stale menu + // stays in scrollback) and the window re-anchors at the live tail — + // "assistant" scrolled into history and is no longer on the grid. + expect(term.getViewport().map(line => line.trimEnd())).toEqual(["prompt", "", "", "", "", ""]); + expect(term.getCursor()).toEqual({ row: 0, col: 6 }); } finally { tui.stop(); } diff --git a/packages/tui/test/render-regressions.test.ts b/packages/tui/test/render-regressions.test.ts index 98172ee88..035ab501d 100644 --- a/packages/tui/test/render-regressions.test.ts +++ b/packages/tui/test/render-regressions.test.ts @@ -1441,7 +1441,7 @@ describe("TUI terminal-state regressions", () => { }); }); - it("tmux: shrink deleting a committed row repaints the window floored at the commit boundary", async () => { + it("tmux: deleting a committed row re-anchors via commit resync without losing rows", async () => { await withEnvPatch({ TMUX: "1", STY: undefined, ZELLIJ: undefined }, async () => { const term = new UnknownViewportTerminal(40, 4, 10_000); const tui = new TUI(term); @@ -1464,18 +1464,26 @@ describe("TUI terminal-state regressions", () => { const writes = captureWrites(term); // Deleting "remove-me" (already committed to pane history) shifts - // every later row index up by one. Committed rows are immutable, so - // the window stays floored at the commit boundary (law 5): it shows - // the shorter tail plus a trailing blank instead of re-showing - // committed rows, and pane history keeps the stale copy. + // every later row up by one. The committed-prefix audit detects the + // shift and re-anchors the commit index at the divergence: pane + // history keeps the stale copy and the shifted rows recommit + // (duplication, never loss), so the window re-anchors to the full + // tail instead of pinning a blank row. component.setLines(["old-0", "old-2", "old-3", "tail-0", "tail-1", "tail-2", "tail-3"]); tui.requestRender(); await settle(term); - expect(visible(term)).toEqual(["tail-1", "tail-2", "tail-3", ""]); + expect(visible(term)).toEqual(["tail-0", "tail-1", "tail-2", "tail-3"]); expect(writes.join("")).not.toContain("\x1b[3J"); const history = term.getScrollBuffer().slice(0, term.getBufferPosition().baseY); - expect(history.map(line => line.trimEnd())).toEqual(["old-0", "remove-me", "old-2", "old-3"]); + expect(history.map(line => line.trimEnd())).toEqual([ + "old-0", + "remove-me", + "old-2", + "old-3", + "old-2", + "old-3", + ]); } finally { tui.stop(); } @@ -1694,7 +1702,7 @@ describe("TUI terminal-state regressions", () => { tui.stop(); } }); - it("keeps an offscreen expansion out of history while seam commits continue in order", async () => { + it("recommits an offscreen expansion behind the stale prefix while seam commits continue in order", async () => { const term = new VirtualTerminal(32, 6); const tui = new TUI(term); const component = new MutableLinesComponent(["status-0", ...rows("line-", 11)]); @@ -1714,7 +1722,8 @@ describe("TUI terminal-state regressions", () => { // Rows 0..5 (status-0, line-0..line-4) are committed. The frame edits // row 0 and inserts a row above the commit boundary while a tail - // append lands in the same frame. + // append lands in the same frame: 2+ prefix tail samples change, so + // the committed-prefix audit re-anchors at row 0 and recommits. component.setLines(["status-1", "expanded-details", ...rows("line-", 12)]); tui.requestRender(); await settle(term); @@ -1729,14 +1738,18 @@ describe("TUI terminal-state regressions", () => { ]); const buffer = term.getScrollBuffer().map(line => line.trimEnd()); const history = buffer.slice(0, term.getBufferPosition().baseY); - // Committed rows are immutable: the offscreen edit and insertion never - // reach native history. - expect(history).not.toContain("status-1"); - expect(history).not.toContain("expanded-details"); - // Rows that crossed the seam this frame committed with their content - // at commit time (law 1): the two-row insertion shifted indices, so - // the newly committed rows repeat line-4/line-5 — accepted artifact. - expect(history).toEqual(["status-0", ...rows("line-", 5), "line-4", "line-5"]); + // RESYNC law: native history keeps the stale committed copy AND gains + // a fresh copy of the diverged frame from row 0 — the offscreen edit + // and the expansion reach history (duplication, never loss). + expect(history).toEqual([ + // stale committed prefix, never rewritten + "status-0", + ...rows("line-", 5), + // recommitted frame rows 0..7 (new committed = 14 - height) + "status-1", + "expanded-details", + ...rows("line-", 6), + ]); // The appended tail row reaches the screen exactly once. expect(buffer.filter(row => row === "line-11").length).toBe(1); } finally { @@ -1779,11 +1792,11 @@ describe("TUI terminal-state regressions", () => { } }); - it("keeps stale collapsed ctrl-o markers in history while the expansion repaints the window", async () => { - // Committed marker rows are immutable: a Ctrl+O expansion repaints the - // live window, but the collapsed markers that already scrolled into - // native history stay there — one stale copy each, never rewritten and - // never duplicated (accepted artifact). + it("keeps stale collapsed ctrl-o markers in history and recommits the expanded rows behind them", async () => { + // A Ctrl+O expansion mutates committed rows, so the committed-prefix + // audit resyncs: the collapsed markers that already scrolled into native + // history stay there — one stale copy each, never rewritten — and the + // expanded rows recommit behind them (duplication, never loss). const term = new VirtualTerminal(48, 6); const tui = new TUI(term); const collapsedLines = [ @@ -1814,6 +1827,7 @@ describe("TUI terminal-state regressions", () => { ]); tui.requestRender(); await settle(term); + const history = term.getScrollBuffer().slice(0, term.getBufferPosition().baseY); expect(visible(term).map(line => line.trim())).toEqual([ "json-6", @@ -1824,13 +1838,20 @@ describe("TUI terminal-state regressions", () => { "editor", ]); const scrollback = term.getScrollBuffer(); - const scrollbackText = scrollback.join("\n"); expect(countMatches(scrollback, /Ctrl\+O: Expand/)).toBe(1); expect(countMatches(scrollback, /ctrl\+o/)).toBe(1); - // The expanded rows sit above the commit boundary: they never paint. - expect(scrollbackText).not.toContain("code line"); - expect(scrollbackText).not.toContain("output line"); - // Rows still below the boundary appear exactly once. + // The resync re-anchors at the first diverged row (the code marker) + // and recommits from there: the expanded rows reach history exactly + // once, right behind the stale markers. + for (const line of ["code line 0", "code line 1", "output line 0", "output line 1"]) { + expect(countMatches(history, new RegExp(`^${line}\\s*$`)), `${line} recommits exactly once`).toBe(1); + } + // json rows inside the recommitted span carry one stale + one fresh + // copy; rows still in the live window appear exactly once. + for (let i = 0; i < 6; i++) { + const pattern = new RegExp(`\\bjson-${i}\\b`); + expect(countMatches(scrollback, pattern), `json-${i} appears twice (stale + recommit)`).toBe(2); + } for (let i = 6; i < 10; i++) { const pattern = new RegExp(`\\bjson-${i}\\b`); expect(countMatches(scrollback, pattern), `json-${i} should appear exactly once`).toBe(1); @@ -1988,11 +2009,14 @@ describe("TUI terminal-state regressions", () => { } }); - it("re-anchors a bottom-anchored high-water collapse and keeps stale history above", async () => { - // Law 4: the collapse shrinks the frame below the committed count, so the - // engine re-anchors the window at the new tail and resets its commit - // counter. Native history is append-only: the high-water preview copy - // stays in scrollback above (accepted artifact) — never clawed back. + it("re-anchors a bottom-anchored high-water collapse at the divergence and recommits the tail into history", async () => { + // RESYNC law: the collapse shrinks the frame below the committed count, + // so the engine re-anchors the commit index at the first diverged row + // (row 8, where preview-* became result-*). Native history is + // append-only: the high-water preview copy stays in scrollback above + // (accepted artifact) — never clawed back. The window starts at the + // re-anchored commit index, which here sits past `length - height`, so + // the short tail is blank-padded rather than overwriting committed rows. const term = new VirtualTerminal(40, 5); const highWaterFrame = [...rows("base-", 8), ...rows("preview-", 10)]; const finalFrame = [...rows("base-", 8), "result-0", "result-1"]; @@ -2012,15 +2036,22 @@ describe("TUI terminal-state regressions", () => { await settle(term); expect(writes.join("")).not.toContain("\x1b[3J"); - expect(visible(term).map(line => line.trim())).toEqual([ - "base-5", - "base-6", - "base-7", - "result-0", - "result-1", - ]); + expect(visible(term).map(line => line.trim())).toEqual(["result-0", "result-1", "", "", ""]); const history = term.getScrollBuffer().slice(0, term.getBufferPosition().baseY); expect(history.map(line => line.trimEnd())).toEqual(highWaterFrame.slice(0, 13)); + + // Once the transcript grows past the window again, the post-collapse + // tail commits: result rows REACH history instead of being ignored. + component.setLines([...finalFrame, ...rows("tail-", 5)]); + tui.requestRender(); + await settle(term); + + expect(visible(term).map(line => line.trim())).toEqual(rows("tail-", 5)); + const grownHistory = term + .getScrollBuffer() + .slice(0, term.getBufferPosition().baseY) + .map(line => line.trimEnd()); + expect(grownHistory).toEqual([...highWaterFrame.slice(0, 13), "result-0", "result-1"]); } finally { tui.stop(); } @@ -2052,11 +2083,12 @@ describe("TUI terminal-state regressions", () => { } }); - it("writes the same offscreen-expansion bytes while the reader is parked in scrollback", async () => { - // Law 7: the engine neither knows nor cares where the reader is — an - // offscreen expansion only appends commits at the seam and rewrites grid - // rows, so a reader scrolled fully into native scrollback keeps a stable - // view and is never yanked. + it("recommits an offscreen expansion at the seam while the reader is parked in scrollback", async () => { + // An expansion above the commit boundary triggers the committed-prefix + // resync: the inserted rows (and the shifted committed rows) recommit + // at the seam, so the expansion reaches native history instead of + // being skipped. The reader scrolled into scrollback keeps a stable + // view — the recommit only appends below their anchor. const term = new VirtualTerminal(32, 5); const tui = new TUI(term); const component = new MutableLinesComponent(rows("line-", 12)); @@ -2081,9 +2113,9 @@ describe("TUI terminal-state regressions", () => { expect(paint).not.toContain("\x1b[H"); expect(term.getBufferPosition().viewportY).toBe(before.viewportY); expect(visible(term).map(line => line.trim())).toEqual(["line-2", "line-3", "line-4", "line-5", "line-6"]); - // The expansion rows sit above the commit boundary: they never enter - // history — not now, and not after the reader returns to the bottom. - expect(term.getScrollBuffer().join("\n")).not.toContain("expanded-0"); + // The resync recommits the expansion: it reaches native history + // exactly because the commit index re-anchored at the divergence. + expect(term.getScrollBuffer().join("\n")).toContain("expanded-0"); term.scrollLines(999); tui.requestRender(); @@ -2091,7 +2123,7 @@ describe("TUI terminal-state regressions", () => { const finalPosition = term.getBufferPosition(); expect(finalPosition.viewportY).toBe(finalPosition.baseY); - expect(term.getScrollBuffer().join("\n")).not.toContain("expanded-0"); + expect(term.getScrollBuffer().join("\n")).toContain("expanded-0"); expect(visible(term).map(line => line.trim())).toEqual([ "line-7", "line-8", @@ -2280,12 +2312,13 @@ describe("TUI terminal-state regressions", () => { scrolledTui.stop(); } }); - it("re-anchors a huge completion-style collapse at the new tail and keeps stale history", async () => { - // Law 4 (#1599 lineage): a 100-row transcript collapsing to 20 rows in a - // 10-row window must keep the new tail (including the prompt) on screen. - // The window re-anchors at `newLength - height` and the commit counter - // resets; the committed line-* rows remain in native history above as an - // accepted stale artifact — history is never rewritten. + it("re-anchors a huge completion-style collapse at the new tail and recommits the diverged head behind stale history", async () => { + // RESYNC law (#1599 lineage): a 100-row transcript collapsing to 20 rows + // in a 10-row window must keep the new tail (including the prompt) on + // screen. The frame no longer covers the committed prefix, so the commit + // index re-anchors at the first diverged row (row 0) and recommits up to + // `newLength - height`: short-0..short-9 land in history right behind + // the stale line-* copy — duplication of the stale prefix, never loss. const term = new UnknownViewportTerminal(40, 10); const tui = new TUI(term); const body = rows("line-", 99); @@ -2315,26 +2348,24 @@ describe("TUI terminal-state regressions", () => { "prompt-row", ]); const buffer = term.getScrollBuffer(); - // The re-anchored window is the only place short-* rows exist; rows - // 0..9 of the new frame fall under the stale committed prefix and are - // never painted at all. - for (let i = 10; i < short.length; i++) { + // The recommit puts short-0..short-9 into history exactly once and + // the re-anchored window holds short-10..prompt-row exactly once — + // nothing is lost, nothing duplicates. + for (let i = 0; i < short.length; i++) { expect(countMatches(buffer, new RegExp(`\\bshort-${i}\\b`)), `short-${i} appears once`).toBe(1); } - for (let i = 0; i < 10; i++) { - expect(countMatches(buffer, new RegExp(`\\bshort-${i}\\b`)), `short-${i} never paints`).toBe(0); - } const history = buffer.slice(0, term.getBufferPosition().baseY).map(line => line.trimEnd()); - expect(history).toEqual(body.slice(0, 90)); + expect(history).toEqual([...body.slice(0, 90), ...rows("short-", 10)]); } finally { tui.stop(); } }); - it("re-anchors a huge collapse without clears while the reader is parked in scrollback", async () => { - // Law 4 + law 2: even a 100→20 row collapse never emits ED2/ED3 — the - // re-anchor is a window rewrite, so a reader parked in native scrollback - // keeps a byte-stable view and their anchor. + it("recommits a huge collapse without clears while the reader is parked in scrollback", async () => { + // RESYNC + no-clear law: even a 100→20 row collapse never emits ED2/ED3 + // — the re-anchor recommits the diverged frame head behind the stale + // prefix and rewrites the window, so a reader parked in native + // scrollback keeps a byte-stable view and their anchor. const term = new UnknownViewportTerminal(40, 10); const tui = new TUI(term); const body = rows("line-", 99); @@ -2363,6 +2394,7 @@ describe("TUI terminal-state regressions", () => { term.scrollLines(999); await settle(term); + expect(visible(term).map(line => line.trim())).toEqual([ "short-10", "short-11", @@ -2375,15 +2407,21 @@ describe("TUI terminal-state regressions", () => { "short-18", "prompt-row", ]); - // Stale committed history stays above — never clawed back. + // Stale committed history stays above — never clawed back — and the + // recommitted frame head follows it (no row loss). const history = term.getScrollBuffer().slice(0, term.getBufferPosition().baseY); expect(history.join("\n")).toContain("line-89"); - expect(history.join("\n")).not.toContain("short-"); + for (let i = 0; i < 10; i++) { + expect(countMatches(history, new RegExp(`\\bshort-${i}\\b`)), `short-${i} recommits once`).toBe(1); + } + for (let i = 10; i < short.length; i++) { + expect(countMatches(history, new RegExp(`\\bshort-${i}\\b`)), `short-${i} stays in the window`).toBe(0); + } } finally { tui.stop(); } }); - it("re-anchors a shrink after an offscreen-edit grow advanced the commit boundary", async () => { + it("resyncs an offscreen-edit grow and re-anchors the following collapse", async () => { const term = new UnknownViewportTerminal(40, 10); const tui = new TUI(term); const initial = rows("line-", 19); @@ -2395,8 +2433,9 @@ describe("TUI terminal-state regressions", () => { await settle(term); // Offscreen edit (row 0) + 100-row growth in one frame: the edit is - // ignored (row 0 is committed) while the growth advances the commit - // boundary — rows commit with their content at commit time (law 1). + // an insertion above the commit boundary, so the audit re-anchors and + // the edited transcript recommits behind the stale original (law 1 + // content-at-commit-time, duplication never loss). const expanded = ["edited-line", ...rows("line-", 118), "prompt-row"]; component.setLines(expanded); tui.requestRender(); @@ -2413,7 +2452,7 @@ describe("TUI terminal-state regressions", () => { "line-117", "prompt-row", ]); - expect(term.getScrollBuffer().join("\n")).not.toContain("edited-line"); + expect(term.getScrollBuffer().join("\n")).toContain("edited-line"); // Collapse far below the commit boundary: law 4 re-anchors the window // at the new tail; the stale committed transcript stays above. @@ -2436,8 +2475,8 @@ describe("TUI terminal-state regressions", () => { ]); const history = term.getScrollBuffer().slice(0, term.getBufferPosition().baseY).join("\n"); expect(history).toContain("line-108"); - expect(history).not.toContain("short-"); - expect(history).not.toContain("edited-line"); + expect(history).toContain("short-4"); + expect(history).toContain("edited-line"); } finally { tui.stop(); } diff --git a/packages/tui/test/render-stress-harness.ts b/packages/tui/test/render-stress-harness.ts index fea8c53ae..3c7462e29 100644 --- a/packages/tui/test/render-stress-harness.ts +++ b/packages/tui/test/render-stress-harness.ts @@ -7,6 +7,7 @@ import { type Component, CURSOR_MARKER, type Focusable, + findCommittedPrefixResync, type OverlayAnchor, type OverlayHandle, type OverlayOptions, @@ -1128,12 +1129,19 @@ class StressDriver { // independently arrive at the same terminal state. #shadowTape: string[] = []; #shadowCommitted = 0; + // Raw-row mirror of the engine's committed prefix. The resync audit must + // run on the same inputs as the engine (raw rows, not normalized ones), or + // width-truncation collisions would let the two ledgers disagree about + // whether a divergence happened. + #shadowRawFrame: string[] = []; + #shadowRawPrefix: string[] = []; #shadowWindowTop = 0; #shadowFrame: string[] = []; #shadowFrameHeight = 0; #shadowFrameWidth = 0; #shadowFrameOverlay = false; #shadowFrameGeometryChanged = false; + #shadowResizePending = false; #shadowAltActive = false; // Every byte the renderer wrote to the terminal, in order. The sync-output // discipline oracle audits bracket balance incrementally from #writeLogScanned @@ -1173,23 +1181,51 @@ class StressDriver { realWrite(data); this.#applyShadowWrite(data); }; + // Mirror the engine's resize-event signal: a net-unchanged resize still + // reflows the terminal, and the engine classifies it as a geometry frame + // (audit skipped, commits frozen in multiplexers) — a dimension compare + // alone cannot see it. + const realResize = this.#term.resize.bind(this.#term); + (this.#term as { resize: (columns: number, rows: number) => void }).resize = (columns: number, rows: number) => { + this.#shadowResizePending = true; + realResize(columns, rows); + }; this.#tui = new TUI(this.#term, true, { renderScheduler: this.#scheduler }); this.#tui.addChild(this.#component); const realRender = this.#tui.render.bind(this.#tui); (this.#tui as { render: (width: number) => string[] }).render = (width: number) => { const lines = realRender(width); this.#shadowFrameGeometryChanged = - this.#shadowFrameWidth > 0 && - (width !== this.#shadowFrameWidth || this.#term.rows !== this.#shadowFrameHeight); - // Markers are engine-internal sentinels and never reach the terminal; - // strip them before normalization (stripVTControlCharacters otherwise - // swallows everything after an APC introducer). - this.#shadowFrame = lines.map(line => - expectedTerminalLine(line.includes(CURSOR_MARKER) ? line.replaceAll(CURSOR_MARKER, "") : line, width), - ); + this.#shadowResizePending || + (this.#shadowFrameWidth > 0 && + (width !== this.#shadowFrameWidth || this.#term.rows !== this.#shadowFrameHeight)); + this.#shadowResizePending = false; + // Markers are engine-internal sentinels; the engine strips them from + // this same array immediately after render returns, and its commit + // ledger (prefix + audit) only ever sees stripped rows — mirror that + // exactly. (Also: stripVTControlCharacters would otherwise swallow + // everything after an APC introducer during normalization.) + const stripped = lines.map(line => (line.includes(CURSOR_MARKER) ? line.replaceAll(CURSOR_MARKER, "") : line)); + this.#shadowRawFrame = stripped; + this.#shadowFrame = stripped.map(line => expectedTerminalLine(line, width)); this.#shadowFrameWidth = width; this.#shadowFrameHeight = this.#term.rows; this.#shadowFrameOverlay = this.#tui.hasOverlay(); + // Mirror the engine's render-time ledger transitions here: the audit + // resync and the shrink-into-prefix re-anchor can both fire on frames + // that emit zero bytes, which the write hook would never observe. + if (!this.#shadowFrameGeometryChanged && this.#shadowRawPrefix.length > 0) { + const resyncTo = findCommittedPrefixResync(stripped, this.#shadowRawPrefix); + if (resyncTo >= 0) { + this.#shadowCommitted = resyncTo; + this.#shadowRawPrefix.length = resyncTo; + } + } + if (stripped.length <= this.#shadowCommitted) { + this.#shadowCommitted = Math.max(0, stripped.length - Math.max(1, this.#term.rows)); + this.#shadowWindowTop = this.#shadowCommitted; + this.#shadowRawPrefix = stripped.slice(0, this.#shadowCommitted); + } return lines; }; } @@ -2075,6 +2111,7 @@ class StressDriver { } #assertOracles(op: AppliedOperation, before: Snapshot, after: Snapshot, index: number): void { this.#assertSyncOutputDiscipline(op, before, after, index); + this.#assertTapeScrollParity(op, before, after, index); this.#assertViewportFidelity(op, before, after, index); this.#assertCleanBufferWhenAligned(op, before, after, index); this.#assertNoFrameNeutralScrollbackGrowth(op, before, after, index); @@ -2102,6 +2139,26 @@ class StressDriver { } } + // The shadow tape and the physical buffer must scroll in lockstep: outside + // gesture replays (checkpoints, geometry) the only thing that ever pushes + // rows into native scrollback is a commit, and every commit appends to the + // tape in the same write. Any disagreement means the ledgers diverged — + // catch it at the op where it happens instead of N ops later when the + // content mismatch surfaces. + #assertTapeScrollParity(op: AppliedOperation, before: Snapshot, after: Snapshot, index: number): void { + if (!this.#traits.strictNativeScrollback) return; + if (op.checkpoint || op.geometryChanged) return; + if (this.#scrollbackCapReached(before) || this.#scrollbackCapReached(after)) return; + const physicalDelta = after.position.baseY - before.position.baseY; + const tapeDelta = after.shadowTapeLength - before.shadowTapeLength; + if (physicalDelta !== tapeDelta) { + this.#fail("tape/physical scroll parity", op, before, after, index, { + physicalDelta, + tapeDelta, + }); + } + } + // Synchronized-output (DEC 2026) + autowrap (DECAWM) bracket discipline. // Every paint write opens with PAINT_BEGIN (`\x1b[?2026h\x1b[?7l`) and closes // with PAINT_END (`\x1b[?7h\x1b[?2026l`); the standalone cursor write brackets @@ -2490,12 +2547,14 @@ class StressDriver { } if (this.#shadowAltActive) return; const frame = this.#shadowFrame; + const raw = this.#shadowRawFrame; const height = Math.max(1, this.#shadowFrameHeight); const length = frame.length; if (data.includes("\x1b[3J")) { this.#shadowCommitted = Math.max(0, length - height); this.#shadowWindowTop = this.#shadowCommitted; this.#shadowTape = frame.slice(0, this.#shadowCommitted); + this.#shadowRawPrefix = raw.slice(0, this.#shadowCommitted); return; } if (data.includes("\x1b[2J")) { @@ -2505,21 +2564,26 @@ class StressDriver { for (let i = 0; i < chunkTo; i++) this.#shadowTape.push(frame[i] ?? ""); this.#shadowCommitted = chunkTo; this.#shadowWindowTop = chunkTo; + this.#shadowRawPrefix = raw.slice(0, chunkTo); return; } - if (length <= this.#shadowCommitted) { - // Shrink into the committed prefix: the engine re-anchors and - // restarts commit bookkeeping; stale history stays on the tape. - this.#shadowCommitted = Math.max(0, length - height); - this.#shadowWindowTop = this.#shadowCommitted; - return; - } + // Audit and shrink re-anchoring are mirrored at render time (they can + // fire on zero-byte frames); the write hook only applies commits. const windowTop = Math.max(this.#shadowCommitted, length - height, 0); this.#shadowWindowTop = windowTop; - // Overlays and multiplexer geometry frames freeze commits. - if (this.#shadowFrameOverlay || this.#shadowFrameGeometryChanged) return; + // Overlays and multiplexer geometry frames freeze commits; a geometry + // frame also re-bases the raw prefix at the new width (accepted wrap + // drift, mirrored from the engine). + if (this.#shadowFrameGeometryChanged) { + this.#shadowRawPrefix = raw.slice(0, this.#shadowCommitted); + return; + } + if (this.#shadowFrameOverlay) return; const chunkTo = Math.max(this.#shadowCommitted, Math.min(length, windowTop)); - for (let i = this.#shadowCommitted; i < chunkTo; i++) this.#shadowTape.push(frame[i] ?? ""); + for (let i = this.#shadowCommitted; i < chunkTo; i++) { + this.#shadowTape.push(frame[i] ?? ""); + this.#shadowRawPrefix.push(raw[i] ?? ""); + } this.#shadowCommitted = chunkTo; } #scrollbackCapReached(snapshot: Snapshot): boolean { @@ -2563,6 +2627,11 @@ class StressDriver { index: number, ): void { if (!this.#scenario.uniqueContent) return; + // All comparisons run with non-spacing marks stripped: the virtual + // terminal drops them on input (ghostty-web 0.4 margin-cluster crash + // workaround), so buffer readback and frame/tape rows would otherwise + // never collide on marked rows. + const strip = (line: string): string => line.replace(NONSPACING_MARKS, ""); // Accumulate even when the check below is skipped (scrolled/overlay): the // frame's legitimate duplicates commit to scrollback regardless of where // the viewport is parked. The shadow tape contributes too: a no-seam @@ -2570,11 +2639,12 @@ class StressDriver { // legitimately commit a second time (the exact tape-equality oracle has // already proven the buffer matches the ledger row for row). for (const line of duplicateNonblankLines(after.frame)) { - this.#everDuplicatedFrameLines.add(line); + this.#everDuplicatedFrameLines.add(strip(line)); } const tapeSeen = new Set(); - for (const line of this.#shadowTape) { - if (line.length === 0) continue; + for (const raw of this.#shadowTape) { + if (raw.length === 0) continue; + const line = strip(raw); if (tapeSeen.has(line)) this.#everDuplicatedFrameLines.add(line); tapeSeen.add(line); } @@ -2582,15 +2652,17 @@ class StressDriver { // at the commit boundary) legitimately appears in both regions of the // whole-tape buffer snapshot. for (let r = 0; r < after.height; r++) { - const line = this.#shadowFrame[this.#shadowWindowTop + r] ?? ""; - if (line.length === 0) continue; + const raw = this.#shadowFrame[this.#shadowWindowTop + r] ?? ""; + if (raw.length === 0) continue; + const line = strip(raw); if (tapeSeen.has(line)) this.#everDuplicatedFrameLines.add(line); } if (this.#hasVisibleOverlay() || !after.atBottom) return; const allowed = this.#everDuplicatedFrameLines; const seen = new Set(); - for (const line of after.buffer) { - if (line.length === 0) continue; + for (const raw of after.buffer) { + if (raw.length === 0) continue; + const line = strip(raw); if (seen.has(line) && !allowed.has(line)) { this.#fail("unexpected duplicate native scrollback line", op, before, after, index, { line }); } @@ -2628,6 +2700,15 @@ class StressDriver { tags: this.#scenario.tags, operationCoverage: Object.fromEntries(this.#operationCoverage.entries()), lastOperations: this.#opLog.slice(-50), + shadow: { + committed: this.#shadowCommitted, + windowTop: this.#shadowWindowTop, + tapeLength: this.#shadowTape.length, + frameLength: this.#shadowFrame.length, + geometryChanged: this.#shadowFrameGeometryChanged, + overlayVisible: this.#shadowFrameOverlay, + }, + lastWrites: this.#writeLog.slice(-4).map(write => JSON.stringify(write.slice(-400))), children: this.#children.map(child => ({ id: child.id, active: child.active, diff --git a/packages/tui/test/virtual-terminal.ts b/packages/tui/test/virtual-terminal.ts index b356d980a..6c3845a0c 100644 --- a/packages/tui/test/virtual-terminal.ts +++ b/packages/tui/test/virtual-terminal.ts @@ -1,4 +1,5 @@ import * as fs from "node:fs"; +import * as os from "node:os"; import type { Terminal, TerminalAppearance } from "@oh-my-pi/pi-tui/terminal"; import { CellFlags, Ghostty, type GhosttyCell, type GhosttyTerminal } from "ghostty-web"; @@ -28,7 +29,32 @@ function loadGhosttyModule(): WebAssembly.Module { return new WebAssembly.Module(fs.readFileSync(wasmPath)); } -const ghosttyModule = loadGhosttyModule(); +let ghosttyModule = loadGhosttyModule(); + +/** + * Recompile the shared WASM module. ghostty-web 0.4 instances created from a + * module that already produced a trapped instance have been observed to trap + * again on byte streams that a freshly compiled module replays cleanly; the + * recovery path swaps the module before rebuilding. + */ +function reloadGhosttyModule(): void { + ghosttyModule = loadGhosttyModule(); +} + +// Non-spacing combining marks (Arabic harakat, Thai/Lao vowels) written so a +// cluster lands on the right margin deterministically corrupt ghostty-web +// 0.4's WASM memory (trap surfaces a few bytes later, with autowrap on or +// off). Mark placement through this engine is already unverifiable — readback +// migrates marks across cells, and the harness compares marked rows with +// non-spacing marks stripped (`sameLinesAllowingMarkDrift`) — so dropping the +// marks before the engine sees them removes the crash class without weakening +// any oracle. Variation selectors are kept: they are width-bearing (VS16 +// promotes emoji to 2 cells) and never combine at the margin. +const UNSAFE_COMBINING_MARKS = /(?![\uFE00-\uFE0F])\p{Mn}/gu; +function stripCombiningMarksForGhostty(data: string): string { + if (!/\p{Mn}/u.test(data)) return data; + return data.replace(UNSAFE_COMBINING_MARKS, ""); +} function createGhosttyEngine(): Ghostty { // libghostty-vt reports unimplemented control sequences (e.g. DECCARA `$r`, @@ -45,11 +71,14 @@ function createGhosttyTerminal( scrollbackCap: number, ): GhosttyTerminal { return ghostty.createTerminal(columns, rows, { - // Byte budget (not a line count), grown lazily to this ceiling. Sized far - // above the requested line cap so the engine never evicts before the - // wrapper's line-cap clamp does — the clamp is the only eviction the - // harness sees, reproducing xterm's line-count scrollback. - scrollbackLimit: Math.min(0xffff_ffff, Math.max((scrollbackCap + rows + 64) * 4096, 4 * 1024 * 1024)), + // Byte budget (not a line count). Sized to hold the wrapper's line cap + // comfortably while staying small enough that ghostty's own page + // eviction kicks in under heavy write volume: ghostty-web 0.4's + // allocator traps once an instance accumulates enough un-evicted + // history (recommit-heavy stress runs hit it). The wrapper still clamps + // the EXPOSED scrollback to the line cap, so eviction beyond the budget + // is invisible to the oracles. + scrollbackLimit: Math.max((scrollbackCap + rows + 64) * 1024, 1024 * 1024), fgColor: DEFAULT_FG_RGB, bgColor: DEFAULT_BG_RGB, }); @@ -59,17 +88,22 @@ function createGhosttyTerminal( // an explicit one. The exposed scrollback is clamped to this many lines (below). const DEFAULT_SCROLLBACK_LINES = 1000; // Packed default colors (0xRRGGBB). Light-grey fg on black bg so a styled SGR -// color is always distinguishable from "default" when reading back cells. +// row differs from a default row in cell readback. const DEFAULT_FG_RGB = 0xcccccc; const DEFAULT_BG_RGB = 0x000000; -// Compare readback against the configured defaults directly; Ghostty's -// getColors() currently reports render-state metadata, not these cell colors. -const DEFAULT_FG_R = (DEFAULT_FG_RGB >> 16) & 0xff; const MAX_GHOSTTY_WRITE_CHUNK = 4096; +// Compact the OOM-recovery event log once it exceeds this many logged chars. +// Kept aggressively small: ghostty-web 0.4 instances can trap on long byte +// histories (interactions that a synthesized text+grid state does not +// reproduce), so recovery must always replay a compact synthetic snapshot +// plus a short tail rather than the raw session history. +const EVENT_LOG_COMPACT_BUDGET = 256_000; const SYNC_OUTPUT_BEGIN = "\x1b[?2026h"; const SYNC_OUTPUT_END = "\x1b[?2026l"; const OSC_SEQUENCE = /\x1b\][\s\S]*?(?:\x07|\x1b\\)/g; - +// Compare readback against the configured defaults directly; Ghostty's +// getColors() currently reports render-state metadata, not these cell colors. +const DEFAULT_FG_R = (DEFAULT_FG_RGB >> 16) & 0xff; const DEFAULT_FG_G = (DEFAULT_FG_RGB >> 8) & 0xff; const DEFAULT_FG_B = DEFAULT_FG_RGB & 0xff; const DEFAULT_BG_R = (DEFAULT_BG_RGB >> 16) & 0xff; @@ -105,6 +139,17 @@ export class VirtualTerminal implements Terminal { #inputHandler?: (data: string) => void; #resizeHandler?: () => void; #pendingEngineResize = false; + // Byte/resize event log since the last engine recreate. ghostty-web 0.4's + // allocator exhausts after enough cumulative write volume in one instance + // (recommit-heavy stress runs hit it); on an OOM trap the wrapper rebuilds + // a fresh engine and replays this log, which reproduces the exact terminal + // state. Full-clear recreates reset the log (prior history is erased), so + // it stays bounded by the bytes since the last destructive replay. + #eventLog: (string | { columns: number; rows: number })[] = []; + #eventLogBytes = 0; + #logBaseColumns: number; + #logBaseRows: number; + #replayingLog = false; // Memoized text of committed scrollback rows, keyed by absolute offset. Safe // because the engine never evicts (its byte budget sits far above the line // cap), so an offset's content is stable until a resize (rewrap) or recreate @@ -115,6 +160,8 @@ export class VirtualTerminal implements Terminal { constructor(columns = 80, rows = 24, scrollback?: number) { this.#columns = columns; this.#rows = rows; + this.#logBaseColumns = columns; + this.#logBaseRows = rows; this.#scrollbackCap = scrollback ?? DEFAULT_SCROLLBACK_LINES; this.#ghostty = createGhosttyEngine(); this.#term = createGhosttyTerminal(this.#ghostty, columns, rows, this.#scrollbackCap); @@ -382,10 +429,12 @@ export class VirtualTerminal implements Terminal { data = data.slice(0, clearIndex) + data.slice(clearIndex + clearScrollbackAfterFullClear.length); } else if (this.#pendingEngineResize) { this.#term.resize(this.#columns, this.#rows); + this.#eventLog.push({ columns: this.#columns, rows: this.#rows }); this.#historyTextCache.length = 0; // engine rewraps scrollback on resize this.#pendingEngineResize = false; } data = this.#stripSynchronizedOutput(data); + data = stripCombiningMarksForGhostty(data); this.#writeToGhostty(data); this.#refollowBottom(wasBottom); } @@ -396,9 +445,9 @@ export class VirtualTerminal implements Terminal { } #writeToGhostty(data: string): void { - if (data.length <= MAX_GHOSTTY_WRITE_CHUNK) { - this.#term.write(data); - return; + if (!this.#replayingLog) { + this.#eventLog.push(data); + this.#eventLogBytes += data.length; } let offset = 0; while (offset < data.length) { @@ -406,9 +455,126 @@ export class VirtualTerminal implements Terminal { const last = data.charCodeAt(end - 1); if (end < data.length && last >= 0xd800 && last <= 0xdbff) end--; if (end <= offset) end = Math.min(offset + 1, data.length); - this.#term.write(data.slice(offset, end)); + const chunk = data.slice(offset, end); + try { + this.#term.write(chunk); + } catch (error) { + if (this.#replayingLog) { + const dumpPath = `${os.tmpdir()}/ghostty-trap-log-${Date.now()}.json`; + try { + fs.writeFileSync(dumpPath, JSON.stringify(this.#eventLog)); + } catch {} + throw new Error( + `ghostty write failed during OOM-recovery replay (chunk ${chunk.length} chars at offset ${offset} of ${data.length}): ${String(error)}\n` + + `event log dumped to ${dumpPath}\n` + + `chunk head: ${JSON.stringify(chunk.slice(0, 200))}`, + { cause: error }, + ); + } + this.#recoverFromEngineOom(); + return; + } offset = end; } + // Healthy write completed: once the log grows past the budget, compact + // it to a bounded synthetic state and rotate onto a fresh engine. + // ghostty-web 0.4 instances cannot be freed safely and grow their WASM + // memory monotonically with write volume; abandoned giants eventually + // starve the process so badly that a fresh instance cannot even grow. + // Rotating early keeps every instance small. + if (!this.#replayingLog && this.#eventLogBytes > EVENT_LOG_COMPACT_BUDGET) { + this.#compactEventLog(); + this.#rebuildEngineFromLog(); + } + } + + /** + * Replace the event log with a synthetic stream rebuilt from the healthy + * engine's readable state: the wrapper-visible history window as plain + * text, the grid repainted with background runs (the only style any oracle + * reads), and the cursor restored. Replaying it reproduces every + * observable the oracles consume. + */ + #compactEventLog(): void { + const historyLen = this.#term.getScrollbackLength(); + const capped = this.#cappedBaseY(); + let synthetic = ""; + for (let i = 0; i < capped; i++) { + synthetic += `${this.#historyRowText(historyLen - capped + i)}\r\n`; + } + // Push exactly `capped` rows into scrollback, leaving a blank grid. + synthetic += "\r\n".repeat(Math.max(0, this.#rows - 1)); + for (let row = 0; row < this.#rows; row++) { + synthetic += `\x1b[${row + 1};1H\x1b[K${this.#syntheticGridRow(row)}`; + } + const cursor = this.getCursor(); + synthetic += `\x1b[${cursor.row + 1};${cursor.col + 1}H`; + this.#eventLog = [synthetic]; + this.#eventLogBytes = synthetic.length; + this.#logBaseColumns = this.#columns; + this.#logBaseRows = this.#rows; + } + + /** Grid row text with minimal background-run SGR, for log compaction. */ + #syntheticGridRow(row: number): string { + const cells = this.#term.getLine(row); + if (!cells) return ""; + let out = ""; + let currentBg = -1; // -1 = default + for (let col = 0; col < cells.length; col++) { + const cell = cells[col]; + if (!cell || cell.width === 0) continue; + const bg = this.#isDefaultBg(cell) ? -1 : (cell.bg_r << 16) | (cell.bg_g << 8) | cell.bg_b; + if (bg !== currentBg) { + out += bg === -1 ? "\x1b[49m" : `\x1b[48;2;${cell.bg_r};${cell.bg_g};${cell.bg_b}m`; + currentBg = bg; + } + if (cell.codepoint === 0) { + out += " "; + } else { + out += + cell.grapheme_len > 0 ? this.#term.getGraphemeString(row, col) : this.#safeCodepointText(cell.codepoint); + } + } + return `${out}\x1b[0m`; + } + + /** + * Rebuild a fresh engine and replay the event log to reproduce the exact + * terminal state. The failed write is already in the log, so the replay + * completes it against a fresh allocator. + */ + #recoverFromEngineOom(): void { + this.#rebuildEngineFromLog(); + } + + /** + * Rebuild a fresh engine and replay the event log to reproduce the exact + * terminal state. Used for proactive rotation (with a compacted log) and + * for OOM recovery, where the failed write is already in the log so the + * replay completes it against a fresh allocator. + */ + #rebuildEngineFromLog(): void { + const log = this.#eventLog; + // Give JSC a chance to collect previously abandoned instances before + // allocating another one. + Bun.gc(true); + reloadGhosttyModule(); + this.#ghostty = createGhosttyEngine(); + this.#term = createGhosttyTerminal(this.#ghostty, this.#logBaseColumns, this.#logBaseRows, this.#scrollbackCap); + this.#historyTextCache.length = 0; + this.#replayingLog = true; + try { + for (const event of log) { + if (typeof event === "string") { + this.#writeToGhostty(event); + } else { + this.#term.resize(event.columns, event.rows); + } + } + } finally { + this.#replayingLog = false; + } } #canRecreateForFullClear(data: string, clearIndex: number): boolean { @@ -443,6 +609,10 @@ export class VirtualTerminal implements Terminal { this.#pendingEngineResize = false; this.#viewportY = 0; this.#historyTextCache.length = 0; // fresh engine: prior scrollback is gone + this.#eventLog.length = 0; + this.#eventLogBytes = 0; + this.#logBaseColumns = this.#columns; + this.#logBaseRows = this.#rows; } /** Cells of the presented viewport row (history when scrolled up, else active grid). */