From 2c161d2a8fc84d6faab29fbe8b7d8f625943f047 Mon Sep 17 00:00:00 2001 From: roboomp Date: Sat, 11 Jul 2026 17:29:38 +0000 Subject: [PATCH 01/80] fix(coding-agent): preserved btw codex websocket routing - Preserved the session websocket preference for ephemeral /btw side-channel turns so Codex websocket-only models do not fall back to SSE. - Prioritized active /btw and /omfg panels in Esc handling before loop, maintenance, and main-turn interrupts. - Added regression coverage for side-channel websocket options and /btw Escape priority. Fixes #5213 --- packages/coding-agent/CHANGELOG.md | 1 + .../src/modes/controllers/input-controller.ts | 15 ++++++---- .../coding-agent/src/session/agent-session.ts | 2 +- .../agent-session-message-pipeline.test.ts | 5 ++-- .../test/input-controller-escape.test.ts | 29 +++++++++++++++++++ 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 049ccb186..08a89fe7b 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -29,6 +29,7 @@ ### Fixed +- Fixed `/btw` side-channel turns on Codex models such as `gpt-5.6-luna` by preserving the session websocket preference instead of forcing SSE, and made Esc dismiss the active `/btw` panel before interrupting loop/maintenance work. ([#5213](https://github.com/can1357/oh-my-pi/issues/5213)) - Fixed visible per-keystroke lag while searching in the `/resume` session picker. Literal matches now rank synchronously from a cached per-session haystack, fuzzy scoring runs in bounded background chunks that converge to the same ranking (large listings previously rebuilt a fuzzy index per token per session on every keystroke), and the prompt-history SQLite lookup — an FTS query plus a LIKE scan over every stored prompt — is debounced off the keystroke path. - Fixed compiled Linux binary extension loading when bundled web-search header generation cannot read `header-generator` data files from the build-time path. ([#5178](https://github.com/can1357/oh-my-pi/issues/5178)) diff --git a/packages/coding-agent/src/modes/controllers/input-controller.ts b/packages/coding-agent/src/modes/controllers/input-controller.ts index f5aa2766a..3ccd61b87 100644 --- a/packages/coding-agent/src/modes/controllers/input-controller.ts +++ b/packages/coding-agent/src/modes/controllers/input-controller.ts @@ -266,6 +266,8 @@ export class InputController { }); } this.ctx.editor.onEscape = () => { + // Side-channel panels are the topmost view. Esc dismisses them before + // touching loop mode, maintenance, or the underlying main turn. // Active context maintenance owns Esc: auto/manual compaction, // handoff generation, and auto-retry backoff all advertise // "(esc to cancel)". Dispatch on live session state instead of @@ -281,6 +283,13 @@ export class InputController { // (see EventController). Main-session maintenance still owns Esc and // stays cancellable from the main view (focused submit gates /compact // and handoff, so manual maintenance is main-only anyway). + if (this.ctx.hasActiveBtw() && this.ctx.handleBtwEscape()) { + return; + } + if (this.ctx.hasActiveOmfg() && this.ctx.handleOmfgEscape()) { + return; + } + if (!this.ctx.focusedAgentId) { const viewSession = this.ctx.viewSession; let aborted = false; @@ -308,12 +317,6 @@ export class InputController { } return; } - if (this.ctx.hasActiveBtw() && this.ctx.handleBtwEscape()) { - return; - } - if (this.ctx.hasActiveOmfg() && this.ctx.handleOmfgEscape()) { - return; - } if (this.ctx.focusedAgentId) { // Esc never interrupts the focused agent's turn: clear typed text, // else return the view to the main session. Interrupt via empty diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 43e469f52..5e11ce750 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -14706,7 +14706,7 @@ export class AgentSession { // stable, but give provider routing a unique request lineage. sessionId: `${cacheSessionId}:side:${Snowflake.next()}`, promptCacheKey: cacheSessionId, - preferWebsockets: false, + preferWebsockets: this.#preferWebsockets, reasoning: toReasoningEffort(this.thinkingLevel), disableReasoning: shouldDisableReasoning(this.thinkingLevel), hideThinkingSummary: this.agent.hideThinkingSummary, diff --git a/packages/coding-agent/test/agent-session-message-pipeline.test.ts b/packages/coding-agent/test/agent-session-message-pipeline.test.ts index 5c47c74b4..cb9528eb2 100644 --- a/packages/coding-agent/test/agent-session-message-pipeline.test.ts +++ b/packages/coding-agent/test/agent-session-message-pipeline.test.ts @@ -233,7 +233,7 @@ describe("AgentSession message pipeline", () => { expect(requestOnPayload).toHaveBeenCalledWith({ original: true, session: true }, undefined); expect(result).toEqual({ original: true, session: true }); }); - it("keeps ephemeral side-channel cache key separate from provider routing", async () => { + it("keeps ephemeral side-channel cache key separate from provider routing while preserving websocket preference", async () => { const api = "test-ephemeral-side-channel"; let capturedOptions: SimpleStreamOptions | undefined; registerCustomApi(api, (_model, _context, options) => { @@ -271,6 +271,7 @@ describe("AgentSession message pipeline", () => { sessionManager: SessionManager.inMemory(), settings: Settings.isolated({ "compaction.enabled": false }), modelRegistry: createModelRegistryStub() as never, + preferWebsockets: true, }); sessions.push(session); const cacheSessionId = session.sessionId; @@ -281,7 +282,7 @@ describe("AgentSession message pipeline", () => { expect(capturedOptions?.promptCacheKey).toBe(cacheSessionId); expect(capturedOptions?.sessionId).toStartWith(`${cacheSessionId}:side:`); expect(capturedOptions?.sessionId).not.toBe(cacheSessionId); - expect(capturedOptions?.preferWebsockets).toBe(false); + expect(capturedOptions?.preferWebsockets).toBe(true); }); it("runs ephemeral side-channel requests through the configured side stream function", async () => { diff --git a/packages/coding-agent/test/input-controller-escape.test.ts b/packages/coding-agent/test/input-controller-escape.test.ts index d2828106d..c5fe89ce8 100644 --- a/packages/coding-agent/test/input-controller-escape.test.ts +++ b/packages/coding-agent/test/input-controller-escape.test.ts @@ -448,6 +448,35 @@ describe("InputController escape behavior", () => { expect(spies.abort).not.toHaveBeenCalled(); }); + it("dismisses an active /btw panel before aborting loop mode", () => { + const { ctx, editor, spies } = createContext(); + ctx.loopModeEnabled = true; + mutableSessionState(ctx).isStreaming = true; + spies.hasActiveBtw.mockReturnValue(true); + const controller = new InputController(ctx); + + controller.setupKeyHandlers(); + editor.onEscape?.(); + + expect(spies.handleBtwEscape).toHaveBeenCalledTimes(1); + expect(spies.abort).not.toHaveBeenCalled(); + expect(ctx.loopModeEnabled).toBe(true); + }); + + it("dismisses an active /btw panel before aborting maintenance", () => { + const { ctx, editor, spies } = createContext(); + abortViewSession(ctx).isGeneratingHandoff = true; + spies.hasActiveBtw.mockReturnValue(true); + const controller = new InputController(ctx); + + controller.setupKeyHandlers(); + editor.onEscape?.(); + + expect(spies.handleBtwEscape).toHaveBeenCalledTimes(1); + expect(spies.abortHandoff).not.toHaveBeenCalled(); + expect(spies.abort).not.toHaveBeenCalled(); + }); + it("aborts an active streaming turn on the first Esc without asking for confirmation", () => { const { ctx, editor, spies } = createContext(); mutableSessionState(ctx).isStreaming = true; From 9269823d16c29190ad3a6dd0ffeddd8ae1bc354d Mon Sep 17 00:00:00 2001 From: roboomp Date: Sat, 11 Jul 2026 17:46:28 +0000 Subject: [PATCH 02/80] fix(coding-agent): shared codex state for btw websockets - Forwarded the session provider state map to ephemeral side-channel turns so Codex can allocate websocket transport state for the side request session id. - Extended the /btw side-channel regression to assert providerSessionState reaches the stream options with the websocket preference. Fixes #5213 --- packages/coding-agent/src/session/agent-session.ts | 5 ++++- .../coding-agent/test/agent-session-message-pipeline.test.ts | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 5e11ce750..09054e5c8 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -14703,10 +14703,13 @@ export class AgentSession { // Side-channel turns must not share OpenAI/Codex append-only // conversation state with the main agent turn: IRC and /btw can run // while the main turn is mid-tool-call. Keep the prompt-cache key - // stable, but give provider routing a unique request lineage. + // stable, but give provider routing a unique request lineage. The + // shared provider state map is still required so Codex can allocate + // websocket state under that side-channel session id. sessionId: `${cacheSessionId}:side:${Snowflake.next()}`, promptCacheKey: cacheSessionId, preferWebsockets: this.#preferWebsockets, + providerSessionState: this.#providerSessionState, reasoning: toReasoningEffort(this.thinkingLevel), disableReasoning: shouldDisableReasoning(this.thinkingLevel), hideThinkingSummary: this.agent.hideThinkingSummary, diff --git a/packages/coding-agent/test/agent-session-message-pipeline.test.ts b/packages/coding-agent/test/agent-session-message-pipeline.test.ts index cb9528eb2..f849e0aa0 100644 --- a/packages/coding-agent/test/agent-session-message-pipeline.test.ts +++ b/packages/coding-agent/test/agent-session-message-pipeline.test.ts @@ -233,7 +233,7 @@ describe("AgentSession message pipeline", () => { expect(requestOnPayload).toHaveBeenCalledWith({ original: true, session: true }, undefined); expect(result).toEqual({ original: true, session: true }); }); - it("keeps ephemeral side-channel cache key separate from provider routing while preserving websocket preference", async () => { + it("keeps ephemeral side-channel cache key separate from provider routing while preserving websocket state", async () => { const api = "test-ephemeral-side-channel"; let capturedOptions: SimpleStreamOptions | undefined; registerCustomApi(api, (_model, _context, options) => { @@ -283,6 +283,7 @@ describe("AgentSession message pipeline", () => { expect(capturedOptions?.sessionId).toStartWith(`${cacheSessionId}:side:`); expect(capturedOptions?.sessionId).not.toBe(cacheSessionId); expect(capturedOptions?.preferWebsockets).toBe(true); + expect(capturedOptions?.providerSessionState).toBe(session.providerSessionState); }); it("runs ephemeral side-channel requests through the configured side stream function", async () => { From 4181ef18b51ab0c4d5cb2622b3a5300bbb04f3bf Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 20:35:02 +0200 Subject: [PATCH 03/80] build(coding-agent): prevented embedding native runtime dependencies - Standardized `fastembed` and `onnxruntime-node` as external dependencies for all compiled binaries. - Removed manual exclusion logic from individual build scripts in favor of a centralized configuration. - Updated release build dry-run output to reflect consistent external dependency handling. Fixes #5195 --- packages/coding-agent/CHANGELOG.md | 4 ++++ packages/coding-agent/scripts/build-binary.ts | 1 - packages/coding-agent/scripts/compile-binary.ts | 7 ++++--- scripts/ci-release-build-binaries.test.ts | 1 + scripts/ci-release-build-binaries.ts | 6 ++++-- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 30fc7c72b..b38f337ee 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed compiled release binaries bundling `fastembed` and baking the build-machine `@anush008/tokenizers` path; native runtime dependencies now stay external for every compiled build path so Mnemopi resolves its on-demand install instead. ([#5195](https://github.com/can1357/oh-my-pi/issues/5195)) + ## [16.4.5] - 2026-07-11 ### Breaking Changes diff --git a/packages/coding-agent/scripts/build-binary.ts b/packages/coding-agent/scripts/build-binary.ts index 221ade91c..1fb51fcfa 100644 --- a/packages/coding-agent/scripts/build-binary.ts +++ b/packages/coding-agent/scripts/build-binary.ts @@ -99,7 +99,6 @@ async function main(): Promise { outfile: outputPath, transformersVersion, target: crossBuild?.target, - external: ["fastembed", "onnxruntime-node"], skipBuiltinCodesign: shouldAdhocSignDarwinBinary(crossBuild), }); diff --git a/packages/coding-agent/scripts/compile-binary.ts b/packages/coding-agent/scripts/compile-binary.ts index 5e7d27710..495612041 100644 --- a/packages/coding-agent/scripts/compile-binary.ts +++ b/packages/coding-agent/scripts/compile-binary.ts @@ -1,6 +1,9 @@ import { buildDocsIndexPayload } from "./generate-docs-index"; import { createLegacyPiVirtualModulePlugin } from "./legacy-pi-virtual-module"; +/** Native runtime dependencies always resolved from the on-demand install instead of embedded into compiled binaries. */ +export const COMPILED_EXTERNAL_DEPENDENCIES: readonly string[] = Object.freeze(["fastembed", "onnxruntime-node"]); + /** Inputs shared by local and release coding-agent binary builds. */ export interface CodingAgentCompileOptions { /** Absolute repository root used for package resolution. */ @@ -13,8 +16,6 @@ export interface CodingAgentCompileOptions { readonly transformersVersion: string; /** Optional cross-compilation runtime target. */ readonly target?: Bun.Build.CompileTarget; - /** Dependencies intentionally resolved from the runtime filesystem. */ - readonly external?: readonly string[]; /** Match release builds that minify identifiers while retaining names. */ readonly minifyIdentifiers?: boolean; /** Disable Bun's built-in Darwin signing before the caller re-signs. */ @@ -34,7 +35,7 @@ export async function compileCodingAgent(options: CodingAgentCompileOptions): Pr const output = await Bun.build({ entrypoints: [options.entrypoint], root: options.repoRoot, - external: options.external ? [...options.external] : undefined, + external: [...COMPILED_EXTERNAL_DEPENDENCIES], define: { "process.env.PI_COMPILED": JSON.stringify("true"), "process.env.PI_TINY_TRANSFORMERS_VERSION": JSON.stringify(options.transformersVersion), diff --git a/scripts/ci-release-build-binaries.test.ts b/scripts/ci-release-build-binaries.test.ts index 19cb6583b..7c65f5f17 100644 --- a/scripts/ci-release-build-binaries.test.ts +++ b/scripts/ci-release-build-binaries.test.ts @@ -18,6 +18,7 @@ describe("Windows release binary target", () => { expect(output).toContain( "DRY RUN Bun.build target=bun-windows-x64-baseline outfile=packages/coding-agent/binaries/omp-windows-x64.exe", ); + expect(output).toContain("external=fastembed,onnxruntime-node"); expect(output).not.toContain("bun-windows-x64-modern"); }); diff --git a/scripts/ci-release-build-binaries.ts b/scripts/ci-release-build-binaries.ts index bcae77ebb..e2dc67071 100644 --- a/scripts/ci-release-build-binaries.ts +++ b/scripts/ci-release-build-binaries.ts @@ -3,7 +3,7 @@ import * as fs from "node:fs/promises"; import { createRequire } from "node:module"; import * as path from "node:path"; -import { compileCodingAgent } from "../packages/coding-agent/scripts/compile-binary"; +import { COMPILED_EXTERNAL_DEPENDENCIES, compileCodingAgent } from "../packages/coding-agent/scripts/compile-binary"; interface BinaryTarget { id: string; @@ -121,7 +121,9 @@ async function buildBinary(target: BinaryTarget): Promise { console.log(`Building ${target.outfile}...`); await embedNative(target); if (isDryRun) { - console.log(`DRY RUN Bun.build target=${target.target} outfile=${target.outfile}`); + console.log( + `DRY RUN Bun.build target=${target.target} outfile=${target.outfile} external=${COMPILED_EXTERNAL_DEPENDENCIES.join(",")}`, + ); return; } From 6c292b97c30489ead2b79df754ca3d9a48c4445e Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 20:35:43 +0200 Subject: [PATCH 04/80] refactor(coding-agent): preserved completed and abandoned tasks in session - Updated task synchronization to include all todo phases regardless of completion status. - Removed logic that filtered out completed or abandoned tasks during session synchronization. --- packages/coding-agent/CHANGELOG.md | 4 ++++ .../src/modes/controllers/todo-command-controller.ts | 3 +-- packages/coding-agent/src/session/agent-session.ts | 7 +------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b38f337ee..7c7964669 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- Retain completed and abandoned tasks in session history for improved context on resume + ### Fixed - Fixed compiled release binaries bundling `fastembed` and baking the build-machine `@anush008/tokenizers` path; native runtime dependencies now stay external for every compiled build path so Mnemopi resolves its on-demand install instead. ([#5195](https://github.com/can1357/oh-my-pi/issues/5195)) diff --git a/packages/coding-agent/src/modes/controllers/todo-command-controller.ts b/packages/coding-agent/src/modes/controllers/todo-command-controller.ts index 3f3855268..9df53fa87 100644 --- a/packages/coding-agent/src/modes/controllers/todo-command-controller.ts +++ b/packages/coding-agent/src/modes/controllers/todo-command-controller.ts @@ -133,8 +133,7 @@ export class TodoCommandController { /** * True latest todo state for the user-facing /todo verbs. Reads from session - * entries so that completed/abandoned tasks remain visible after resume - * (where `session.getTodoPhases()` would have stripped them). + * entries or falls back to the active session state. */ #currentPhases(): TodoPhase[] { const fromEntries = getLatestTodoPhasesFromEntries(this.ctx.sessionManager.getBranch()); diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 479cc615b..694df6024 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -8724,12 +8724,7 @@ export class AgentSession { #syncTodoPhasesFromBranch(): void { const phases = getLatestTodoPhasesFromEntries(this.sessionManager.getBranch()); - // Strip completed/abandoned tasks — they were done in a previous run, - // so they have no bearing on progress tracking for the new turn. - for (const phase of phases) { - phase.tasks = phase.tasks.filter(t => t.status !== "completed" && t.status !== "abandoned"); - } - this.setTodoPhases(phases.filter(p => p.tasks.length > 0)); + this.setTodoPhases(phases); } #cloneTodoPhases(phases: TodoPhase[]): TodoPhase[] { From 0ae8efd64973b7f93b8b66201f42a974e36fc2d8 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 20:35:56 +0200 Subject: [PATCH 05/80] feat: integrated coreutils as in-process shell builtins - Integrated 17 new coreutils-based shell builtins including `diff`, `date`, `ln`, `stat`, `seq`, `touch`, and others. - Refactored vendored utilities to execute as in-process shell builtins by routing I/O, environment access, and path resolution through `pi_uutils_ctx`. - Disabled process-level modifications (e.g., clock setting, hostname modification) to ensure safety and scope adherence within the shell environment. - Implemented shell-specific features such as cancellation polling, custom exit code management, and efficient output streaming for all new builtins. --- Cargo.lock | 275 ++- crates/pi-shell/Cargo.toml | 17 + crates/pi-shell/src/coreutils.rs | 17 + crates/pi-shell/src/lib.rs | 1 + crates/pi-shell/src/shell.rs | 19 + crates/pi-shell/src/which.rs | 252 +++ crates/pi-uu-diff/Cargo.toml | 21 + crates/pi-uu-diff/src/lib.rs | 608 ++++++ crates/vendor/uu-date/Cargo.toml | 36 + crates/vendor/uu-date/LICENSE | 18 + crates/vendor/uu-date/src/date.rs | 1336 ++++++++++++ crates/vendor/uu-date/src/format_modifiers.rs | 760 +++++++ crates/vendor/uu-hostname/Cargo.toml | 31 + crates/vendor/uu-hostname/LICENSE | 18 + crates/vendor/uu-hostname/src/hostname.rs | 308 +++ crates/vendor/uu-ln/Cargo.toml | 23 + crates/vendor/uu-ln/LICENSE | 18 + crates/vendor/uu-ln/src/ln.rs | 831 ++++++++ crates/vendor/uu-mktemp/Cargo.toml | 24 + crates/vendor/uu-mktemp/LICENSE | 18 + crates/vendor/uu-mktemp/src/mktemp.rs | 877 ++++++++ crates/vendor/uu-nproc/Cargo.toml | 22 + crates/vendor/uu-nproc/LICENSE | 18 + crates/vendor/uu-nproc/src/nproc.rs | 287 +++ crates/vendor/uu-printenv/Cargo.toml | 21 + crates/vendor/uu-printenv/LICENSE | 18 + crates/vendor/uu-printenv/src/printenv.rs | 243 +++ crates/vendor/uu-readlink/Cargo.toml | 22 + crates/vendor/uu-readlink/LICENSE | 18 + crates/vendor/uu-readlink/src/readlink.rs | 422 ++++ crates/vendor/uu-realpath/Cargo.toml | 22 + crates/vendor/uu-realpath/LICENSE | 18 + crates/vendor/uu-realpath/src/realpath.rs | 568 ++++++ crates/vendor/uu-seq/Cargo.toml | 32 + crates/vendor/uu-seq/LICENSE | 18 + crates/vendor/uu-seq/src/error.rs | 57 + crates/vendor/uu-seq/src/number.rs | 52 + crates/vendor/uu-seq/src/numberparse.rs | 351 ++++ crates/vendor/uu-seq/src/seq.rs | 561 ++++++ crates/vendor/uu-stat/Cargo.toml | 23 + crates/vendor/uu-stat/LICENSE | 18 + crates/vendor/uu-stat/src/stat.rs | 1785 +++++++++++++++++ crates/vendor/uu-tac/Cargo.toml | 26 + crates/vendor/uu-tac/LICENSE | 18 + crates/vendor/uu-tac/src/error.rs | 47 + crates/vendor/uu-tac/src/tac.rs | 634 ++++++ crates/vendor/uu-touch/Cargo.toml | 36 + crates/vendor/uu-touch/LICENSE | 18 + crates/vendor/uu-touch/src/touch.rs | 1170 +++++++++++ crates/vendor/uu-truncate/Cargo.toml | 22 + crates/vendor/uu-truncate/LICENSE | 18 + crates/vendor/uu-truncate/src/truncate.rs | 557 +++++ crates/vendor/uu-uname/Cargo.toml | 21 + crates/vendor/uu-uname/LICENSE | 18 + crates/vendor/uu-uname/src/uname.rs | 342 ++++ crates/vendor/uu-whoami/Cargo.toml | 27 + crates/vendor/uu-whoami/LICENSE | 18 + crates/vendor/uu-whoami/src/whoami.rs | 198 ++ crates/vendor/uu-yes/Cargo.toml | 23 + crates/vendor/uu-yes/LICENSE | 18 + crates/vendor/uu-yes/src/yes.rs | 351 ++++ packages/natives/CHANGELOG.md | 5 + 62 files changed, 13648 insertions(+), 3 deletions(-) create mode 100644 crates/pi-shell/src/which.rs create mode 100644 crates/pi-uu-diff/Cargo.toml create mode 100644 crates/pi-uu-diff/src/lib.rs create mode 100644 crates/vendor/uu-date/Cargo.toml create mode 100644 crates/vendor/uu-date/LICENSE create mode 100644 crates/vendor/uu-date/src/date.rs create mode 100644 crates/vendor/uu-date/src/format_modifiers.rs create mode 100644 crates/vendor/uu-hostname/Cargo.toml create mode 100644 crates/vendor/uu-hostname/LICENSE create mode 100644 crates/vendor/uu-hostname/src/hostname.rs create mode 100644 crates/vendor/uu-ln/Cargo.toml create mode 100644 crates/vendor/uu-ln/LICENSE create mode 100644 crates/vendor/uu-ln/src/ln.rs create mode 100644 crates/vendor/uu-mktemp/Cargo.toml create mode 100644 crates/vendor/uu-mktemp/LICENSE create mode 100644 crates/vendor/uu-mktemp/src/mktemp.rs create mode 100644 crates/vendor/uu-nproc/Cargo.toml create mode 100644 crates/vendor/uu-nproc/LICENSE create mode 100644 crates/vendor/uu-nproc/src/nproc.rs create mode 100644 crates/vendor/uu-printenv/Cargo.toml create mode 100644 crates/vendor/uu-printenv/LICENSE create mode 100644 crates/vendor/uu-printenv/src/printenv.rs create mode 100644 crates/vendor/uu-readlink/Cargo.toml create mode 100644 crates/vendor/uu-readlink/LICENSE create mode 100644 crates/vendor/uu-readlink/src/readlink.rs create mode 100644 crates/vendor/uu-realpath/Cargo.toml create mode 100644 crates/vendor/uu-realpath/LICENSE create mode 100644 crates/vendor/uu-realpath/src/realpath.rs create mode 100644 crates/vendor/uu-seq/Cargo.toml create mode 100644 crates/vendor/uu-seq/LICENSE create mode 100644 crates/vendor/uu-seq/src/error.rs create mode 100644 crates/vendor/uu-seq/src/number.rs create mode 100644 crates/vendor/uu-seq/src/numberparse.rs create mode 100644 crates/vendor/uu-seq/src/seq.rs create mode 100644 crates/vendor/uu-stat/Cargo.toml create mode 100644 crates/vendor/uu-stat/LICENSE create mode 100644 crates/vendor/uu-stat/src/stat.rs create mode 100644 crates/vendor/uu-tac/Cargo.toml create mode 100644 crates/vendor/uu-tac/LICENSE create mode 100644 crates/vendor/uu-tac/src/error.rs create mode 100644 crates/vendor/uu-tac/src/tac.rs create mode 100644 crates/vendor/uu-touch/Cargo.toml create mode 100644 crates/vendor/uu-touch/LICENSE create mode 100644 crates/vendor/uu-touch/src/touch.rs create mode 100644 crates/vendor/uu-truncate/Cargo.toml create mode 100644 crates/vendor/uu-truncate/LICENSE create mode 100644 crates/vendor/uu-truncate/src/truncate.rs create mode 100644 crates/vendor/uu-uname/Cargo.toml create mode 100644 crates/vendor/uu-uname/LICENSE create mode 100644 crates/vendor/uu-uname/src/uname.rs create mode 100644 crates/vendor/uu-whoami/Cargo.toml create mode 100644 crates/vendor/uu-whoami/LICENSE create mode 100644 crates/vendor/uu-whoami/src/whoami.rs create mode 100644 crates/vendor/uu-yes/Cargo.toml create mode 100644 crates/vendor/uu-yes/LICENSE create mode 100644 crates/vendor/uu-yes/src/yes.rs diff --git a/Cargo.lock b/Cargo.lock index 5cd6d3ef9..cc40d1526 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1115,6 +1115,18 @@ dependencies = [ "syn", ] +[[package]] +name = "dns-lookup" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" +dependencies = [ + "cfg-if", + "libc", + "socket2", + "windows-sys 0.60.2", +] + [[package]] name = "downcast-rs" version = "1.2.1" @@ -3022,6 +3034,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parse_datetime" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413775a7eac2261d2211a79d10ef275e5b6f7b527eec42ad09adce2ffa92b6e5" +dependencies = [ + "jiff", + "num-traits", + "winnow 0.7.15", +] + [[package]] name = "pcre2" version = "0.2.11" @@ -3392,6 +3415,7 @@ dependencies = [ "parking_lot", "pi-uutils-ctx", "pi-walker", + "pi_uu_diff", "pi_uu_grep", "regex", "serde", @@ -3406,28 +3430,44 @@ dependencies = [ "uu_cat", "uu_comm", "uu_cut", + "uu_date", "uu_dirname", "uu_find", "uu_head", + "uu_hostname", + "uu_ln", "uu_ls", "uu_md5sum", "uu_mkdir", + "uu_mktemp", "uu_mv", + "uu_nproc", "uu_paste", + "uu_printenv", + "uu_readlink", + "uu_realpath", "uu_rm", "uu_sed", + "uu_seq", "uu_sha1sum", "uu_sha224sum", "uu_sha256sum", "uu_sha384sum", "uu_sha512sum", "uu_sort", + "uu_stat", + "uu_tac", "uu_tail", "uu_tee", + "uu_touch", "uu_tr", + "uu_truncate", + "uu_uname", "uu_uniq", "uu_wc", + "uu_whoami", "uu_xargs", + "uu_yes", "windows-sys 0.61.2", "winreg 0.56.0", "xxhash-rust", @@ -3453,6 +3493,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pi_uu_diff" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "similar 3.1.1", + "tempfile", +] + [[package]] name = "pi_uu_grep" version = "0.8.0" @@ -3484,6 +3535,16 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "platform-info" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9368d62437c8cbb7c31ee37fd8c08a7d390e09a3ff75698a674953f46705ffcb" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "png" version = "0.18.1" @@ -4474,7 +4535,7 @@ dependencies = [ "toml_datetime", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.3", ] [[package]] @@ -4495,7 +4556,7 @@ dependencies = [ "indexmap", "toml_datetime", "toml_parser", - "winnow", + "winnow 1.0.3", ] [[package]] @@ -4504,7 +4565,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.3", ] [[package]] @@ -5338,6 +5399,21 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_date" +version = "0.8.0" +dependencies = [ + "clap", + "jiff", + "parking_lot", + "parse_datetime", + "pi-uutils-ctx", + "regex", + "rustix", + "tempfile", + "uucore 0.8.0", +] + [[package]] name = "uu_dirname" version = "0.8.0" @@ -5377,6 +5453,31 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_hostname" +version = "0.8.0" +dependencies = [ + "clap", + "dns-lookup", + "hostname", + "parking_lot", + "pi-uutils-ctx", + "uucore 0.8.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "uu_ln" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "tempfile", + "thiserror 2.0.18", + "uucore 0.8.0", +] + [[package]] name = "uu_ls" version = "0.8.0" @@ -5412,6 +5513,19 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_mktemp" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "rand 0.10.2", + "tempfile", + "thiserror 2.0.18", + "uucore 0.8.0", +] + [[package]] name = "uu_mv" version = "0.8.0" @@ -5427,6 +5541,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "uu_nproc" +version = "0.8.0" +dependencies = [ + "clap", + "libc", + "parking_lot", + "pi-uutils-ctx", + "uucore 0.8.0", +] + [[package]] name = "uu_paste" version = "0.8.0" @@ -5436,6 +5561,38 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_printenv" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "uucore 0.8.0", +] + +[[package]] +name = "uu_readlink" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "tempfile", + "uucore 0.8.0", +] + +[[package]] +name = "uu_realpath" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "tempfile", + "uucore 0.8.0", +] + [[package]] name = "uu_rm" version = "0.8.0" @@ -5464,6 +5621,20 @@ dependencies = [ "uucore 0.9.0", ] +[[package]] +name = "uu_seq" +version = "0.8.0" +dependencies = [ + "bigdecimal", + "clap", + "num-bigint", + "num-traits", + "parking_lot", + "pi-uutils-ctx", + "thiserror 2.0.18", + "uucore 0.8.0", +] + [[package]] name = "uu_sha1sum" version = "0.8.0" @@ -5536,6 +5707,33 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_stat" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "tempfile", + "thiserror 2.0.18", + "uucore 0.8.0", +] + +[[package]] +name = "uu_tac" +version = "0.8.0" +dependencies = [ + "clap", + "memchr", + "memmap2", + "parking_lot", + "pi-uutils-ctx", + "regex", + "tempfile", + "thiserror 2.0.18", + "uucore 0.8.0", +] + [[package]] name = "uu_tail" version = "0.8.0" @@ -5560,6 +5758,24 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_touch" +version = "0.8.0" +dependencies = [ + "clap", + "filetime", + "jiff", + "libc", + "parking_lot", + "parse_datetime", + "pi-uutils-ctx", + "rustix", + "tempfile", + "thiserror 2.0.18", + "uucore 0.8.0", + "windows-sys 0.61.2", +] + [[package]] name = "uu_tr" version = "0.8.0" @@ -5571,6 +5787,28 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_truncate" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "tempfile", + "uucore 0.8.0", +] + +[[package]] +name = "uu_uname" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "platform-info", + "uucore 0.8.0", +] + [[package]] name = "uu_uniq" version = "0.8.0" @@ -5594,6 +5832,17 @@ dependencies = [ "uucore 0.8.0", ] +[[package]] +name = "uu_whoami" +version = "0.8.0" +dependencies = [ + "clap", + "parking_lot", + "pi-uutils-ctx", + "uucore 0.8.0", + "windows-sys 0.61.2", +] + [[package]] name = "uu_xargs" version = "0.8.0" @@ -5605,6 +5854,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "uu_yes" +version = "0.8.0" +dependencies = [ + "clap", + "itertools", + "parking_lot", + "pi-uutils-ctx", + "uucore 0.8.0", +] + [[package]] name = "uucore" version = "0.0.30" @@ -6330,6 +6590,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.3" diff --git a/crates/pi-shell/Cargo.toml b/crates/pi-shell/Cargo.toml index adad30dd9..f62d2d3b3 100644 --- a/crates/pi-shell/Cargo.toml +++ b/crates/pi-shell/Cargo.toml @@ -54,6 +54,23 @@ uu_sha512sum = { path = "../vendor/uu-sha512sum" } uu_b2sum = { path = "../vendor/uu-b2sum" } uu_basename = { path = "../vendor/uu-basename" } uu_dirname = { path = "../vendor/uu-dirname" } +uu_readlink = { path = "../vendor/uu-readlink" } +uu_realpath = { path = "../vendor/uu-realpath" } +uu_touch = { path = "../vendor/uu-touch" } +uu_stat = { path = "../vendor/uu-stat" } +uu_date = { path = "../vendor/uu-date" } +uu_mktemp = { path = "../vendor/uu-mktemp" } +uu_seq = { path = "../vendor/uu-seq" } +uu_yes = { path = "../vendor/uu-yes" } +uu_printenv = { path = "../vendor/uu-printenv" } +uu_ln = { path = "../vendor/uu-ln" } +uu_truncate = { path = "../vendor/uu-truncate" } +uu_tac = { path = "../vendor/uu-tac" } +uu_nproc = { path = "../vendor/uu-nproc" } +uu_uname = { path = "../vendor/uu-uname" } +uu_whoami = { path = "../vendor/uu-whoami" } +uu_hostname = { path = "../vendor/uu-hostname" } +pi_uu_diff = { path = "../pi-uu-diff" } uu_cut = { path = "../vendor/uu-cut" } uu_tee = { path = "../vendor/uu-tee" } uu_tr = { path = "../vendor/uu-tr" } diff --git a/crates/pi-shell/src/coreutils.rs b/crates/pi-shell/src/coreutils.rs index a1372f404..caf7c064c 100644 --- a/crates/pi-shell/src/coreutils.rs +++ b/crates/pi-shell/src/coreutils.rs @@ -219,6 +219,23 @@ uutil_builtin!(pub fn sha512sum_builtin => uu_sha512sum::run); uutil_builtin!(pub fn b2sum_builtin => uu_b2sum::run); uutil_builtin!(pub fn basename_builtin => uu_basename::run); uutil_builtin!(pub fn dirname_builtin => uu_dirname::run); +uutil_builtin!(pub fn readlink_builtin => uu_readlink::run); +uutil_builtin!(pub fn realpath_builtin => uu_realpath::run); +uutil_builtin!(pub fn touch_builtin => uu_touch::run); +uutil_builtin!(pub fn stat_builtin => uu_stat::run); +uutil_builtin!(pub fn date_builtin => uu_date::run); +uutil_builtin!(pub fn mktemp_builtin => uu_mktemp::run); +uutil_builtin!(pub fn seq_builtin => uu_seq::run); +uutil_builtin!(pub fn yes_builtin => uu_yes::run); +uutil_builtin!(pub fn printenv_builtin => uu_printenv::run); +uutil_builtin!(pub fn ln_builtin => uu_ln::run); +uutil_builtin!(pub fn truncate_builtin => uu_truncate::run); +uutil_builtin!(pub fn tac_builtin => uu_tac::run); +uutil_builtin!(pub fn nproc_builtin => uu_nproc::run); +uutil_builtin!(pub fn uname_builtin => uu_uname::run); +uutil_builtin!(pub fn whoami_builtin => uu_whoami::run); +uutil_builtin!(pub fn hostname_builtin => uu_hostname::run); +uutil_builtin!(pub fn diff_builtin => pi_uu_diff::run); uutil_builtin!(pub fn cut_builtin => uu_cut::run); uutil_builtin!(pub fn tee_builtin => uu_tee::run); uutil_builtin!(pub fn tr_builtin => uu_tr::run); diff --git a/crates/pi-shell/src/lib.rs b/crates/pi-shell/src/lib.rs index 2f3f5929f..204724703 100644 --- a/crates/pi-shell/src/lib.rs +++ b/crates/pi-shell/src/lib.rs @@ -4,6 +4,7 @@ mod fd; pub mod minimizer; pub mod process; pub mod shell; +mod which; #[cfg(windows)] pub mod windows; diff --git a/crates/pi-shell/src/shell.rs b/crates/pi-shell/src/shell.rs index 46e454e8f..f2254a258 100644 --- a/crates/pi-shell/src/shell.rs +++ b/crates/pi-shell/src/shell.rs @@ -632,6 +632,23 @@ async fn create_session_for_run( shell.register_builtin("b2sum", crate::coreutils::b2sum_builtin()); shell.register_builtin("basename", crate::coreutils::basename_builtin()); shell.register_builtin("dirname", crate::coreutils::dirname_builtin()); + shell.register_builtin("readlink", crate::coreutils::readlink_builtin()); + shell.register_builtin("realpath", crate::coreutils::realpath_builtin()); + shell.register_builtin("touch", crate::coreutils::touch_builtin()); + shell.register_builtin("stat", crate::coreutils::stat_builtin()); + shell.register_builtin("date", crate::coreutils::date_builtin()); + shell.register_builtin("mktemp", crate::coreutils::mktemp_builtin()); + shell.register_builtin("seq", crate::coreutils::seq_builtin()); + shell.register_builtin("yes", crate::coreutils::yes_builtin()); + shell.register_builtin("printenv", crate::coreutils::printenv_builtin()); + shell.register_builtin("truncate", crate::coreutils::truncate_builtin()); + shell.register_builtin("tac", crate::coreutils::tac_builtin()); + shell.register_builtin("nproc", crate::coreutils::nproc_builtin()); + shell.register_builtin("uname", crate::coreutils::uname_builtin()); + shell.register_builtin("whoami", crate::coreutils::whoami_builtin()); + shell.register_builtin("hostname", crate::coreutils::hostname_builtin()); + shell.register_builtin("which", crate::which::which_builtin()); + shell.register_builtin("diff", crate::coreutils::diff_builtin()); shell.register_builtin("cut", crate::coreutils::cut_builtin()); shell.register_builtin("tee", crate::coreutils::tee_builtin()); shell.register_builtin("tr", crate::coreutils::tr_builtin()); @@ -647,6 +664,8 @@ async fn create_session_for_run( if !uutils_env_disabled(config, "PI_DISABLE_MV_BUILTIN") { shell.register_builtin("mv", crate::coreutils::mv_builtin()); } + // ln can clobber existing files via -f; gate it with the destructive set. + shell.register_builtin("ln", crate::coreutils::ln_builtin()); } } diff --git a/crates/pi-shell/src/which.rs b/crates/pi-shell/src/which.rs new file mode 100644 index 000000000..253a414e7 --- /dev/null +++ b/crates/pi-shell/src/which.rs @@ -0,0 +1,252 @@ +//! In-process `which` builtin backed by brush's PATH-search helpers. +//! +//! Follows which(1) (GNU/debianutils) semantics: each name operand is looked +//! up in the shell's `PATH`; the first match is printed (all matches with +//! `-a`). Lookup failures are silent; the exit status is 0 when every name +//! was found and 1 when any name was missing. + +use std::{ + ffi::OsString, + io::{self, Write}, + path::{Path, PathBuf}, +}; + +use brush_core::{ + Error, + builtins::{BoxFuture, ContentOptions, ContentType, Registration}, + commands::{CommandArg, ExecutionContext}, + extensions::ShellExtensions, + openfiles::{OpenFile, OpenFiles, null}, + pathsearch, + results::ExecutionResult, + sys, +}; +use clap::{Parser, error::ErrorKind}; + +#[derive(Parser, Debug)] +#[command(name = "which", about = "Locate a command's executable in the shell's PATH")] +struct WhichCli { + /// Print all matching executables in PATH, not just the first. + #[arg(short = 'a', long = "all")] + all: bool, + + /// Command names to locate. + #[arg(value_name = "name")] + names: Vec, +} + +/// Creates the `which` shell builtin registration. +pub fn which_builtin() -> Registration { + fn execute( + context: ExecutionContext<'_, SE>, + args: Vec, + ) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(Ok(run_which(context, args)))) + } + + Registration { + execute_func: execute::, + content_func: which_content, + disabled: false, + special_builtin: false, + declaration_builtin: false, + transparent_background_wrapper: false, + } +} + +fn run_which( + context: ExecutionContext<'_, SE>, + args: Vec, +) -> ExecutionResult { + let mut stdout = context + .try_fd(OpenFiles::STDOUT_FD) + .unwrap_or_else(null_sink); + let mut stderr = context + .try_fd(OpenFiles::STDERR_FD) + .unwrap_or_else(null_sink); + let cwd = context.shell.working_dir().to_path_buf(); + let path_var = context + .shell + .env_str("PATH") + .map(std::borrow::Cow::into_owned) + .unwrap_or_default(); + let argv: Vec = args + .iter() + .map(|arg| OsString::from(arg.to_string())) + .collect(); + + let cli = match WhichCli::try_parse_from(argv) { + Ok(cli) => cli, + Err(err) => { + let rendered = err.to_string(); + let code = match err.kind() { + ErrorKind::DisplayHelp | ErrorKind::DisplayVersion => { + let _ = write!(stdout, "{rendered}"); + 0 + }, + _ => { + let _ = write!(stderr, "{rendered}"); + 2 + }, + }; + return ExecutionResult::new(code); + }, + }; + + let mut all_found = true; + for name in &cli.names { + let matches = find_matches(name, &path_var, &cwd, cli.all); + if matches.is_empty() { + // which(1) reports missing names via the exit status only. + all_found = false; + } + for path in matches { + let _ = writeln!(stdout, "{}", path.display()); + } + } + + ExecutionResult::new(u8::from(!all_found)) +} + +/// Collects the executable matches for a single `which` name operand. +/// +/// A name containing a path separator is checked directly against `cwd` +/// (yielding at most one match); otherwise each `PATH` entry — with relative +/// and empty entries resolved against `cwd` — is probed in `PATH` order. +/// Returns only the first match unless `all` is set. Windows `PATHEXT` +/// resolution is handled by [`brush_core::sys::fs::resolve_executable`]. +fn find_matches(name: &str, path_var: &str, cwd: &Path, all: bool) -> Vec { + if sys::fs::contains_path_separator(name) { + let candidate = cwd.join(name); + if candidate.is_dir() { + return Vec::new(); + } + return sys::fs::resolve_executable(candidate).into_iter().collect(); + } + + let dirs = sys::fs::split_paths(path_var).map(|dir| { + if dir.as_os_str().is_empty() { + // POSIX: an empty PATH entry names the current directory. + cwd.to_path_buf() + } else if dir.is_relative() { + cwd.join(dir) + } else { + dir + } + }); + + let mut found = pathsearch::search_for_executable(dirs, name); + if all { + found.collect() + } else { + found.next().into_iter().collect() + } +} + +fn null_sink() -> OpenFile { + null().unwrap_or_else(|_| OpenFile::from(io::stdout())) +} + +#[allow( + clippy::unnecessary_wraps, + reason = "signature must match brush's CommandContentFunc fn pointer" +)] +fn which_content( + _name: &str, + _content_type: ContentType, + _options: &ContentOptions, +) -> Result { + Ok("which: which [-a] name [name ...]\n".to_string()) +} + +#[cfg(test)] +#[cfg(unix)] +mod tests { + use std::{ + env, fs, + os::unix::fs::PermissionsExt, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::{SystemTime, UNIX_EPOCH}, + }; + + use super::find_matches; + + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + /// Creates a fresh, canonicalized temp directory (macOS `/var` is a + /// symlink; canonicalizing keeps constructed and probed paths identical). + fn temp_root(tag: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let root = env::temp_dir().join(format!( + "pi-shell-which-{tag}-{}-{}-{}", + std::process::id(), + nanos, + COUNTER.fetch_add(1, Ordering::Relaxed), + )); + fs::create_dir_all(&root).expect("temp dir should be created"); + fs::canonicalize(&root).expect("temp dir should canonicalize") + } + + fn place_file(dir: &std::path::Path, name: &str, executable: bool) -> PathBuf { + let path = dir.join(name); + fs::write(&path, b"#!/bin/sh\n").expect("file should be written"); + let mode = if executable { 0o755 } else { 0o644 }; + fs::set_permissions(&path, fs::Permissions::from_mode(mode)) + .expect("permissions should be set"); + path + } + + #[test] + fn finds_only_executable_files() { + let dir = temp_root("exec-only"); + let tool = place_file(&dir, "tool", true); + place_file(&dir, "blob", false); + let path_var = dir.display().to_string(); + + assert_eq!(find_matches("tool", &path_var, &dir, false), vec![tool]); + assert!(find_matches("blob", &path_var, &dir, false).is_empty()); + assert!(find_matches("missing", &path_var, &dir, false).is_empty()); + } + + #[test] + fn all_flag_returns_matches_in_path_order() { + let dir_a = temp_root("all-a"); + let dir_b = temp_root("all-b"); + let tool_a = place_file(&dir_a, "tool", true); + let tool_b = place_file(&dir_b, "tool", true); + let path_var = format!("{}:{}", dir_a.display(), dir_b.display()); + let cwd = temp_root("all-cwd"); + + assert_eq!(find_matches("tool", &path_var, &cwd, true), vec![tool_a.clone(), tool_b]); + // Without -a only the first PATH entry's match is returned. + assert_eq!(find_matches("tool", &path_var, &cwd, false), vec![tool_a]); + } + + #[test] + fn name_with_separator_resolves_against_cwd() { + let cwd = temp_root("slash"); + let bin = cwd.join("bin"); + fs::create_dir_all(&bin).expect("bin dir should be created"); + let tool = place_file(&bin, "tool", true); + place_file(&bin, "blob", false); + + // PATH is irrelevant for names containing a separator. + assert_eq!(find_matches("bin/tool", "", &cwd, false), vec![tool]); + assert!(find_matches("bin/blob", "", &cwd, false).is_empty()); + // A directory is never a match, even with execute bits set. + assert!(find_matches("./bin", "", &cwd, false).is_empty()); + } + + #[test] + fn relative_path_entries_resolve_against_cwd() { + let cwd = temp_root("rel-entry"); + let bin = cwd.join("bin"); + fs::create_dir_all(&bin).expect("bin dir should be created"); + let tool = place_file(&bin, "tool", true); + + assert_eq!(find_matches("tool", "bin", &cwd, false), vec![tool]); + } +} diff --git a/crates/pi-uu-diff/Cargo.toml b/crates/pi-uu-diff/Cargo.toml new file mode 100644 index 000000000..4d9bed570 --- /dev/null +++ b/crates/pi-uu-diff/Cargo.toml @@ -0,0 +1,21 @@ +# diff implemented from scratch on top of the `similar` diffing library, with +# I/O and path resolution routed through pi-uutils-ctx so it can run in-process +# as a shell builtin. Entry point: `pi_uu_diff::run`. +[package] +name = "pi_uu_diff" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "diff ~ similar-backed file comparison (in-process shell builtin)" + +[lib] +path = "src/lib.rs" + +[dependencies] +clap = { version = "4", features = ["wrap_help"] } +pi-uutils-ctx = { path = "../pi-uutils-ctx" } +similar = "3.1.0" + +[dev-dependencies] +parking_lot.workspace = true +tempfile = "3" diff --git a/crates/pi-uu-diff/src/lib.rs b/crates/pi-uu-diff/src/lib.rs new file mode 100644 index 000000000..330e8833e --- /dev/null +++ b/crates/pi-uu-diff/src/lib.rs @@ -0,0 +1,608 @@ +//! `diff` implemented as an in-process shell builtin on top of the `similar` +//! diffing library. All I/O and path resolution is routed through +//! `pi-uutils-ctx` so the builtin writes to the command's redirected file +//! descriptors and resolves relative paths against the shell's working +//! directory, while operands are printed as typed. +//! +//! Scope: unified output only (`-u` is accepted and implied, `-U N` controls +//! the context size), `-q/--brief`, `-N/--new-file` (absent files compare as +//! empty), binary detection, `-` for the context stdin, and unconditional +//! recursive directory comparison (`Only in : ` lines plus +//! `diff -r A/x B/x`-headed per-pair diffs). +//! +//! Entry point: [`run`]. It never calls `std::process::exit`; clap +//! help/usage/error output is rendered to the context streams and an exit code +//! is returned following the GNU convention (0 = identical, 1 = differences +//! found, 2 = trouble). + +use std::{ + collections::BTreeSet, + ffi::{OsStr, OsString}, + fs, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use pi_uutils_ctx::format_usage; +use similar::TextDiff; + +const OPT_UNIFIED_FLAG: &str = "unified-flag"; +const OPT_UNIFIED: &str = "unified"; +const OPT_BRIEF: &str = "brief"; +const OPT_RECURSIVE: &str = "recursive"; +const OPT_NEW_FILE: &str = "new-file"; +const OPT_COLOR: &str = "color"; +const ARG_FILES: &str = "files"; + +/// In-process builtin entry point. Parses the arguments directly, renders clap +/// help/usage/version to the context streams, and maps errors to the GNU diff +/// exit-code convention, so it is safe to run inside the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 2; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match diff_main(&matches) { + Ok(code) => code, + Err(msg) => { + let _ = writeln!(pi_uutils_ctx::stderr(), "diff: {msg}"); + 2 + }, + } +} + +pub fn uu_app() -> Command { + Command::new("diff") + .version(concat!("diff (pi-uu-diff) ", env!("CARGO_PKG_VERSION"))) + .about("Compare files line by line.") + .override_usage(format_usage("diff [OPTION]... FILE1 FILE2")) + .infer_long_args(true) + .arg( + Arg::new(OPT_UNIFIED_FLAG) + .short('u') + .help("output 3 lines of unified context (the default output format)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_UNIFIED) + .short('U') + .long(OPT_UNIFIED) + .value_name("NUM") + .help("output NUM lines of unified context") + .value_parser(clap::value_parser!(usize)), + ) + .arg( + Arg::new(OPT_BRIEF) + .short('q') + .long(OPT_BRIEF) + .help("report only when files differ") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_RECURSIVE) + .short('r') + .long(OPT_RECURSIVE) + .help("recursively compare subdirectories (always on for directories)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_NEW_FILE) + .short('N') + .long(OPT_NEW_FILE) + .help("treat absent files as empty") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_COLOR) + .long(OPT_COLOR) + .value_name("WHEN") + .num_args(0..=1) + .require_equals(true) + .default_missing_value("auto") + .help("accepted for compatibility; output is never colorized"), + ) + .arg( + Arg::new(ARG_FILES) + .required(true) + .num_args(2) + .value_parser(clap::value_parser!(OsString)) + .value_hint(clap::ValueHint::AnyPath), + ) +} + +#[derive(Clone, Copy)] +struct Options { + context: usize, + brief: bool, + new_file: bool, +} + +/// A classified operand: what the name as typed refers to on disk after +/// resolution against the scope working directory. +enum Operand { + /// The context stdin (`-`). + Stdin, + /// A regular (or other non-directory) file at the resolved path. + File(PathBuf), + /// A directory at the resolved path. + Dir(PathBuf), + /// A missing file tolerated by `-N` and compared as empty. + Absent, +} + +fn diff_main(matches: &ArgMatches) -> Result { + let files: Vec<&OsString> = matches.get_many::(ARG_FILES).unwrap().collect(); + let opts = Options { + context: matches.get_one::(OPT_UNIFIED).copied().unwrap_or(3), + brief: matches.get_flag(OPT_BRIEF), + new_file: matches.get_flag(OPT_NEW_FILE), + }; + + let (mut name_a, mut name_b) = (PathBuf::from(files[0]), PathBuf::from(files[1])); + let mut op_a = classify(&name_a, opts.new_file)?; + let mut op_b = classify(&name_b, opts.new_file)?; + + // GNU: comparing a directory with a non-directory compares + // / with the other operand. + let a_is_dir = matches!(op_a, Operand::Dir(_)); + let b_is_dir = matches!(op_b, Operand::Dir(_)); + if a_is_dir != b_is_dir { + if matches!(op_a, Operand::Stdin) || matches!(op_b, Operand::Stdin) { + return Err("cannot compare '-' to a directory".to_string()); + } + if a_is_dir { + name_a = descend(&name_a, &name_b)?; + op_a = classify(&name_a, opts.new_file)?; + } else { + name_b = descend(&name_b, &name_a)?; + op_b = classify(&name_b, opts.new_file)?; + } + } + + let differed = if let (Operand::Dir(res_a), Operand::Dir(res_b)) = (&op_a, &op_b) { + diff_dirs(&name_a, res_a, &name_b, res_b, opts)? + } else { + let bytes_a = read_operand(&op_a, &name_a)?; + let bytes_b = read_operand(&op_b, &name_b)?; + diff_pair(&name_a, &bytes_a, &name_b, &bytes_b, opts, None)? + }; + Ok(i32::from(differed)) +} + +/// Replaces a directory operand with `/` for the GNU +/// dir-vs-file comparison form. +fn descend(dir: &Path, other: &Path) -> Result { + let base = other + .file_name() + .ok_or_else(|| format!("cannot compare {} to a directory", other.display()))?; + Ok(dir.join(base)) +} + +fn classify(name: &Path, new_file: bool) -> Result { + if name.as_os_str() == OsStr::new("-") { + return Ok(Operand::Stdin); + } + // Resolve the operand against the shell working directory; `name` is kept + // for display (GNU prints operands as typed). + let resolved = pi_uutils_ctx::resolve(name); + match fs::metadata(&resolved) { + Ok(meta) if meta.is_dir() => Ok(Operand::Dir(resolved)), + Ok(_) => Ok(Operand::File(resolved)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound && new_file => Ok(Operand::Absent), + Err(err) => Err(format!("{}: {}", name.display(), io_msg(&err))), + } +} + +fn read_operand(op: &Operand, name: &Path) -> Result, String> { + match op { + Operand::Stdin => { + let mut buf = Vec::new(); + pi_uutils_ctx::stdin() + .read_to_end(&mut buf) + .map_err(|err| format!("-: {}", io_msg(&err)))?; + Ok(buf) + }, + Operand::File(resolved) => { + fs::read(resolved).map_err(|err| format!("{}: {}", name.display(), io_msg(&err))) + }, + Operand::Dir(_) => unreachable!("directories are handled by diff_dirs"), + Operand::Absent => Ok(Vec::new()), + } +} + +/// Diffs one pair of already-read inputs, writing to the context stdout. +/// `prefix` is the `diff -r A/x B/x` line emitted before per-pair output in +/// directory mode. Returns whether the inputs differed. +fn diff_pair( + name_a: &Path, + bytes_a: &[u8], + name_b: &Path, + bytes_b: &[u8], + opts: Options, + prefix: Option<&str>, +) -> Result { + if bytes_a == bytes_b { + return Ok(false); + } + let mut out = pi_uutils_ctx::stdout(); + let (label_a, label_b) = (name_a.display().to_string(), name_b.display().to_string()); + if opts.brief { + writeln!(out, "Files {label_a} and {label_b} differ").map_err(|e| io_msg(&e))?; + return Ok(true); + } + if is_binary(bytes_a) || is_binary(bytes_b) { + writeln!(out, "Binary files {label_a} and {label_b} differ").map_err(|e| io_msg(&e))?; + return Ok(true); + } + if let Some(line) = prefix { + writeln!(out, "{line}").map_err(|e| io_msg(&e))?; + } + let old = String::from_utf8_lossy(bytes_a); + let new = String::from_utf8_lossy(bytes_b); + let diff = TextDiff::from_lines(old.as_ref(), new.as_ref()); + write!( + out, + "{}", + diff + .unified_diff() + .context_radius(opts.context) + .header(&label_a, &label_b) + ) + .map_err(|e| io_msg(&e))?; + Ok(true) +} + +/// Recursively compares two directories over the sorted union of their +/// entries, GNU `diff -r` style. Returns whether any difference was found. +fn diff_dirs( + name_a: &Path, + res_a: &Path, + name_b: &Path, + res_b: &Path, + opts: Options, +) -> Result { + let mut names: BTreeSet = BTreeSet::new(); + for (dir_name, dir_res) in [(name_a, res_a), (name_b, res_b)] { + let entries = fs::read_dir(dir_res) + .map_err(|err| format!("{}: {}", dir_name.display(), io_msg(&err)))?; + for entry in entries { + let entry = entry.map_err(|err| format!("{}: {}", dir_name.display(), io_msg(&err)))?; + names.insert(entry.file_name()); + } + } + + let mut differed = false; + for name in names { + if pi_uutils_ctx::is_cancelled() { + return Err("interrupted".to_string()); + } + let (child_name_a, child_res_a) = (name_a.join(&name), res_a.join(&name)); + let (child_name_b, child_res_b) = (name_b.join(&name), res_b.join(&name)); + let meta_a = fs::metadata(&child_res_a).ok(); + let meta_b = fs::metadata(&child_res_b).ok(); + match (meta_a.as_ref(), meta_b.as_ref()) { + (Some(ma), Some(mb)) if ma.is_dir() && mb.is_dir() => { + differed |= diff_dirs(&child_name_a, &child_res_a, &child_name_b, &child_res_b, opts)?; + }, + (Some(ma), Some(mb)) if ma.is_dir() != mb.is_dir() => { + let (dir, file) = if ma.is_dir() { + (&child_name_a, &child_name_b) + } else { + (&child_name_b, &child_name_a) + }; + writeln!( + pi_uutils_ctx::stdout(), + "File {} is a directory while file {} is a regular file", + dir.display(), + file.display() + ) + .map_err(|e| io_msg(&e))?; + differed = true; + }, + (Some(_), Some(_)) => { + let bytes_a = fs::read(&child_res_a) + .map_err(|err| format!("{}: {}", child_name_a.display(), io_msg(&err)))?; + let bytes_b = fs::read(&child_res_b) + .map_err(|err| format!("{}: {}", child_name_b.display(), io_msg(&err)))?; + let prefix = format!("diff -r {} {}", child_name_a.display(), child_name_b.display()); + differed |= + diff_pair(&child_name_a, &bytes_a, &child_name_b, &bytes_b, opts, Some(&prefix))?; + }, + (Some(meta), None) | (None, Some(meta)) => { + let in_a = meta_b.is_none(); + if opts.new_file && meta.is_file() { + // -N: compare the present file against an empty absent one. + let (present_name, present_res) = if in_a { + (&child_name_a, &child_res_a) + } else { + (&child_name_b, &child_res_b) + }; + let bytes = fs::read(present_res) + .map_err(|err| format!("{}: {}", present_name.display(), io_msg(&err)))?; + let prefix = + format!("diff -r {} {}", child_name_a.display(), child_name_b.display()); + let (ba, bb): (&[u8], &[u8]) = if in_a { (&bytes, &[]) } else { (&[], &bytes) }; + differed |= diff_pair(&child_name_a, ba, &child_name_b, bb, opts, Some(&prefix))?; + } else { + let present_dir = if in_a { name_a } else { name_b }; + writeln!( + pi_uutils_ctx::stdout(), + "Only in {}: {}", + present_dir.display(), + Path::new(&name).display() + ) + .map_err(|e| io_msg(&e))?; + differed = true; + } + }, + (None, None) => {}, + } + } + Ok(differed) +} + +/// NUL byte within the first 8 KiB marks the input as binary, matching the +/// heuristic GNU diff applies to decide between text and binary output. +fn is_binary(bytes: &[u8]) -> bool { + bytes.iter().take(8192).any(|&b| b == 0) +} + +/// Renders an I/O error without the Rust-specific ` (os error N)` suffix so +/// messages read like GNU diff's (`diff: x: No such file or directory`). +fn io_msg(err: &std::io::Error) -> String { + let msg = err.to_string(); + match msg.find(" (os error") { + Some(idx) => msg[..idx].to_string(), + None => msg, + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_with(cwd: PathBuf, stdin: &[u8], args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::Cursor::new(stdin.to_vec())), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("diff") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + run_with(cwd, b"", args) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + #[test] + fn identical_files_print_nothing_and_exit_zero() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "one\ntwo\n").unwrap(); + fs::write(root.join("b.txt"), "one\ntwo\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["a.txt", "b.txt"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + } + + /// Relative operands must resolve against the scope cwd (a tempdir), not + /// the process cwd — the pi-specific contract. + #[test] + fn differing_files_emit_unified_diff_with_typed_headers() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "one\ntwo\nthree\n").unwrap(); + fs::write(root.join("b.txt"), "one\nTWO\nthree\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["a.txt", "b.txt"]); + assert_eq!(code, 1); + assert_eq!(stderr, ""); + assert!(stdout.starts_with("--- a.txt\n+++ b.txt\n@@ "), "got: {stdout}"); + assert!(stdout.contains("\n-two\n"), "got: {stdout}"); + assert!(stdout.contains("\n+TWO\n"), "got: {stdout}"); + // Context lines around the change (default -U 3). + assert!(stdout.contains("\n one\n"), "got: {stdout}"); + assert!(stdout.contains("\n three\n"), "got: {stdout}"); + } + + #[test] + fn unified_zero_drops_context_lines() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "one\ntwo\nthree\n").unwrap(); + fs::write(root.join("b.txt"), "one\nTWO\nthree\n").unwrap(); + + let (code, stdout, _) = run_in(root, vec!["-U", "0", "a.txt", "b.txt"]); + assert_eq!(code, 1); + assert!(!stdout.contains("\n one\n"), "got: {stdout}"); + assert!(!stdout.contains("\n three\n"), "got: {stdout}"); + assert!(stdout.contains("\n-two\n"), "got: {stdout}"); + assert!(stdout.contains("\n+TWO\n"), "got: {stdout}"); + } + + #[test] + fn brief_reports_one_line_per_differing_pair() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "x\n").unwrap(); + fs::write(root.join("b.txt"), "y\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["-q", "a.txt", "b.txt"]); + assert_eq!(code, 1); + assert_eq!(stdout, "Files a.txt and b.txt differ\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn compat_flags_are_accepted_and_ignored() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "x\n").unwrap(); + fs::write(root.join("b.txt"), "y\n").unwrap(); + + let (code, stdout, stderr) = + run_in(root, vec!["-u", "-r", "--color=always", "a.txt", "b.txt"]); + assert_eq!(code, 1); + assert_eq!(stderr, ""); + // Plain unified output, no ANSI escapes. + assert!(stdout.starts_with("--- a.txt\n+++ b.txt\n"), "got: {stdout}"); + assert!(!stdout.contains('\u{1b}'), "got: {stdout}"); + } + + #[test] + fn binary_inputs_report_binary_difference() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.bin"), b"aa\x00bb").unwrap(); + fs::write(root.join("b.bin"), b"aa\x00cc").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["a.bin", "b.bin"]); + assert_eq!(code, 1); + assert_eq!(stdout, "Binary files a.bin and b.bin differ\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn missing_operand_file_is_trouble() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "x\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["a.txt", "nope.txt"]); + assert_eq!(code, 2); + assert_eq!(stdout, ""); + assert_eq!(stderr, "diff: nope.txt: No such file or directory\n"); + } + + #[test] + fn missing_second_operand_is_usage_error() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["only-one"]); + assert_eq!(code, 2); + assert_eq!(stdout, ""); + assert!(stderr.contains("required"), "got: {stderr}"); + } + + #[test] + fn new_file_treats_missing_operand_as_empty() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "one\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["-N", "nope.txt", "a.txt"]); + assert_eq!(code, 1); + assert_eq!(stderr, ""); + assert!(stdout.starts_with("--- nope.txt\n+++ a.txt\n"), "got: {stdout}"); + assert!(stdout.contains("\n+one\n"), "got: {stdout}"); + } + + #[test] + fn dash_reads_context_stdin() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a.txt"), "one\ntwo\n").unwrap(); + + let (code, stdout, stderr) = run_with(root.clone(), b"one\ntwo\n", vec!["a.txt", "-"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + + let (code, stdout, _) = run_with(root, b"one\nTWO\n", vec!["a.txt", "-"]); + assert_eq!(code, 1); + assert!(stdout.starts_with("--- a.txt\n+++ -\n"), "got: {stdout}"); + } + + #[test] + fn directories_diff_recursively_with_only_in_lines() { + let (_dir, root) = canonical_tempdir(); + let (a, b) = (root.join("a"), root.join("b")); + fs::create_dir_all(a.join("sub")).unwrap(); + fs::create_dir_all(b.join("sub")).unwrap(); + fs::write(a.join("common.txt"), "same\n").unwrap(); + fs::write(b.join("common.txt"), "same\n").unwrap(); + fs::write(a.join("only.txt"), "left\n").unwrap(); + fs::write(b.join("other.txt"), "right\n").unwrap(); + fs::write(a.join("sub/inner.txt"), "old\n").unwrap(); + fs::write(b.join("sub/inner.txt"), "new\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["a", "b"]); + assert_eq!(code, 1); + assert_eq!(stderr, ""); + assert!(stdout.contains("Only in a: only.txt\n"), "got: {stdout}"); + assert!(stdout.contains("Only in b: other.txt\n"), "got: {stdout}"); + assert!( + stdout.contains( + "diff -r a/sub/inner.txt b/sub/inner.txt\n--- a/sub/inner.txt\n+++ b/sub/inner.txt\n" + ), + "got: {stdout}" + ); + assert!(stdout.contains("\n-old\n"), "got: {stdout}"); + assert!(stdout.contains("\n+new\n"), "got: {stdout}"); + // Identical common.txt must not appear at all. + assert!(!stdout.contains("common.txt"), "got: {stdout}"); + } + + #[test] + fn identical_directories_exit_zero() { + let (_dir, root) = canonical_tempdir(); + let (a, b) = (root.join("a"), root.join("b")); + fs::create_dir_all(&a).unwrap(); + fs::create_dir_all(&b).unwrap(); + fs::write(a.join("f.txt"), "same\n").unwrap(); + fs::write(b.join("f.txt"), "same\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["-r", "a", "b"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("Compare files line by line")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-date/Cargo.toml b/crates/vendor/uu-date/Cargo.toml new file mode 100644 index 000000000..0d23dea93 --- /dev/null +++ b/crates/vendor/uu-date/Cargo.toml @@ -0,0 +1,36 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/date), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. The set-date +# capability (and its libc/windows-sys settime dependencies) is removed +# entirely, along with the fluent/icu localization stack. See src/date.rs for +# the patch markers (`pi-uutils:` comments). +[package] +name = "uu_date" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "date ~ (uutils) display the current time (vendored + patched for in-process embedding)" + +[lib] +path = "src/date.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +jiff = { version = "0.2", features = [ + "tzdb-bundle-platform", + "tzdb-zoneinfo", + "tzdb-concatenated", +] } +parse_datetime = "0.14" +regex = "1.11" +uucore = { version = "0.8.0", features = ["parser"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +# pi-uutils: rustix is kept only for `clock_getres` (--resolution); the +# `clock_settime` user is gone with the set-date capability. +[target.'cfg(unix)'.dependencies] +rustix = { version = "1.1", features = ["time"] } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-date/LICENSE b/crates/vendor/uu-date/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-date/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-date/src/date.rs b/crates/vendor/uu-date/src/date.rs new file mode 100644 index 000000000..e4db89445 --- /dev/null +++ b/crates/vendor/uu-date/src/date.rs @@ -0,0 +1,1336 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore strtime ; (format) DATEFILE MMDDhhmm ; (vars) datetime +// datetimes getres AWST ACST AEST foobarbaz + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. The set-date capability (`--set`, clock_settime / +// SetSystemTime) is removed entirely — a builtin must never mutate the host +// system clock — and `--set` now reports "setting the date is not supported by +// this builtin". The fluent/icu localization stack is dropped (`translate!` +// strings are literalized with the en-US locale text, the i18n-datetime +// feature is not vendored, and the locale.rs default-format probe — which +// calls the process-global setlocale(3) — is replaced by upstream's 24-hour +// fallback format). File operands (`--file`, `--reference`) resolve against +// the shell working directory via `pi_uutils_ctx::resolve` AT THE CALL SITE +// while the original operands are kept for display/error messages, stdio is +// routed through `pi_uutils_ctx`, and the entry point no longer calls +// `std::process::exit`. Time-zone handling stays process-global: jiff reads +// the host TZ environment variable and tzdb (same behavior as upstream). + +mod format_modifiers; + +use std::{ + borrow::Cow, + collections::HashMap, + ffi::OsString, + fs::File, + io::{BufRead, BufReader, BufWriter, Read, Write}, + path::PathBuf, + sync::LazyLock, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use jiff::{ + Timestamp, Zoned, + fmt::strtime::{self, BrokenDownTime, Config, PosixCustom}, + tz::{Offset, TimeZone, TimeZoneDatabase}, +}; +use pi_uutils_ctx::format_usage; +use uucore::{ + display::Quotable, + error::{FromIo, UResult, USimpleError}, + parser::shortcut_value_parser::ShortcutValueParser, +}; + +// Options +const DATE: &str = "date"; +const HOURS: &str = "hours"; +const MINUTES: &str = "minutes"; +const SECONDS: &str = "seconds"; +const NS: &str = "ns"; + +const OPT_DATE: &str = "date"; +const OPT_FORMAT: &str = "format"; +const OPT_FILE: &str = "file"; +const OPT_DEBUG: &str = "debug"; +const OPT_ISO_8601: &str = "iso-8601"; +const OPT_RESOLUTION: &str = "resolution"; +const OPT_RFC_EMAIL: &str = "rfc-email"; +const OPT_RFC_822: &str = "rfc-822"; +const OPT_RFC_2822: &str = "rfc-2822"; +const OPT_RFC_3339: &str = "rfc-3339"; +const OPT_SET: &str = "set"; +const OPT_REFERENCE: &str = "reference"; +const OPT_UNIVERSAL: &str = "universal"; +const OPT_UNIVERSAL_2: &str = "utc"; + +/// Settings for this program, parsed from the command line +// pi-uutils: the upstream `set_to` field is gone with the set-date capability. +struct Settings { + utc: bool, + format: Format, + date_source: DateSource, + debug: bool, +} + +/// Options for parsing dates +#[derive(Clone, Copy)] +struct DebugOptions { + /// Enable debug output + debug: bool, + /// Warn when midnight is used without explicit time specification + warn_midnight: bool, +} + +impl DebugOptions { + fn new(debug: bool, warn_midnight: bool) -> Self { + Self { debug, warn_midnight } + } +} + +/// Various ways of displaying the date +enum Format { + Iso8601(Iso8601Format), + Rfc5322, + Rfc3339(Rfc3339Format), + Resolution, + Custom(String), + Default, +} + +/// Various places that dates can come from +enum DateSource { + Now, + File(PathBuf), + FileMtime(PathBuf), + Stdin, + Human(String), + Resolution, +} + +enum Iso8601Format { + Date, + Hours, + Minutes, + Seconds, + Ns, +} + +impl From<&str> for Iso8601Format { + fn from(s: &str) -> Self { + match s { + HOURS => Self::Hours, + MINUTES => Self::Minutes, + SECONDS => Self::Seconds, + NS => Self::Ns, + DATE => Self::Date, + // Note: This is caught by clap via `possible_values` + _ => unreachable!(), + } + } +} + +enum Rfc3339Format { + Date, + Seconds, + Ns, +} + +impl From<&str> for Rfc3339Format { + fn from(s: &str) -> Self { + match s { + DATE => Self::Date, + SECONDS => Self::Seconds, + NS => Self::Ns, + // Should be caught by clap + _ => panic!("Invalid format: {s}"), + } + } +} + +/// Indicates whether parsing a military timezone causes the date to remain the +/// same, roll back to the previous day, or advance to the next day. +/// This can occur when applying a military timezone with an optional hour +/// offset crosses midnight in either direction. +#[derive(PartialEq, Debug)] +enum DayDelta { + /// The date does not change + Same, + /// The date rolls back to the previous day. + Previous, + /// The date advances to the next day. + Next, +} + +/// Escape invalid UTF-8 bytes in GNU-compatible octal notation. +/// +/// Converts bytes to a string with printable ASCII characters preserved +/// and non-printable/invalid UTF-8 bytes escaped as `\NNN` octal sequences. +/// +/// This matches GNU date's behavior for invalid input. +/// +/// # Arguments +/// * `bytes` - The byte sequence to escape +/// +/// # Returns +/// A string with invalid bytes escaped in octal notation +/// +/// # Example +/// ```ignore +/// let invalid = b"\xb0"; +/// assert_eq!(escape_invalid_bytes(invalid), "\\260"); +/// ``` +fn escape_invalid_bytes(bytes: &[u8]) -> String { + let escaped = bytes + .iter() + .flat_map(|&b| { + // Preserve printable ASCII except backslash + if (0x20..0x7f).contains(&b) && b != b'\\' { + vec![b] + } else { + // Escape as octal: \NNN + format!("\\{b:03o}").into_bytes() + } + }) + .collect::>(); + String::from_utf8_lossy(&escaped).into_owned() +} + +/// Strip parenthesized comments from a date string. +/// +/// GNU date removes balanced parentheses and their content, treating them as +/// comments. If parentheses are unbalanced, everything from the unmatched '(' +/// onwards is ignored. +/// +/// Examples: +/// - "2026(comment)-01-05" -> "2026-01-05" +/// - "1(ignore comment to eol" -> "1" +/// - "(" -> "" +/// - "((foo)2026-01-05)" -> "" +fn strip_parenthesized_comments(input: &str) -> Cow<'_, str> { + if !input.contains('(') { + return Cow::Borrowed(input); + } + + let mut result = String::with_capacity(input.len()); + let mut depth = 0; + + for c in input.chars() { + match c { + '(' => { + depth += 1; + }, + ')' if depth > 0 => { + depth -= 1; + }, + _ if depth == 0 => { + result.push(c); + }, + _ => {}, + } + } + + Cow::Owned(result) +} + +/// Parse military timezone with optional hour offset. +/// Pattern: single letter (a-z except j) optionally followed by 1-2 digits. +/// Returns Some(total_hours_in_utc) or None if pattern doesn't match. +/// +/// Military timezone mappings: +/// - A-I: UTC+1 to UTC+9 (J is skipped for local time) +/// - K-M: UTC+10 to UTC+12 +/// - N-Y: UTC-1 to UTC-12 +/// - Z: UTC+0 +/// +/// The hour offset from digits is added to the base military timezone offset. +/// Examples: "m" -> 12 (noon UTC), "m9" -> 21 (9pm UTC), "a5" -> 4 (4am UTC +/// next day) +fn parse_military_timezone_with_offset(s: &str) -> Option<(i32, DayDelta)> { + if s.is_empty() || s.len() > 3 { + return None; + } + + let mut chars = s.chars(); + let letter = chars.next()?.to_ascii_lowercase(); + + // Check if first character is a letter (a-z, except j which is handled + // separately) + if !letter.is_ascii_lowercase() || letter == 'j' { + return None; + } + + // Parse optional digits (1-2 digits for hour offset) + let additional_hours: i32 = if let Some(rest) = chars.as_str().chars().next() { + if !rest.is_ascii_digit() { + return None; + } + chars.as_str().parse().ok()? + } else { + 0 + }; + + // Map military timezone letter to UTC offset + let tz_offset = match letter { + 'a'..='i' => (letter as i32 - 'a' as i32) + 1, // A=+1, B=+2, ..., I=+9 + 'k'..='m' => (letter as i32 - 'k' as i32) + 10, // K=+10, L=+11, M=+12 + 'n'..='y' => -((letter as i32 - 'n' as i32) + 1), // N=-1, O=-2, ..., Y=-12 + 'z' => 0, // Z=+0 + _ => return None, + }; + + let day_delta = match additional_hours - tz_offset { + h if h < 0 => DayDelta::Previous, + h if h >= 24 => DayDelta::Next, + _ => DayDelta::Same, + }; + + // Calculate total hours: midnight (0) + tz_offset + additional_hours + // Midnight in timezone X converted to UTC + let hours_from_midnight = (0 - tz_offset + additional_hours).rem_euclid(24); + + Some((hours_from_midnight, day_delta)) +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match date_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "date: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +#[allow(clippy::cognitive_complexity)] +fn date_main(matches: &ArgMatches) -> UResult<()> { + // pi-uutils: the set-date capability is removed — a shell builtin must + // never mutate the host system clock, so `--set` fails up front instead of + // parsing the operand and calling clock_settime(2)/SetSystemTime. + if matches.get_one::(OPT_SET).is_some() { + return Err(USimpleError::new(1, "setting the date is not supported by this builtin")); + } + + let date_source = if let Some(date_os) = matches.get_one::(OPT_DATE) { + // Convert OsString to String, handling invalid UTF-8 with GNU-compatible error + let date = date_os.to_str().ok_or_else(|| { + let bytes = date_os.as_encoded_bytes(); + let escaped_str = escape_invalid_bytes(bytes); + USimpleError::new(1, format!("invalid date '{escaped_str}'")) + })?; + DateSource::Human(date.into()) + } else if let Some(file) = matches.get_one::(OPT_FILE) { + match file.as_ref() { + "-" => DateSource::Stdin, + _ => DateSource::File(file.into()), + } + } else if let Some(file) = matches.get_one::(OPT_REFERENCE) { + DateSource::FileMtime(file.into()) + } else if matches.get_flag(OPT_RESOLUTION) { + DateSource::Resolution + } else { + DateSource::Now + }; + + // Check for extra operands (multiple positional arguments) + if let Some(formats) = matches.get_many::(OPT_FORMAT) { + let format_args: Vec<&String> = formats.collect(); + if format_args.len() > 1 { + return Err(USimpleError::new(1, format!("extra operand '{}'", format_args[1]))); + } + } + + let format = if let Some(form) = matches.get_one::(OPT_FORMAT) { + if !form.starts_with('+') { + // if an optional Format String was found but the user has not provided an input + // date GNU prints an invalid date Error + if !matches!(date_source, DateSource::Human(_)) { + return Err(USimpleError::new(1, format!("invalid date '{form}'"))); + } + // If the user did provide an input date with the --date flag and the Format + // String is not starting with '+' GNU prints the missing '+' error message + return Err(USimpleError::new( + 1, + format!( + "the argument {form} lacks a leading '+';\nwhen using an option to specify \ + date(s), any non-option\nargument must be a format string beginning with '+'" + ), + )); + } + let form = form[1..].to_string(); + Format::Custom(form) + } else if let Some(fmt) = matches + .get_many::(OPT_ISO_8601) + .map(|mut iter| iter.next().unwrap_or(&DATE.to_string()).as_str().into()) + { + Format::Iso8601(fmt) + } else if matches.get_flag(OPT_RFC_EMAIL) { + Format::Rfc5322 + } else if let Some(fmt) = matches + .get_one::(OPT_RFC_3339) + .map(|s| s.as_str().into()) + { + Format::Rfc3339(fmt) + } else if matches.get_flag(OPT_RESOLUTION) { + Format::Resolution + } else { + Format::Default + }; + + let utc = matches.get_flag(OPT_UNIVERSAL); + let debug_mode = matches.get_flag(OPT_DEBUG); + + // Get the current time, either in the local time zone or UTC. + // pi-uutils: time-zone handling stays process-global — jiff reads the host + // TZ environment variable and system tzdb here, as upstream does. + let now = if utc { + Timestamp::now().to_zoned(TimeZone::UTC) + } else { + Zoned::now() + }; + + let settings = Settings { utc, format, date_source, debug: debug_mode }; + + // Iterate over all dates - whether it's a single date or a file. + let dates: Box> = match &settings.date_source { + DateSource::Human(input) => { + // GNU compatibility (Comments in parentheses) + let input = strip_parenthesized_comments(input); + let input = input.trim(); + + // GNU compatibility (Empty string): + // An empty string (or whitespace-only) should be treated as midnight today. + let is_empty_or_whitespace = input.is_empty(); + + // GNU compatibility (Military timezone 'J'): + // 'J' is reserved for local time in military timezones. + // GNU date accepts it and treats it as midnight today (00:00:00). + let is_military_j = input.eq_ignore_ascii_case("j"); + + // GNU compatibility (Military timezone with optional hour offset): + // Single letter (a-z except j) optionally followed by 1-2 digits. + // Letter represents midnight in that military timezone (UTC offset). + // Digits represent additional hours to add. + // Examples: "m" -> noon UTC (12:00); "m9" -> 21:00 UTC; "a5" -> 04:00 UTC + let military_tz_with_offset = parse_military_timezone_with_offset(input); + + // GNU compatibility (Pure numbers in date strings): + // - Manual: https://www.gnu.org/software/coreutils/manual/html_node/Pure-numbers-in-date-strings.html + // - Semantics: a pure decimal number denotes today's time-of-day (HH or HHMM). + // Examples: "0"/"00" => 00:00 today; "7"/"07" => 07:00 today; "0700" => 07:00 + // today. + // For all other forms, fall back to the general parser. + let is_pure_digits = + !input.is_empty() && input.len() <= 4 && input.chars().all(|c| c.is_ascii_digit()); + + let date = if is_empty_or_whitespace || is_military_j { + // Treat empty string or 'J' as midnight today (00:00:00) in local time + let date_part = + strtime::format("%F", &now).unwrap_or_else(|_| String::from("1970-01-01")); + let offset = if settings.utc { + String::from("+00:00") + } else { + strtime::format("%:z", &now).unwrap_or_default() + }; + let composed = if offset.is_empty() { + format!("{date_part} 00:00") + } else { + format!("{date_part} 00:00 {offset}") + }; + if settings.debug { + let _ = writeln!( + pi_uutils_ctx::stderr(), + "date: warning: using midnight as starting time: 00:00:00" + ); + } + parse_date(composed, &now, DebugOptions::new(settings.debug, false)) + } else if let Some((total_hours, day_delta)) = military_tz_with_offset { + // Military timezone with optional hour offset + // Convert to UTC time: midnight + military_tz_offset + additional_hours + + // When calculating a military timezone with an optional hour offset, midnight + // may be crossed in either direction. `day_delta` indicates whether the + // date remains the same, moves to the previous day, or advances to the next + // day. Changing day can result in error, this closure will help handle + // these errors gracefully. + let format_date_with_epoch_fallback = |date: Result| -> String { + date + .and_then(|d| strtime::format("%F", &d)) + .unwrap_or_else(|_| String::from("1970-01-01")) + }; + let date_part = match day_delta { + DayDelta::Same => format_date_with_epoch_fallback(Ok(now.clone())), + DayDelta::Next => format_date_with_epoch_fallback(now.tomorrow()), + DayDelta::Previous => format_date_with_epoch_fallback(now.yesterday()), + }; + let composed = format!("{date_part} {total_hours:02}:00:00 +00:00"); + parse_date(composed, &now, DebugOptions::new(settings.debug, false)) + } else if is_pure_digits { + // Derive HH and MM from the input + let (hh_opt, mm_opt) = if input.len() <= 2 { + (input.parse::().ok(), Some(0u32)) + } else { + let (h, m) = input.split_at(input.len() - 2); + (h.parse::().ok(), m.parse::().ok()) + }; + + if let (Some(hh), Some(mm)) = (hh_opt, mm_opt) { + // Compose a concrete datetime string for today with zone offset. + // Use the already-determined 'now' and settings.utc to select offset. + let date_part = + strtime::format("%F", &now).unwrap_or_else(|_| String::from("1970-01-01")); + // If -u, force +00:00; otherwise use the local offset of 'now'. + let offset = if settings.utc { + String::from("+00:00") + } else { + strtime::format("%:z", &now).unwrap_or_default() + }; + let composed = if offset.is_empty() { + format!("{date_part} {hh:02}:{mm:02}") + } else { + format!("{date_part} {hh:02}:{mm:02} {offset}") + }; + parse_date(composed, &now, DebugOptions::new(settings.debug, false)) + } else { + // Fallback on parse failure of digits + parse_date(input, &now, DebugOptions::new(settings.debug, true)) + } + } else { + parse_date(input, &now, DebugOptions::new(settings.debug, true)) + }; + + let iter = std::iter::once(date); + Box::new(iter) + }, + // pi-uutils: `-f -` reads the context stdin, not the process stdin. + DateSource::Stdin => parse_dates_from_reader( + pi_uutils_ctx::stdin(), + &now, + DebugOptions::new(settings.debug, true), + ), + DateSource::File(path) => { + // pi-uutils: resolve the DATEFILE operand against the shell working + // directory; `path` is kept for display. + let resolved = pi_uutils_ctx::resolve(path); + if resolved.is_dir() { + return Err(USimpleError::new( + 2, + format!("expected file, got directory {}", path.quote()), + )); + } + let file = + File::open(&resolved).map_err_context(|| path.as_os_str().maybe_quote().to_string())?; + parse_dates_from_reader(file, &now, DebugOptions::new(settings.debug, true)) + }, + DateSource::FileMtime(path) => { + // pi-uutils: resolve the --reference FILE against the shell working + // directory; `path` is kept for display. + let metadata = std::fs::metadata(pi_uutils_ctx::resolve(path)) + .map_err_context(|| path.as_os_str().maybe_quote().to_string())?; + let mtime = metadata.modified()?; + let ts = Timestamp::try_from(mtime) + .map_err(|_| USimpleError::new(1, "cannot set date".to_string()))?; + // pi-uutils: process-global TZ lookup, as upstream. + let date = ts.to_zoned(TimeZone::try_system().unwrap_or(TimeZone::UTC)); + let iter = std::iter::once(Ok(date)); + Box::new(iter) + }, + DateSource::Resolution => { + let resolution = get_clock_resolution(); + // pi-uutils: process-global TZ lookup, as upstream. + let date = resolution.to_zoned(TimeZone::system()); + let iter = std::iter::once(Ok(date)); + Box::new(iter) + }, + DateSource::Now => { + let iter = std::iter::once(Ok(now.clone())); + Box::new(iter) + }, + }; + + let format_string = make_format_string(&settings); + // pi-uutils: buffered context stdout instead of the process stdout. + let mut stdout = BufWriter::new(pi_uutils_ctx::stdout()); + + // Format all the dates + let config = Config::new().custom(PosixCustom::new()).lenient(true); + for date in dates { + // pi-uutils: a DATEFILE/stdin stream can be arbitrarily long; observe + // host cancellation between lines. + if pi_uutils_ctx::is_cancelled() { + break; + } + match date { + Ok(date) => { + let date = if settings.utc { + date.with_time_zone(TimeZone::UTC) + } else { + date + }; + match format_date(&date, format_string, &config) { + Ok(s) => writeln!(stdout, "{s}") + .map_err(|e| USimpleError::new(1, format!("write error: {e}")))?, + Err(e) => { + let _ = stdout.flush(); + return Err(USimpleError::new( + 1, + format!("invalid format '{format_string}' ({e})"), + )); + }, + } + }, + Err((input, _err)) => { + let _ = stdout.flush(); + // pi-uutils: upstream `show!` — report the bad line to the + // context stderr, record the failure exit code, and keep + // processing the remaining lines. + let _ = writeln!(pi_uutils_ctx::stderr(), "date: invalid date '{input}'"); + pi_uutils_ctx::set_exit_code(1); + }, + } + } + + stdout + .flush() + .map_err(|e| USimpleError::new(1, format!("write error: {e}")))?; + Ok(()) +} + +pub fn uu_app() -> Command { + Command::new("date") + .version(uucore::crate_version!()) + .about("Print or set the system date and time") + // pi-uutils: the localized usage blob's FORMAT reference table moved to + // `after_help` below; the usage proper is just the two command lines. + .override_usage(format_usage( + "date [OPTION]... [+FORMAT]...\ndate [OPTION]... [MMDDhhmm[[CC]YY][.ss]]", + )) + .after_help(FORMAT_HELP) + .infer_long_args(true) + .arg( + Arg::new(OPT_DATE) + .short('d') + .long(OPT_DATE) + .value_name("STRING") + .allow_hyphen_values(true) + .overrides_with(OPT_DATE) + .value_parser(clap::value_parser!(OsString)) + .help("display time described by STRING, not 'now'"), + ) + .arg( + Arg::new(OPT_FILE) + .short('f') + .long(OPT_FILE) + .value_name("DATEFILE") + .value_hint(clap::ValueHint::FilePath) + .conflicts_with(OPT_DATE) + .help("like --date; once for each line of DATEFILE"), + ) + .arg( + Arg::new(OPT_ISO_8601) + .short('I') + .long(OPT_ISO_8601) + .value_name("FMT") + .value_parser(ShortcutValueParser::new([DATE, HOURS, MINUTES, SECONDS, NS])) + .num_args(0..=1) + .default_missing_value(OPT_DATE) + .help( + "output date/time in ISO 8601 format.\nFMT='date' for date only (the \ + default),\n'hours', 'minutes', 'seconds', or 'ns'\nfor date and time to the \ + indicated precision.\nExample: 2006-08-14T02:34:56-06:00", + ), + ) + .arg( + Arg::new(OPT_RESOLUTION) + .long(OPT_RESOLUTION) + .conflicts_with_all([OPT_DATE, OPT_FILE]) + .overrides_with(OPT_RESOLUTION) + .help("output the available resolution of timestamps\nExample: 0.000000001") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_RFC_EMAIL) + .short('R') + .long(OPT_RFC_EMAIL) + .alias(OPT_RFC_2822) + .alias(OPT_RFC_822) + .overrides_with(OPT_RFC_EMAIL) + .help( + "output date and time in RFC 5322 format.\nExample: Mon, 14 Aug 2006 02:34:56 -0600", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_RFC_3339) + .long(OPT_RFC_3339) + .value_name("FMT") + .value_parser(ShortcutValueParser::new([DATE, SECONDS, NS])) + .help( + "output date/time in RFC 3339 format.\nFMT='date', 'seconds', or 'ns'\nfor date \ + and time to the indicated precision.\nExample: 2006-08-14 02:34:56-06:00", + ), + ) + .arg( + Arg::new(OPT_DEBUG) + .long(OPT_DEBUG) + .help("annotate the parsed date, and warn about questionable usage to stderr") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_REFERENCE) + .short('r') + .long(OPT_REFERENCE) + .value_name("FILE") + .value_hint(clap::ValueHint::AnyPath) + .conflicts_with_all([OPT_DATE, OPT_FILE, OPT_RESOLUTION]) + .help("display the last modification time of FILE"), + ) + .arg( + Arg::new(OPT_SET) + .short('s') + .long(OPT_SET) + .value_name("STRING") + .allow_hyphen_values(true) + // pi-uutils: the set-date capability is removed; the option is + // still parsed so it fails with a clear message instead of a + // clap "unexpected argument" error. + .help("set time described by STRING (not supported by this builtin)"), + ) + .arg( + Arg::new(OPT_UNIVERSAL) + .short('u') + .long(OPT_UNIVERSAL) + .visible_alias(OPT_UNIVERSAL_2) + .alias("uct") + .overrides_with(OPT_UNIVERSAL) + .help("print or set Coordinated Universal Time (UTC)") + .action(ArgAction::SetTrue), + ) + .arg(Arg::new(OPT_FORMAT).num_args(0..)) +} + +// pi-uutils: literalized en-US FORMAT reference from the `date-usage` locale +// blob, rendered as plain text for clap's after_help. +const FORMAT_HELP: &str = "\ +FORMAT controls the output. Interpreted sequences are: + %% a literal % + %a locale's abbreviated weekday name (e.g., Sun) + %A locale's full weekday name (e.g., Sunday) + %b locale's abbreviated month name (e.g., Jan) + %B locale's full month name (e.g., January) + %c locale's date and time (e.g., Thu Mar 3 23:05:25 2005) + %C century; like %Y, except omit last two digits (e.g., 20) + %d day of month (e.g., 01) + %D date; same as %m/%d/%y + %e day of month, space padded; same as %_d + %F full date; same as %Y-%m-%d + %g last two digits of year of ISO week number (see %G) + %G year of ISO week number (see %V); normally useful only with %V + %h same as %b + %H hour (00..23) + %I hour (01..12) + %j day of year (001..366) + %k hour, space padded ( 0..23); same as %_H + %l hour, space padded ( 1..12); same as %_I + %m month (01..12) + %M minute (00..59) + %n a newline + %N nanoseconds (000000000..999999999) + %p locale's equivalent of either AM or PM; blank if not known + %P like %p, but lower case + %q quarter of year (1..4) + %r locale's 12-hour clock time (e.g., 11:11:04 PM) + %R 24-hour hour and minute; same as %H:%M + %s seconds since 1970-01-01 00:00:00 UTC + %S second (00..60) + %t a tab + %T time; same as %H:%M:%S + %u day of week (1..7); 1 is Monday + %U week number of year, with Sunday as first day of week (00..53) + %V ISO week number, with Monday as first day of week (01..53) + %w day of week (0..6); 0 is Sunday + %W week number of year, with Monday as first day of week (00..53) + %x locale's date representation (e.g., 03/03/2005) + %X locale's time representation (e.g., 23:30:30) + %y last two digits of year (00..99) + %Y year + %z +hhmm numeric time zone (e.g., -0400) + %:z +hh:mm numeric time zone (e.g., -04:00) + %::z +hh:mm:ss numeric time zone (e.g., -04:00:00) + %:::z numeric time zone with : to necessary precision (e.g., -04, +05:30) + %Z alphabetic time zone abbreviation (e.g., EDT) + +By default, date pads numeric fields with zeroes. +The following optional flags may follow '%': + - (hyphen) do not pad the field + _ (underscore) pad with spaces + 0 (zero) pad with zeros + ^ use upper case if possible + # use opposite case if possible +After any flags comes an optional field width, as a decimal number; +then an optional modifier, which is either + E to use the locale's alternate representations if available, or + O to use the locale's alternate numeric symbols if available. + +Examples: + Convert seconds since the epoch (1970-01-01 UTC) to a date + date --date='@2147483647' + Show the time on the west coast of the US (use tzselect(1) to find TZ) + TZ='America/Los_Angeles' date"; + +/// pi-uutils: upstream's `format_date_with_locale_aware_months` minus the +/// optional icu locale-aware month/day name substitution (the i18n-datetime +/// feature is not vendored, so no localization ever applies). +fn format_date( + date: &Zoned, + format_string: &str, + config: &Config, +) -> Result { + // Check if format string has GNU modifiers (width/flags) and format if present + if let Some(result) = + format_modifiers::format_with_modifiers_if_present(date, format_string, config) + { + return result.map_err(|e| e.to_string()); + } + + let broken_down = BrokenDownTime::from(date); + broken_down + .to_string_with_config(config, format_string) + .map_err(|e| e.to_string()) +} + +/// Return the appropriate format string for the given settings. +fn make_format_string(settings: &Settings) -> &str { + match &settings.format { + Format::Iso8601(fmt) => match fmt { + Iso8601Format::Date => "%F", + Iso8601Format::Hours => "%FT%H%:z", + Iso8601Format::Minutes => "%FT%H:%M%:z", + Iso8601Format::Seconds => "%FT%T%:z", + Iso8601Format::Ns => "%FT%T,%N%:z", + }, + Format::Rfc5322 => "%a, %d %h %Y %T %z", + Format::Rfc3339(fmt) => match fmt { + Rfc3339Format::Date => "%F", + Rfc3339Format::Seconds => "%F %T%:z", + Rfc3339Format::Ns => "%F %T.%N%:z", + }, + Format::Resolution => "%s.%N", + Format::Custom(fmt) => fmt, + // pi-uutils: upstream derives the default format from the process + // locale via setlocale(3)/nl_langinfo(3) (src/uu/date/src/locale.rs). + // setlocale mutates process-global state, which a builtin must not do, + // so upstream's 24-hour fallback format is used unconditionally. + Format::Default => "%a %b %e %X %Z %Y", + } +} + +/// Timezone abbreviations with known fixed UTC offsets. +/// Checked first because the abbreviation encodes the exact offset +/// (e.g., EDT always means UTC-4, even in winter when New York observes EST). +/// Offset is in seconds to support half-hour zones like IST (UTC+5:30). +/// All other timezones (JST, CET, etc.) are dynamically resolved from IANA +/// database. +/* spell-checker: disable */ +static FIXED_OFFSET_ABBREVIATIONS: &[(&str, i32)] = &[ + ("UTC", 0), + ("GMT", 0), + ("MEST", 7200), // UTC+2 Middle European Summer Time + // US timezones (GNU compatible) + ("PST", -28800), // UTC-8 + ("PDT", -25200), // UTC-7 + ("MST", -25200), // UTC-7 + ("MDT", -21600), // UTC-6 + ("CST", -21600), // UTC-6 (Ambiguous: US Central, not China/Cuba) + ("CDT", -18000), // UTC-5 + ("EST", -18000), // UTC-5 + ("EDT", -14400), // UTC-4 + // Indian Standard Time (Ambiguous: India vs Israel vs Ireland) + ("IST", 19800), // UTC+5:30 + // Australian timezones + ("AWST", 28800), // UTC+8 + ("ACST", 34200), // UTC+9:30 + ("ACDT", 37800), // UTC+10:30 + ("AEST", 36000), // UTC+10 + ("AEDT", 39600), // UTC+11 + // German timezones + ("MEZ", 3600), // UTC+1 + ("MESZ", 7200), // UTC+2 + // Asian timezones + ("KST", 32400), // UTC+9 Korean Standard Time +]; +/* spell-checker: enable */ + +/// Lazy-loaded timezone abbreviation lookup map built from IANA database. +// pi-uutils: `LazyLock` instead of upstream's `OnceLock` + `get_or_init`. +static TZ_ABBREV_CACHE: LazyLock> = LazyLock::new(build_tz_abbrev_map); + +/// Build timezone abbreviation lookup map from IANA database. +/// This is a fallback for abbreviations not covered by +/// FIXED_OFFSET_ABBREVIATIONS. +fn build_tz_abbrev_map() -> HashMap { + let mut map = HashMap::new(); + + let tzdb = TimeZoneDatabase::from_env(); // spell-checker:disable-line + // spell-checker:disable-next-line + for tz_name in tzdb.available() { + let tz_str = tz_name.as_str(); + // Use last component as potential abbreviation + // e.g., "Pacific/Fiji" could map to "FIJI" + if let Some(last_part) = tz_str.split('/').next_back() { + let potential_abbrev = last_part.to_uppercase(); + // Only add if it looks like an abbreviation (2-5 uppercase chars) + if potential_abbrev.len() >= 2 + && potential_abbrev.len() <= 5 + && potential_abbrev.chars().all(|c| c.is_ascii_uppercase()) + { + map.entry(potential_abbrev) + .or_insert_with(|| tz_str.to_string()); + } + } + } + + map +} + +/// Get IANA timezone name for a given abbreviation. +/// Uses lazy-loaded cache with preferred mappings for disambiguation. +fn tz_abbrev_to_iana(abbrev: &str) -> Option<&str> { + TZ_ABBREV_CACHE.get(abbrev).map(String::as_str) +} + +/// Attempts to parse a date string that contains a timezone abbreviation (e.g. +/// "EST"). +/// +/// If an abbreviation is found and the date is parsable, returns `Some(Zoned)`. +/// Returns `None` if no abbreviation is detected or if parsing fails, +/// indicating that standard parsing should be attempted. +fn try_parse_with_abbreviation>(date_str: S, now: &Zoned) -> Option { + let s = date_str.as_ref(); + + // Look for timezone abbreviation at the end of the string + // Pattern: ends with uppercase letters (2-5 chars) + if let Some(last_word) = s.split_whitespace().last() { + // Check if it's a potential timezone abbreviation (all uppercase, 2-5 chars) + if last_word.len() >= 2 + && last_word.len() <= 5 + && last_word.chars().all(|c| c.is_ascii_uppercase()) + { + let tz = if let Some(&(_, offset_secs)) = FIXED_OFFSET_ABBREVIATIONS + .iter() + .find(|(abbr, _)| *abbr == last_word) + { + Offset::from_seconds(offset_secs).ok().map(TimeZone::fixed) + } else { + tz_abbrev_to_iana(last_word).and_then(|name| TimeZone::get(name).ok()) + }; + + if let Some(tz) = tz { + let date_part = s.trim_end_matches(last_word).trim(); + // Parse in the target timezone so "10:30 EDT" means 10:30 in EDT + if let Ok(parsed) = parse_datetime::parse_datetime_at_date(now.clone(), date_part) { + let dt = parsed.datetime(); + if let Ok(zoned) = dt.to_zoned(tz) { + return Some(zoned); + } + } + } + } + } + + // No abbreviation found or couldn't resolve, return original + None +} + +/// Helper function to parse dates from a line-based reader (stdin or file) +/// +/// Takes any `Read` source, reads it line by line, and parses each line as a +/// date. Returns a boxed iterator over the parse results. +fn parse_dates_from_reader( + reader: R, + now: &Zoned, + dbg_opts: DebugOptions, +) -> Box> + '_> { + let lines = BufReader::new(reader).lines(); + Box::new( + lines + .map_while(Result::ok) + .map(move |s| parse_date(s, now, dbg_opts)), + ) +} + +/// Parse a `String` into a `DateTime`. +/// If it fails, return a tuple of the `String` along with its `ParseError`. +fn parse_date + Clone>( + s: S, + now: &Zoned, + dbg_opts: DebugOptions, +) -> Result { + let input_str = s.as_ref(); + + if dbg_opts.debug { + let _ = writeln!(pi_uutils_ctx::stderr(), "date: input string: {input_str}"); + } + + // First, try to parse any timezone abbreviations + if let Some(zoned) = try_parse_with_abbreviation(input_str, now) { + if dbg_opts.debug { + // pi-uutils: context stderr instead of `stderr().lock()`. + let mut err = pi_uutils_ctx::stderr(); + let _ = writeln!( + err, + "date: parsed date part: (Y-M-D) {}", + strtime::format("%Y-%m-%d", &zoned).unwrap_or_default() + ); + let _ = writeln!( + err, + "date: parsed time part: {}", + strtime::format("%H:%M:%S", &zoned).unwrap_or_default() + ); + let tz_display = zoned.time_zone().iana_name().unwrap_or("system default"); + let _ = writeln!(err, "date: input timezone: {tz_display}"); + } + return Ok(zoned); + } + + match parse_datetime::parse_datetime_at_date(now.clone(), input_str) { + // Convert to system timezone for display + // (parse_datetime returns Zoned in the input's timezone) + Ok(date) => { + let result = date.timestamp().to_zoned(now.time_zone().clone()); + if dbg_opts.debug { + // Show final parsed date and time + // pi-uutils: context stderr instead of `stderr().lock()`. + let mut err = pi_uutils_ctx::stderr(); + let _ = writeln!( + err, + "date: parsed date part: (Y-M-D) {}", + strtime::format("%Y-%m-%d", &result).unwrap_or_default() + ); + let _ = writeln!( + err, + "date: parsed time part: {}", + strtime::format("%H:%M:%S", &result).unwrap_or_default() + ); + + // Show timezone information + let _ = writeln!(err, "date: input timezone: system default"); + + // Check if time component was specified, if not warn about midnight usage + // Only warn for date-only inputs (no time specified), but not for epoch formats + // (@N) or inputs that explicitly specify a time (containing ':') + if dbg_opts.warn_midnight && !input_str.contains(':') && !input_str.contains('@') { + // Input likely didn't specify a time, so midnight was assumed + let time_str = strtime::format("%H:%M:%S", &result).unwrap_or_default(); + if time_str == "00:00:00" { + let _ = writeln!(err, "date: warning: using midnight as starting time: 00:00:00"); + } + } + } + Ok(result) + }, + Err(e) => Err((input_str.into(), e)), + } +} + +#[cfg(not(any(unix, windows)))] +fn get_clock_resolution() -> Timestamp { + unimplemented!("getting clock resolution not implemented (unsupported target)"); +} + +#[cfg(all(unix, not(target_os = "redox")))] +/// Returns the resolution of the system’s realtime clock. +/// +/// # Panics +/// +/// Panics if `clock_getres` fails. On a POSIX-compliant system this should not +/// occur, as `CLOCK_REALTIME` is required to be supported. +/// Failure would indicate a non-conforming or otherwise broken implementation. +fn get_clock_resolution() -> Timestamp { + use rustix::time::{ClockId, clock_getres}; + + let timespec = clock_getres(ClockId::Realtime); + + #[allow(clippy::unnecessary_cast, reason = "needed for 32 bit target")] + Timestamp::constant(timespec.tv_sec as _, timespec.tv_nsec as _) +} + +#[cfg(all(unix, target_os = "redox"))] +fn get_clock_resolution() -> Timestamp { + // Redox OS does not support the posix clock_getres function, however + // internally it uses a resolution of 1ns to represent timestamps. + // https://gitlab.redox-os.org/redox-os/kernel/-/blob/master/src/time.rs + Timestamp::constant(0, 1) +} + +#[cfg(windows)] +fn get_clock_resolution() -> Timestamp { + // Windows does not expose a system call for getting the resolution of the + // clock, however the FILETIME struct returned by GetSystemTimeAsFileTime, + // and GetSystemTimePreciseAsFileTime has a resolution of 100ns. + // https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-filetime + Timestamp::constant(0, 100) +} + +// pi-uutils: upstream's `convert_for_set` and the `set_system_datetime` +// variants (clock_settime / SetSystemTime) are removed with the set-date +// capability. + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("date") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = std::fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + // --- upstream unit tests (0.8.0) --- + + #[test] + fn test_parse_military_timezone_with_offset() { + // Valid cases: letter only, letter + digit, uppercase + assert_eq!(parse_military_timezone_with_offset("m"), Some((12, DayDelta::Previous))); // UTC+12 -> 12:00 UTC + assert_eq!(parse_military_timezone_with_offset("m9"), Some((21, DayDelta::Previous))); // 12 + 9 = 21 + assert_eq!(parse_military_timezone_with_offset("a5"), Some((4, DayDelta::Same))); // 23 + 5 = 28 % 24 = 4 + assert_eq!(parse_military_timezone_with_offset("z"), Some((0, DayDelta::Same))); // UTC+0 -> 00:00 UTC + assert_eq!(parse_military_timezone_with_offset("M9"), Some((21, DayDelta::Previous))); // Uppercase works + + // Invalid cases: 'j' reserved, empty, too long, starts with digit + assert_eq!(parse_military_timezone_with_offset("j"), None); // Reserved for local time + assert_eq!(parse_military_timezone_with_offset(""), None); // Empty + assert_eq!(parse_military_timezone_with_offset("m999"), None); // Too long + assert_eq!(parse_military_timezone_with_offset("9m"), None); // Starts with digit + } + + #[test] + fn test_abbreviation_resolves_relative_date_against_now() { + let now = "2025-03-15T20:00:00+00:00[UTC]".parse::().unwrap(); + let result = + parse_date("yesterday 10:00 GMT", &now, DebugOptions::new(false, false)).unwrap(); + assert_eq!(result.date(), jiff::civil::date(2025, 3, 14)); + } + + #[test] + fn test_strip_parenthesized_comments() { + assert_eq!(strip_parenthesized_comments("hello"), "hello"); + assert_eq!(strip_parenthesized_comments("2026-01-05"), "2026-01-05"); + assert_eq!(strip_parenthesized_comments("("), ""); + assert_eq!(strip_parenthesized_comments("1(comment"), "1"); + assert_eq!(strip_parenthesized_comments("2026-01-05(this is a comment"), "2026-01-05"); + assert_eq!(strip_parenthesized_comments("2026(comment)-01-05"), "2026-01-05"); + assert_eq!(strip_parenthesized_comments("()"), ""); + assert_eq!(strip_parenthesized_comments("((foo)2026-01-05)"), ""); + + // These cases test the balanced parentheses removal feature + // which extends beyond what GNU date strictly supports + assert_eq!(strip_parenthesized_comments("a(b)c"), "ac"); + assert_eq!(strip_parenthesized_comments("a(b)c(d)e"), "ace"); + assert_eq!(strip_parenthesized_comments("(a)(b)"), ""); + + // When parentheses are unmatched, processing stops at the unmatched opening + // paren + assert_eq!(strip_parenthesized_comments("a(b)c(d"), "ac"); + + // Additional edge cases for nested and complex parentheses + assert_eq!(strip_parenthesized_comments("a(b(c)d)e"), "ae"); // Nested balanced + assert_eq!(strip_parenthesized_comments("a(b(c)d"), "a"); // Nested unbalanced + assert_eq!(strip_parenthesized_comments("a(b)c(d)e(f"), "ace"); // Multiple groups, last unmatched + } + + // --- pi-uutils behavior contracts --- + + #[test] + fn utc_date_string_formats_exactly() { + let (code, stdout, stderr) = + run_in(PathBuf::from("."), vec!["-u", "-d", "2026-01-02 03:04:05", "+%Y-%m-%dT%H:%M:%S"]); + assert_eq!(code, 0); + assert_eq!(stdout, "2026-01-02T03:04:05\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn epoch_input_round_trips_through_seconds_format() { + let (code, stdout, stderr) = + run_in(PathBuf::from("."), vec!["-u", "-d", "@1767323045", "+%s"]); + assert_eq!(code, 0); + assert_eq!(stdout, "1767323045\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn set_is_unsupported() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--set", "2026-01-02 03:04:05"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "date: setting the date is not supported by this builtin\n"); + + let (code, _, stderr) = run_in(PathBuf::from("."), vec!["-s", "now"]); + assert_eq!(code, 1); + assert!(stderr.contains("not supported by this builtin")); + } + + #[test] + fn datefile_relative_path_resolves_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + std::fs::write(root.join("dates.txt"), "2026-01-02 03:04:05\n@0\n").unwrap(); + + // Relative operand + scope cwd differing from the process cwd: only the + // call-site `pi_uutils_ctx::resolve` patch makes this find the file. + let (code, stdout, stderr) = run_in(root, vec!["-u", "-f", "dates.txt", "+%F"]); + assert_eq!(code, 0); + assert_eq!(stdout, "2026-01-02\n1970-01-01\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn datefile_bad_line_reports_but_keeps_processing() { + let (_dir, root) = canonical_tempdir(); + std::fs::write(root.join("dates.txt"), "foobarbaz\n@86400\n").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["-u", "-f", "dates.txt", "+%F"]); + assert_eq!(code, 1, "a bad line must fail the invocation"); + assert_eq!(stdout, "1970-01-02\n", "good lines after a bad one still print"); + assert!(stderr.contains("invalid date 'foobarbaz'"), "stderr: {stderr}"); + } + + #[test] + fn invalid_date_string_reports_and_fails() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["-d", "foobarbaz"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "date: invalid date 'foobarbaz'\n"); + } + + #[test] + fn rfc_email_iso_and_rfc3339_formats() { + let fixed = "2026-01-02 03:04:05"; + + let (code, stdout, _) = run_in(PathBuf::from("."), vec!["-u", "-d", fixed, "-R"]); + assert_eq!((code, stdout.as_str()), (0, "Fri, 02 Jan 2026 03:04:05 +0000\n")); + + let (code, stdout, _) = run_in(PathBuf::from("."), vec!["-u", "-d", fixed, "-Iseconds"]); + assert_eq!((code, stdout.as_str()), (0, "2026-01-02T03:04:05+00:00\n")); + + let (code, stdout, _) = + run_in(PathBuf::from("."), vec!["-u", "-d", fixed, "--rfc-3339=seconds"]); + assert_eq!((code, stdout.as_str()), (0, "2026-01-02 03:04:05+00:00\n")); + } + + #[test] + fn reference_relative_path_resolves_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + std::fs::write(root.join("ref-file"), b"x").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-u", "-r", "ref-file", "+%s"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let secs: i64 = stdout.trim().parse().unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() as i64; + assert!((now - secs).abs() < 60, "mtime epoch {secs} should be close to now {now}"); + + // Missing file fails, naming the operand as typed. + let (code, _, stderr) = run_in(root, vec!["-r", "missing-file"]); + assert_eq!(code, 1); + assert!(stderr.contains("missing-file"), "stderr: {stderr}"); + } + + #[test] + fn format_operand_without_plus_is_rejected() { + // With -d: GNU's "lacks a leading '+'" message. + let (code, _, stderr) = run_in(PathBuf::from("."), vec!["-d", "2026-01-02", "%F"]); + assert_eq!(code, 1); + assert!(stderr.contains("lacks a leading '+'"), "stderr: {stderr}"); + + // Without a date source: GNU treats the operand as an invalid date. + let (code, _, stderr) = run_in(PathBuf::from("."), vec!["%F"]); + assert_eq!(code, 1); + assert!(stderr.contains("invalid date '%F'"), "stderr: {stderr}"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("FORMAT controls the output")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-date/src/format_modifiers.rs b/crates/vendor/uu-date/src/format_modifiers.rs new file mode 100644 index 000000000..20f3f987c --- /dev/null +++ b/crates/vendor/uu-date/src/format_modifiers.rs @@ -0,0 +1,760 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore strtime + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin: the `translate!` error string is literalized with the +// en-US locale text and the regex cache uses `LazyLock` instead of `OnceLock`. +// No behavior changes. + +//! GNU date format modifier support +//! +//! This module implements GNU-compatible format modifiers for date formatting. +//! These modifiers extend standard strftime format specifiers with optional +//! width and flag modifiers. +//! +//! ## Syntax +//! +//! Format: `%[flags][width]specifier` +//! +//! ### Flags +//! - `-`: Do not pad the field +//! - `_`: Pad with spaces instead of zeros +//! - `0`: Pad with zeros (default for numeric fields) +//! - `^`: Convert to uppercase +//! - `#`: Use opposite case (uppercase becomes lowercase and vice versa) +//! - `+`: Force display of sign (+ for positive, - for negative) +//! +//! ### Width +//! - One or more digits specifying minimum field width +//! - Field will be padded to this width using the padding character +//! +//! ### Examples +//! - `%10Y`: Year padded to 10 digits with zeros (0000001999) +//! - `%_10m`: Month padded to 10 digits with spaces ( 06) +//! - `%-d`: Day without padding (1 instead of 01) +//! - `%^B`: Month name in uppercase (JUNE) +//! - `%+4C`: Century with sign, padded to 4 characters (+019) + +use std::{fmt, sync::LazyLock}; + +use jiff::{ + Zoned, + fmt::strtime::{BrokenDownTime, Config, PosixCustom}, +}; +use regex::Regex; + +/// Error type for format modifier operations +#[derive(Debug)] +pub enum FormatError { + /// Error from the underlying jiff library + JiffError(jiff::Error), + /// Field width calculation overflowed or required allocation failed + FieldWidthTooLarge { width: usize, specifier: String }, +} + +impl fmt::Display for FormatError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::JiffError(e) => write!(f, "{e}"), + // pi-uutils: literalized en-US translation of + // `date-error-format-modifier-width-too-large`. + Self::FieldWidthTooLarge { width, specifier } => { + write!(f, "format modifier width '{width}' is too large for specifier '%{specifier}'") + }, + } + } +} + +impl From for FormatError { + fn from(e: jiff::Error) -> Self { + Self::JiffError(e) + } +} + +/// Regex to match format specifiers with optional modifiers +/// Pattern: % \[flags\] \[width\] specifier +/// Flags: -, _, 0, ^, #, + +/// Width: one or more digits +/// Specifier: any letter or special sequence like :z, ::z, :::z +// pi-uutils: `LazyLock` instead of upstream's function-local `OnceLock`. +static FORMAT_SPEC_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"%([_0^#+-]*)(\d*)(:*[a-zA-Z])").unwrap()); + +/// Check if format string contains any GNU modifiers and format if present. +/// +/// This function combines modifier detection and formatting in a single pass +/// for better performance. If no modifiers are found, returns None and the +/// caller should use standard formatting. If modifiers are found, returns +/// the formatted string. +pub fn format_with_modifiers_if_present( + date: &Zoned, + format_string: &str, + config: &Config, +) -> Option> { + let re = &*FORMAT_SPEC_REGEX; + + // Quick check: does the string contain any modifiers? + let has_modifiers = re.captures_iter(format_string).any(|cap| { + let flags = cap.get(1).map_or("", |m| m.as_str()); + let width_str = cap.get(2).map_or("", |m| m.as_str()); + !flags.is_empty() || !width_str.is_empty() + }); + + if !has_modifiers { + return None; + } + + // If we have modifiers, format the string + Some(format_with_modifiers(date, format_string, config)) +} + +/// Process a format string with GNU modifiers. +/// +/// # Arguments +/// * `date` - The date to format +/// * `format_string` - Format string with GNU modifiers +/// * `config` - Strftime configuration +/// +/// # Returns +/// Formatted string with modifiers applied +/// +/// # Errors +/// Returns `FormatError` if formatting fails +fn format_with_modifiers( + date: &Zoned, + format_string: &str, + config: &Config, +) -> Result { + // First, replace %% with a placeholder to avoid matching it + let placeholder = "\x00PERCENT\x00"; + let temp_format = format_string.replace("%%", placeholder); + + let re = &*FORMAT_SPEC_REGEX; + let mut result = String::new(); + let mut last_end = 0; + + let broken_down = BrokenDownTime::from(date); + + for cap in re.captures_iter(&temp_format) { + let whole_match = cap.get(0).unwrap(); + let flags = cap.get(1).map_or("", |m| m.as_str()); + let width_str = cap.get(2).map_or("", |m| m.as_str()); + let spec = cap.get(3).unwrap().as_str(); + + // Add text before this match + result.push_str(&temp_format[last_end..whole_match.start()]); + + // Format the base specifier first + let base_format = format!("%{spec}"); + let formatted = broken_down.to_string_with_config(config, &base_format)?; + + // Check if this specifier has modifiers + if !flags.is_empty() || !width_str.is_empty() { + // Apply modifiers to the formatted value + let width: usize = width_str.parse().unwrap_or(0); + let explicit_width = !width_str.is_empty(); + let modified = apply_modifiers(&formatted, flags, width, spec, explicit_width)?; + result.push_str(&modified); + } else { + // No modifiers, use formatted value as-is + result.push_str(&formatted); + } + + last_end = whole_match.end(); + } + + // Add remaining text + result.push_str(&temp_format[last_end..]); + + // Restore %% by converting placeholder to % + let result = result.replace(placeholder, "%"); + + Ok(result) +} + +/// Returns true if the specifier produces text output (default pad is space) +/// rather than numeric output (default pad is zero). +fn is_text_specifier(specifier: &str) -> bool { + matches!(specifier.chars().last(), Some('A' | 'a' | 'B' | 'b' | 'h' | 'Z' | 'p' | 'P')) +} + +/// Returns true if the specifier defaults to space padding. +/// This includes text specifiers and numeric specifiers like %e and %k +/// that use blank-padding by default in GNU date. +fn is_space_padded_specifier(specifier: &str) -> bool { + matches!( + specifier.chars().last(), + Some('A' | 'a' | 'B' | 'b' | 'h' | 'Z' | 'p' | 'P' | 'e' | 'k' | 'l') + ) +} + +/// Returns the default width for a specifier. +/// This is used when a flag like `_` is used without an explicit width. +fn get_default_width(specifier: &str) -> usize { + match specifier.chars().last() { + // Day of month: 2 digits (01-31) + Some('d') | Some('e') => 2, + // Month: 2 digits (01-12) + Some('m') => 2, + // Hour: 2 digits (00-23) + Some('H') | Some('k') => 2, + // Hour (12-hour): 2 digits (01-12) + Some('I') | Some('l') => 2, + // Minute: 2 digits (00-59) + Some('M') => 2, + // Second: 2 digits (00-60) + Some('S') => 2, + // Year (2-digit): 2 digits + Some('y') => 2, + // Day of year: 3 digits (001-366) + Some('j') => 3, + // Week number: 2 digits (00-53) + Some('U') | Some('W') | Some('V') => 2, + // Day of week: 1 digit (0-6 or 1-7) + Some('w') | Some('u') => 1, + // Century: 2 digits (00-99) + Some('C') => 2, + // Full year: 4 digits + Some('Y') | Some('G') => 4, + // ISO week year (2-digit): 2 digits + Some('g') => 2, + // Epoch seconds: typically 10 digits (but variable) + Some('s') => 0, + // Nanoseconds: 9 digits + Some('N') => 9, + // Quarter: 1 digit + Some('q') => 1, + // Timezone offset: varies + Some('z') => 0, + // Text specifiers have no default width + _ => 0, + } +} + +/// Strip default padding (leading zeros or leading spaces) from a value, +/// preserving at least one character. +fn strip_default_padding(value: &str) -> String { + if value.starts_with('0') && value.len() >= 2 { + let stripped = value.trim_start_matches('0'); + if stripped.is_empty() { + return "0".to_string(); + } + if let Some(first_char) = stripped.chars().next() + && first_char.is_ascii_digit() + { + return stripped.to_string(); + } + } + if value.starts_with(' ') { + let stripped = value.trim_start(); + if !stripped.is_empty() { + return stripped.to_string(); + } + } + value.to_string() +} + +/// Apply width and flag modifiers to a formatted value. +/// +/// The `specifier` parameter is the format specifier (e.g., "d", "B", "Y") +/// which determines the default padding character (space for text, zero for +/// numeric). Flags are processed in order so that when conflicting flags +/// appear, the last one takes precedence (e.g., `_+` means `+` wins for +/// padding). +/// +/// The `explicit_width` parameter indicates whether a width was explicitly +/// specified in the format string (true) or if width is 0 (false). +fn apply_modifiers( + value: &str, + flags: &str, + width: usize, + specifier: &str, + explicit_width: bool, +) -> Result { + let mut result = value.to_string(); + + // Determine default pad character based on specifier type + // Determine default pad character based on specifier type. + // Text specifiers (month names, etc.) and numeric specifiers like %e, %k, %l + // default to space padding; other numeric specifiers default to zero padding. + let default_pad = if is_space_padded_specifier(specifier) { + ' ' + } else { + '0' + }; + + // Process flags in order - last conflicting flag wins + let mut pad_char = default_pad; + let mut no_pad = false; + let mut uppercase = false; + let mut swap_case = false; + let mut force_sign = false; + let mut underscore_flag = false; + + for flag in flags.chars() { + match flag { + '-' => { + no_pad = true; + }, + '_' => { + no_pad = false; + pad_char = ' '; + underscore_flag = true; + }, + '0' => { + no_pad = false; + pad_char = '0'; + }, + '^' => { + uppercase = true; + swap_case = false; // ^ overrides # + }, + '#' if !uppercase => { + // Only apply # if ^ hasn't been set + swap_case = true; + }, + '+' => { + force_sign = true; + no_pad = false; + pad_char = '0'; + }, + _ => {}, + } + } + + // Apply case modifications (uppercase takes precedence over swap_case) + if uppercase { + result = result.to_uppercase(); + } else if swap_case { + if result + .chars() + .all(|c| !c.is_alphabetic() || c.is_uppercase()) + { + result = result.to_lowercase(); + } else if !result + .chars() + .all(|c| !c.is_alphabetic() || c.is_lowercase()) + { + result = result.to_uppercase(); + } + } + + // If no_pad flag is active, suppress all padding and return + if no_pad { + return Ok(strip_default_padding(&result)); + } + + // Handle padding flag without explicit width: use default width + // This applies when _ or 0 flag overrides the default padding character + // and no explicit width is specified (e.g., %_m, %0e) + let effective_width = if !explicit_width && (underscore_flag || pad_char != default_pad) { + get_default_width(specifier) + } else { + width + }; + + // When the requested width is narrower than the default formatted width, GNU + // first removes default padding and then reapplies the requested width. + if effective_width > 0 && effective_width < result.len() { + result = strip_default_padding(&result); + } + + // Strip default padding when switching pad characters on numeric fields + if !is_text_specifier(specifier) && result.len() >= 2 { + if pad_char == ' ' && result.starts_with('0') { + // Switching to space padding: strip leading zeros + result = strip_default_padding(&result); + } else if pad_char == '0' && result.starts_with(' ') { + // Switching to zero padding: strip leading spaces + result = strip_default_padding(&result); + } + } + + // Apply force sign for numeric values + // GNU behavior: + only adds sign if: + // 1. An explicit width is provided, OR + // 2. The value exceeds the default width for that specifier (e.g., year > 4 + // digits) + if force_sign + && !result.starts_with('+') + && !result.starts_with('-') + && result.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + let default_w = get_default_width(specifier); + // Add sign only if explicit width provided OR result exceeds default width + if explicit_width || (default_w > 0 && result.len() > default_w) { + result.insert(0, '+'); + } + } + + // Apply width padding + if effective_width > result.len() { + let padding = effective_width - result.len(); + let has_sign = result.starts_with('+') || result.starts_with('-'); + + if pad_char == '0' && has_sign { + // Zero padding: sign first, then zeros (e.g., "-0022") + let sign = result.chars().next().unwrap(); + let rest = &result[1..]; + let mut padded = try_alloc_padded(result.len(), padding, effective_width, specifier)?; + padded.push(sign); + padded.extend(std::iter::repeat_n('0', padding)); + padded.push_str(rest); + result = padded; + } else { + // Default: pad on the left (e.g., " -22" or " 1999") + let mut padded = try_alloc_padded(result.len(), padding, effective_width, specifier)?; + padded.extend(std::iter::repeat_n(pad_char, padding)); + padded.push_str(&result); + result = padded; + } + } + + Ok(result) +} + +/// Allocate a `String` with enough capacity for `current_len + padding`, +/// returning `FieldWidthTooLarge` on arithmetic overflow or allocation failure. +fn try_alloc_padded( + current_len: usize, + padding: usize, + width: usize, + specifier: &str, +) -> Result { + let target_len = current_len + .checked_add(padding) + .ok_or_else(|| FormatError::FieldWidthTooLarge { width, specifier: specifier.to_string() })?; + let mut s = String::new(); + s.try_reserve(target_len) + .map_err(|_| FormatError::FieldWidthTooLarge { width, specifier: specifier.to_string() })?; + Ok(s) +} + +#[cfg(test)] +mod tests { + use jiff::{civil, tz::TimeZone}; + + use super::*; + + fn make_test_date(year: i16, month: i8, day: i8, hour: i8) -> Zoned { + civil::date(year, month, day) + .at(hour, 0, 0, 0) + .to_zoned(TimeZone::UTC) + .unwrap() + } + + fn get_config() -> Config { + Config::new().custom(PosixCustom::new()).lenient(true) + } + + #[test] + fn test_width_and_padding_modifiers() { + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + + // Test basic width with zero padding + let result = format_with_modifiers(&date, "%10Y", &config).unwrap(); + assert_eq!(result, "0000001999"); + + // Test large width + let result = format_with_modifiers(&date, "%20Y", &config).unwrap(); + assert_eq!(result, "00000000000000001999"); + assert_eq!(result.len(), 20); + + // Test underscore (space) padding with month + let result = format_with_modifiers(&date, "%_10m", &config).unwrap(); + assert_eq!(result, " 6"); + assert_eq!(result.len(), 10); + + // Test underscore padding with day + let date_day5 = make_test_date(1999, 6, 5, 0); + let result = format_with_modifiers(&date_day5, "%_10d", &config).unwrap(); + assert_eq!(result, " 5"); + } + + #[test] + fn test_no_pad_and_case_flags() { + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + + // Test no-pad: %-10Y suppresses all padding (width ignored) + let result = format_with_modifiers(&date, "%-10Y", &config).unwrap(); + assert_eq!(result, "1999"); + + // Test no-pad: %-d strips default zero padding + let result = format_with_modifiers(&date, "%-d", &config).unwrap(); + assert_eq!(result, "1"); + + // Test uppercase: %^B should uppercase month name + let result = format_with_modifiers(&date, "%^B", &config).unwrap(); + assert_eq!(result, "JUNE"); + + // Test uppercase with width: %^10B should uppercase and space-pad (text + // specifier) + let result = format_with_modifiers(&date, "%^10B", &config).unwrap(); + assert_eq!(result, " JUNE"); + assert_eq!(result.len(), 10); + } + + #[test] + fn test_sign_flags() { + let date = make_test_date(1970, 1, 1, 0); + let config = get_config(); + + // Test force sign with century: %+4C + let result = format_with_modifiers(&date, "%+4C", &config).unwrap(); + assert!(result.starts_with('+')); + assert_eq!(result.len(), 4); + + // Test force sign with zero padding: %+6Y + let result = format_with_modifiers(&date, "%+6Y", &config).unwrap(); + assert_eq!(result, "+01970"); + } + + #[test] + fn test_combined_flags_underscore_and_sign() { + let date = make_test_date(1970, 1, 1, 0); + let config = get_config(); + // %_+6Y: _ sets space pad, then + overrides to zero pad with sign (last wins) + let result = format_with_modifiers(&date, "%_+6Y", &config).unwrap(); + assert_eq!(result, "+01970"); + } + + #[test] + fn test_combined_flags_no_pad_and_uppercase() { + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + // %-^10B: uppercase + no-pad (- suppresses all padding, width ignored) + let result = format_with_modifiers(&date, "%-^10B", &config).unwrap(); + assert_eq!(result, "JUNE"); + } + + #[test] + fn test_swap_case_flag() { + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + // %#B: swap case on "June" (mixed case) → uppercase + let result = format_with_modifiers(&date, "%#B", &config).unwrap(); + assert_eq!(result, "JUNE"); + } + + #[test] + fn test_width_smaller_than_result() { + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + // %1d: width 1 < "01".len() → strip zero padding → "1" + let result = format_with_modifiers(&date, "%1d", &config).unwrap(); + assert_eq!(result, "1"); + } + + #[test] + fn test_edge_cases_and_special_formats() { + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + + // Test width zero (no effect) + let result = format_with_modifiers(&date, "%Y", &config).unwrap(); + assert_eq!(result, "1999"); + + // Test no modifiers (standard format) + let result = format_with_modifiers(&date, "%Y-%m-%d", &config).unwrap(); + assert_eq!(result, "1999-06-01"); + + // Test %% escape sequence + let result = format_with_modifiers(&date, "%%Y=%Y", &config).unwrap(); + assert_eq!(result, "%Y=1999"); + + // Test multiple modifiers in one format string + // %-5d: no-pad suppresses all padding → "1" (width ignored) + let result = format_with_modifiers(&date, "%10Y-%_5m-%-5d", &config).unwrap(); + assert_eq!(result, "0000001999- 6-1"); + } + + #[test] + fn test_modifier_detection() { + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + + // Should detect modifiers + let result = format_with_modifiers_if_present(&date, "%10Y", &config); + assert!(result.is_some()); + + // Should not detect modifiers + let result = format_with_modifiers_if_present(&date, "%Y-%m-%d", &config); + assert!(result.is_none()); + + // Should detect flag without width + let result = format_with_modifiers_if_present(&date, "%^B", &config); + assert!(result.is_some()); + } + + #[test] + fn test_negative_values_with_space_padding() { + // Test case from GNU test: neg-secs2 + // Format: %_5s with value -22 should produce " -22" (space-padded) + use jiff::Timestamp; + + let ts = Timestamp::from_second(-22).unwrap(); + let date = ts.to_zoned(TimeZone::UTC); + let config = get_config(); + + let result = format_with_modifiers(&date, "%_5s", &config).unwrap(); + assert_eq!(result, " -22", "Space padding should pad before the sign for negative numbers"); + } + + // Unit tests for apply_modifiers function + #[test] + fn test_apply_modifiers_basic() { + // No modifiers (numeric specifier) + assert_eq!(apply_modifiers("1999", "", 0, "Y", false).unwrap(), "1999"); + // Zero padding + assert_eq!(apply_modifiers("1999", "0", 10, "Y", true).unwrap(), "0000001999"); + // Space padding (strips leading zeros) + assert_eq!(apply_modifiers("06", "_", 5, "m", true).unwrap(), " 6"); + // No-pad (strips leading zeros, width ignored) + assert_eq!(apply_modifiers("01", "-", 5, "d", true).unwrap(), "1"); + // Uppercase + assert_eq!(apply_modifiers("june", "^", 0, "B", false).unwrap(), "JUNE"); + // Swap case: all uppercase → lowercase + assert_eq!(apply_modifiers("UTC", "#", 0, "Z", false).unwrap(), "utc"); + // Swap case: mixed case → uppercase + assert_eq!(apply_modifiers("June", "#", 0, "B", false).unwrap(), "JUNE"); + } + + #[test] + fn test_apply_modifiers_signs() { + // Force sign with explicit width + assert_eq!(apply_modifiers("1970", "+", 6, "Y", true).unwrap(), "+01970"); + // Force sign without explicit width: should NOT add sign for 4-digit year + assert_eq!(apply_modifiers("1999", "+", 0, "Y", false).unwrap(), "1999"); + // Force sign without explicit width: SHOULD add sign for year > 4 digits + assert_eq!(apply_modifiers("12345", "+", 0, "Y", false).unwrap(), "+12345"); + // Negative with zero padding: sign first, then zeros + assert_eq!(apply_modifiers("-22", "0", 5, "s", true).unwrap(), "-0022"); + // Negative with space padding: spaces first, then sign + assert_eq!(apply_modifiers("-22", "_", 5, "s", true).unwrap(), " -22"); + // Force sign (_+): + is last, overrides _ → zero pad with sign + assert_eq!(apply_modifiers("5", "_+", 5, "s", true).unwrap(), "+0005"); + // No-pad + uppercase: no padding applied + assert_eq!(apply_modifiers("june", "-^", 10, "B", true).unwrap(), "JUNE"); + } + + #[test] + fn test_case_flag_precedence() { + // Test that ^ (uppercase) overrides # (swap case) + assert_eq!(apply_modifiers("June", "^#", 0, "B", false).unwrap(), "JUNE"); + assert_eq!(apply_modifiers("June", "#^", 0, "B", false).unwrap(), "JUNE"); + // Test # alone (swap case) + assert_eq!(apply_modifiers("June", "#", 0, "B", false).unwrap(), "JUNE"); + assert_eq!(apply_modifiers("JUNE", "#", 0, "B", false).unwrap(), "june"); + } + + #[test] + fn test_apply_modifiers_text_specifiers() { + // Text specifiers default to space padding + assert_eq!(apply_modifiers("June", "", 10, "B", true).unwrap(), " June"); + assert_eq!(apply_modifiers("Mon", "", 10, "a", true).unwrap(), " Mon"); + // Numeric specifiers default to zero padding + assert_eq!(apply_modifiers("6", "", 10, "m", true).unwrap(), "0000000006"); + } + + #[test] + fn test_apply_modifiers_width_smaller_than_result() { + // Width smaller than result strips default padding + assert_eq!(apply_modifiers("01", "", 1, "d", true).unwrap(), "1"); + assert_eq!(apply_modifiers("06", "", 1, "m", true).unwrap(), "6"); + } + + #[test] + fn test_apply_modifiers_parametrized() { + let test_cases = vec![ + ("1", "0", 3, "Y", true, "001"), + ("1", "_", 3, "d", true, " 1"), + ("1", "-", 3, "d", true, "1"), // no-pad: width ignored + ("abc", "^", 5, "B", true, " ABC"), // text specifier: space pad + ("5", "+", 4, "s", true, "+005"), + ("5", "_+", 4, "s", true, "+005"), // + is last: zero pad with sign + ("-3", "0", 5, "s", true, "-0003"), + ("05", "_", 3, "d", true, " 5"), + ("09", "-", 4, "d", true, "9"), // no-pad: width ignored + ("1970", "_+", 6, "Y", true, "+01970"), // + is last: zero pad with sign + ]; + + for (value, flags, width, spec, explicit_width, expected) in test_cases { + assert_eq!( + apply_modifiers(value, flags, width, spec, explicit_width).unwrap(), + expected, + "value='{value}', flags='{flags}', width={width}, spec='{spec}', \ + explicit_width={explicit_width}", + ); + } + } + + #[test] + fn test_apply_modifiers_width_too_large() { + let err = apply_modifiers("x", "", usize::MAX, "c", true).unwrap_err(); + assert!(matches!( + err, + FormatError::FieldWidthTooLarge { width, specifier } + if width == usize::MAX && specifier == "c" + )); + } + + #[test] + fn test_underscore_flag_without_width() { + // %_m should pad month to default width 2 with spaces + assert_eq!(apply_modifiers("6", "_", 0, "m", false).unwrap(), " 6"); + // %_d should pad day to default width 2 with spaces + assert_eq!(apply_modifiers("1", "_", 0, "d", false).unwrap(), " 1"); + // %_H should pad hour to default width 2 with spaces + assert_eq!(apply_modifiers("5", "_", 0, "H", false).unwrap(), " 5"); + // %_Y should pad year to default width 4 with spaces + assert_eq!(apply_modifiers("1999", "_", 0, "Y", false).unwrap(), "1999"); + // already at default width + } + + #[test] + fn test_plus_flag_without_width() { + // %+Y without width should NOT add sign for 4-digit year + assert_eq!(apply_modifiers("1999", "+", 0, "Y", false).unwrap(), "1999"); + // %+Y without width SHOULD add sign for year > 4 digits + assert_eq!(apply_modifiers("12345", "+", 0, "Y", false).unwrap(), "+12345"); + // %+Y with explicit width should add sign + assert_eq!(apply_modifiers("1999", "+", 6, "Y", true).unwrap(), "+01999"); + } + + #[test] + fn test_zero_flag_on_space_padded_specifiers() { + // GNU date: %0e should override space-padding with zero-padding + // Verified: `date -d "2024-06-05" "+%0e"` → "05" + let date = make_test_date(1999, 6, 5, 5); + let config = get_config(); + + // %0e: day-of-month (normally space-padded) with 0 flag → zero-padded + let result = format_with_modifiers(&date, "%0e", &config).unwrap(); + assert_eq!(result, "05", "GNU: %0e should produce '05', not ' 5'"); + + // %0k: hour (normally space-padded) with 0 flag → zero-padded + let result = format_with_modifiers(&date, "%0k", &config).unwrap(); + assert_eq!(result, "05", "GNU: %0k should produce '05', not ' 5'"); + } + + #[test] + fn test_underscore_century_default_width() { + // GNU date: %C default width is 2, not 4 + // Verified: `date -d "2024-06-15" "+%_C"` → "20" (no extra padding) + let date = make_test_date(1999, 6, 1, 0); + let config = get_config(); + + // %_C: century with underscore flag, no explicit width + // Default width for %C should be 2 (century is 00-99) + let result = format_with_modifiers(&date, "%_C", &config).unwrap(); + assert_eq!( + result, "19", + "GNU: %_C should produce '19', not ' 19' (default width is 2, not 4)" + ); + } +} diff --git a/crates/vendor/uu-hostname/Cargo.toml b/crates/vendor/uu-hostname/Cargo.toml new file mode 100644 index 000000000..94660b7d7 --- /dev/null +++ b/crates/vendor/uu-hostname/Cargo.toml @@ -0,0 +1,31 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/hostname), patched to route +# output through pi-uutils-ctx and to reject the set-hostname path so it can run +# in-process as a shell builtin. See src/hostname.rs for the patch markers +# (`pi-uutils:` comments). +[package] +name = "uu_hostname" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "hostname ~ (uutils) display the host name of the current host (vendored + patched for in-process embedding)" + +[lib] +path = "src/hostname.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +hostname = "0.4" +uucore = { version = "0.8.0", features = ["wide"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[target.'cfg(any(target_os = "freebsd", target_os = "openbsd"))'.dependencies] +dns-lookup = "3.0.0" + +[target.'cfg(target_os = "windows")'.dependencies] +windows-sys = { version = "0.61.0", features = [ + "Win32_Networking_WinSock", + "Win32_Foundation", +] } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-hostname/LICENSE b/crates/vendor/uu-hostname/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-hostname/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-hostname/src/hostname.rs b/crates/vendor/uu-hostname/src/hostname.rs new file mode 100644 index 000000000..24e79b3e9 --- /dev/null +++ b/crates/vendor/uu-hostname/src/hostname.rs @@ -0,0 +1,308 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore hashset Addrs addrs + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. The set-hostname path is removed entirely (a NAME operand +// is rejected with an "unsupported" error instead of calling `hostname::set`, +// so the `hostname` crate's "set" feature is dropped), all output is routed +// through the context streams, `translate!` strings are literalized, and the +// entry point no longer calls `std::process::exit`. + +#[cfg(not(any(target_os = "freebsd", target_os = "openbsd")))] +use std::net::ToSocketAddrs; +use std::{collections::hash_set::HashSet, ffi::OsString, io::Write, str}; + +use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser}; +#[cfg(any(target_os = "freebsd", target_os = "openbsd"))] +use dns_lookup::lookup_host; +use pi_uutils_ctx::format_usage; +use uucore::error::{FromIo, UResult, USimpleError}; + +static OPT_DOMAIN: &str = "domain"; +static OPT_IP_ADDRESS: &str = "ip-address"; +static OPT_FQDN: &str = "fqdn"; +static OPT_SHORT: &str = "short"; +static OPT_HOST: &str = "host"; + +#[cfg(windows)] +mod wsa { + use std::io; + + use windows_sys::Win32::Networking::WinSock::{WSACleanup, WSADATA, WSAStartup}; + + pub(super) struct WsaHandle(()); + + pub(super) fn start() -> io::Result { + let mut data = std::mem::MaybeUninit::::uninit(); + let err = unsafe { WSAStartup(0x0202, data.as_mut_ptr()) }; + if err == 0 { + Ok(WsaHandle(())) + } else { + Err(io::Error::from_raw_os_error(err)) + } + } + + impl Drop for WsaHandle { + fn drop(&mut self) { + // This possibly returns an error but we can't handle it + let _ = unsafe { WSACleanup() }; + } + } +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match hostname_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "hostname: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn hostname_main(matches: &ArgMatches) -> UResult<()> { + #[cfg(windows)] + let _handle = wsa::start().map_err_context(|| "failed to start Winsock".to_string())?; + + match matches.get_one::(OPT_HOST) { + None => display_hostname(matches), + // pi-uutils: setting the process-global hostname from inside a shell + // builtin is refused (upstream calls `hostname::set` here). + Some(_host) => Err(USimpleError::new( + 1, + "setting the hostname is not supported by this builtin".to_string(), + )), + } +} + +pub fn uu_app() -> Command { + Command::new("hostname") + .version(uucore::crate_version!()) + .about("Display or set the system's host name.") + .override_usage(format_usage("hostname [OPTION]... [HOSTNAME]")) + .infer_long_args(true) + .arg( + Arg::new(OPT_DOMAIN) + .short('d') + .long("domain") + .overrides_with_all([OPT_DOMAIN, OPT_IP_ADDRESS, OPT_FQDN, OPT_SHORT]) + .help("Display the name of the DNS domain if possible") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_IP_ADDRESS) + .short('i') + .long("ip-address") + .overrides_with_all([OPT_DOMAIN, OPT_IP_ADDRESS, OPT_FQDN, OPT_SHORT]) + .help("Display the network address(es) of the host") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_FQDN) + .short('f') + .long("fqdn") + .overrides_with_all([OPT_DOMAIN, OPT_IP_ADDRESS, OPT_FQDN, OPT_SHORT]) + .help("Display the FQDN (Fully Qualified Domain Name) (default)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_SHORT) + .short('s') + .long("short") + .overrides_with_all([OPT_DOMAIN, OPT_IP_ADDRESS, OPT_FQDN, OPT_SHORT]) + .help("Display the short hostname (the portion before the first dot) if possible") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_HOST) + .value_parser(ValueParser::os_string()) + .value_hint(clap::ValueHint::Hostname), + ) +} + +fn display_hostname(matches: &ArgMatches) -> UResult<()> { + let hostname = hostname::get() + .map_err_context(|| "failed to get hostname".to_owned())? + .to_string_lossy() + .into_owned(); + + // pi-uutils: all output below goes to the context stdout instead of the + // process stdout. + let mut out = pi_uutils_ctx::stdout(); + + if matches.get_flag(OPT_IP_ADDRESS) { + let addresses; + + #[cfg(not(any(target_os = "freebsd", target_os = "openbsd")))] + { + let hostname = hostname + ":1"; + let addrs = hostname + .to_socket_addrs() + .map_err_context(|| "failed to resolve socket addresses".to_owned())?; + addresses = addrs; + } + + // DNS reverse lookup via "hostname:1" does not work on FreeBSD and OpenBSD + // use dns-lookup crate instead + #[cfg(any(target_os = "freebsd", target_os = "openbsd"))] + { + let addrs: Vec = lookup_host(hostname.as_str()) + .map_err_context(|| "failed to lookup hostname".to_owned())? + .collect(); + addresses = addrs; + } + + let mut hashset = HashSet::new(); + let mut output = String::new(); + for addr in addresses { + // XXX: not sure why this is necessary... + if !hashset.contains(&addr) { + let mut ip = addr.to_string(); + if ip.ends_with(":1") { + let len = ip.len(); + ip.truncate(len - 2); + } + output.push_str(&ip); + output.push(' '); + hashset.insert(addr); + } + } + let len = output.len(); + if len > 0 { + writeln!(out, "{}", &output[0..len - 1])?; + } + + Ok(()) + } else { + if matches.get_flag(OPT_SHORT) || matches.get_flag(OPT_DOMAIN) { + let mut it = hostname.char_indices().filter(|&ci| ci.1 == '.'); + if let Some(ci) = it.next() { + if matches.get_flag(OPT_SHORT) { + writeln!(out, "{}", &hostname[0..ci.0])?; + } else { + writeln!(out, "{}", &hostname[ci.0 + 1..])?; + } + } else if matches.get_flag(OPT_SHORT) { + writeln!(out, "{hostname}")?; + } + return Ok(()); + } + + writeln!(out, "{hostname}")?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd: PathBuf::from("."), + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("hostname") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + #[test] + fn bare_invocation_prints_hostname() { + let (code, stdout, stderr) = run_in(vec![]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert!(stdout.ends_with('\n')); + assert!(!stdout.trim_end().is_empty()); + } + + #[test] + fn set_attempt_is_rejected() { + let (code, stdout, stderr) = run_in(vec!["new-name.example.com"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "hostname: setting the hostname is not supported by this builtin\n"); + } + + #[test] + fn short_is_dotless_prefix_of_full_hostname() { + let (code, short, stderr) = run_in(vec!["-s"]); + let (_, full, _) = run_in(vec![]); + assert_eq!((code, stderr.as_str()), (0, "")); + let short = short.trim_end(); + assert!(!short.contains('.'), "-s must strip everything after the first dot"); + assert!(full.trim_end().starts_with(short)); + } + + #[test] + fn fqdn_flag_matches_default_display() { + // -f is the default display mode; it must print the same name as the + // bare invocation, not attempt any set path. + let (code, fqdn, stderr) = run_in(vec!["-f"]); + let (_, bare, _) = run_in(vec![]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert_eq!(fqdn, bare); + } +} diff --git a/crates/vendor/uu-ln/Cargo.toml b/crates/vendor/uu-ln/Cargo.toml new file mode 100644 index 000000000..2cc4020a4 --- /dev/null +++ b/crates/vendor/uu-ln/Cargo.toml @@ -0,0 +1,23 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/ln), patched to resolve path +# arguments against the shell working directory and route I/O + prompts through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/ln.rs for +# the patch markers (`pi-uutils:` comments). +[package] +name = "uu_ln" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "ln ~ (uutils) create a (file system) link to TARGET (vendored + patched for in-process embedding)" + +[lib] +path = "src/ln.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +thiserror = "2.0.3" +uucore = { version = "0.8.0", features = ["backup-control", "fs"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-ln/LICENSE b/crates/vendor/uu-ln/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-ln/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-ln/src/ln.rs b/crates/vendor/uu-ln/src/ln.rs new file mode 100644 index 000000000..d193c8b26 --- /dev/null +++ b/crates/vendor/uu-ln/src/ln.rs @@ -0,0 +1,831 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) srcpath targetpath EEXIST + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Every filesystem syscall resolves its path operand +// against the shell working directory via `pi_uutils_ctx::resolve` AT THE CALL +// SITE, while the original operands are kept for display/error messages (GNU +// prints operands as typed) — and, crucially, for the CONTENT of symbolic +// links, which stays exactly as typed like GNU ln (only the location where the +// link is created gets resolved). All process-global stdio and the `-i` prompt +// are routed through `pi_uutils_ctx`, `translate!` strings are literalized, and +// the entry point no longer calls `std::process::exit`. + +#[cfg(any(unix, target_os = "redox"))] +use std::os::unix::fs::symlink; +#[cfg(windows)] +use std::os::windows::fs::{symlink_dir, symlink_file}; +use std::{ + borrow::Cow, + collections::HashSet, + ffi::OsString, + fs, + io::Write, + path::{Path, PathBuf}, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use pi_uutils_ctx::format_usage; +use thiserror::Error; +use uucore::{ + backup_control::{self, BackupMode}, + display::Quotable, + error::{FromIo, UError, UResult, USimpleError, strip_errno}, + fs::{ + MissingHandling, ResolveMode, canonicalize, make_path_relative_to, paths_refer_to_same_file, + }, +}; + +pub struct Settings { + overwrite: OverwriteMode, + backup: BackupMode, + suffix: OsString, + symbolic: bool, + relative: bool, + logical: bool, + target_dir: Option, + no_target_dir: bool, + no_dereference: bool, + verbose: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum OverwriteMode { + NoClobber, + Interactive, + Force, +} + +// pi-uutils: the `translate!` message templates are literalized with the +// en-US strings from upstream's locales/en-US.ftl. +#[derive(Error, Debug)] +enum LnError { + #[error("target {} is not a directory", _0.quote())] + TargetIsNotADirectory(PathBuf), + + #[error("")] + SomeLinksFailed, + + #[error("{} and {} are the same file", _0.quote(), _1.quote())] + SameFile(PathBuf, PathBuf), + + #[error("missing destination file operand after {}", _0.quote())] + MissingDestination(PathBuf), + + #[error("extra operand {}\nTry '{} --help' for more information.", _0.quote(), _1)] + ExtraOperand(OsString, String), + + #[error("{}: hard link not allowed for directory", _0.to_string_lossy())] + FailedToCreateHardLinkDir(PathBuf), +} + +impl UError for LnError { + fn code(&self) -> i32 { + 1 + } +} + +mod options { + pub const FORCE: &str = "force"; + //pub const DIRECTORY: &str = "directory"; + pub const INTERACTIVE: &str = "interactive"; + pub const NO_DEREFERENCE: &str = "no-dereference"; + pub const SYMBOLIC: &str = "symbolic"; + pub const LOGICAL: &str = "logical"; + pub const PHYSICAL: &str = "physical"; + pub const TARGET_DIRECTORY: &str = "target-directory"; + pub const NO_TARGET_DIRECTORY: &str = "no-target-directory"; + pub const RELATIVE: &str = "relative"; + pub const VERBOSE: &str = "verbose"; +} + +static ARG_FILES: &str = "files"; + +/// pi-uutils: replacement for uucore's `show_error!` — writes the diagnostic +/// to the context stderr instead of the process-global one. Errors that render +/// to an empty message (e.g. [`LnError::SomeLinksFailed`]) print nothing +/// rather than a dangling "ln: " prefix. +fn show_error(msg: impl std::fmt::Display) { + let rendered = msg.to_string(); + if !rendered.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "ln: {rendered}"); + } +} + +/// pi-uutils: replacement for uucore's `read_yes`, reading from the context +/// stdin one byte at a time (no buffering) so consecutive prompts don't +/// over-read into a later prompt's input. Returns true when the first character +/// of the line is `y`/`Y`. +fn read_yes() -> bool { + use std::io::Read as _; + let mut stdin = pi_uutils_ctx::stdin(); + let mut buf = [0u8; 1]; + let mut first = None; + loop { + match stdin.read(&mut buf) { + Ok(0) => break, // EOF + Ok(_) => { + if buf[0] == b'\n' { + break; + } + if first.is_none() { + first = Some(buf[0]); + } + }, + Err(_) => return false, + } + } + matches!(first, Some(b'y' | b'Y')) +} + +/// pi-uutils: replacement for uucore's `prompt_yes!` — writes +/// "ln: \ " to the context stderr, then reads the answer from the +/// context stdin. +fn prompt_yes(prompt: impl std::fmt::Display) -> bool { + let mut err = pi_uutils_ctx::stderr(); + let _ = write!(err, "ln: {prompt} "); + let _ = err.flush(); + read_yes() +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match ln_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + // pi-uutils: `SomeLinksFailed` renders to an empty message + // (upstream prints the per-file diagnostics as it goes); don't + // emit a dangling "ln: " prefix for it. + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "ln: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn ln_main(matches: &ArgMatches) -> UResult<()> { + /* the list of files */ + + let paths: Vec = matches + .get_many::(ARG_FILES) + .unwrap() + .map(PathBuf::from) + .collect(); + + let symbolic = matches.get_flag(options::SYMBOLIC); + + let overwrite_mode = if matches.get_flag(options::FORCE) { + OverwriteMode::Force + } else if matches.get_flag(options::INTERACTIVE) { + OverwriteMode::Interactive + } else { + OverwriteMode::NoClobber + }; + + let backup_mode = backup_control::determine_backup_mode(matches)?; + let backup_suffix = backup_control::determine_backup_suffix(matches); + + // When we have "-L" or "-L -P", false otherwise + let logical = matches.get_flag(options::LOGICAL); + + let settings = Settings { + overwrite: overwrite_mode, + backup: backup_mode, + suffix: OsString::from(backup_suffix), + symbolic, + logical, + relative: matches.get_flag(options::RELATIVE), + target_dir: matches + .get_one::(options::TARGET_DIRECTORY) + .map(PathBuf::from), + no_target_dir: matches.get_flag(options::NO_TARGET_DIRECTORY), + no_dereference: matches.get_flag(options::NO_DEREFERENCE), + verbose: matches.get_flag(options::VERBOSE), + }; + + exec(&paths[..], &settings) +} + +pub fn uu_app() -> Command { + let after_help = format!( + "In the 1st form, create a link to TARGET with the name LINK_NAME.\nIn the 2nd form, create \ + a link to TARGET in the current directory.\nIn the 3rd and 4th forms, create links to each \ + TARGET in DIRECTORY.\nCreate hard links by default, symbolic links with --symbolic.\nBy \ + default, each destination (name of new link) should not already exist.\nWhen creating hard \ + links, each TARGET must exist. Symbolic links\ncan hold arbitrary text; if later resolved, \ + a relative link is\ninterpreted in relation to its parent directory.\n\n{}", + backup_control::BACKUP_CONTROL_LONG_HELP + ); + + Command::new("ln") + .version(uucore::crate_version!()) + .about("Make links between files.") + .override_usage(format_usage( + "ln [OPTION]... [-T] TARGET LINK_NAME\nln [OPTION]... TARGET\nln [OPTION]... TARGET... \ + DIRECTORY\nln [OPTION]... -t DIRECTORY TARGET...", + )) + .infer_long_args(true) + .after_help(after_help) + .arg(backup_control::arguments::backup()) + .arg(backup_control::arguments::backup_no_args()) + /*.arg( + Arg::new(options::DIRECTORY) + .short('d') + .long(options::DIRECTORY) + .help("allow users with appropriate privileges to attempt to make hard links to directories") + )*/ + .arg( + Arg::new(options::FORCE) + .short('f') + .long(options::FORCE) + .help("remove existing destination files") + .overrides_with(options::INTERACTIVE) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::INTERACTIVE) + .short('i') + .long(options::INTERACTIVE) + .help("prompt whether to remove existing destination files") + .overrides_with(options::FORCE) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::NO_DEREFERENCE) + .short('n') + .long(options::NO_DEREFERENCE) + .help("treat LINK_NAME as a normal file if it is a\nsymbolic link to a directory") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::LOGICAL) + .short('L') + .long(options::LOGICAL) + .help("follow TARGETs that are symbolic links") + .overrides_with(options::PHYSICAL) + .action(ArgAction::SetTrue), + ) + .arg( + // Not implemented yet + Arg::new(options::PHYSICAL) + .short('P') + .long(options::PHYSICAL) + .help("make hard links directly to symbolic links") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::SYMBOLIC) + .short('s') + .long(options::SYMBOLIC) + .help("make symbolic links instead of hard links") + // override added for https://github.com/uutils/coreutils/issues/2359 + .overrides_with(options::SYMBOLIC) + .action(ArgAction::SetTrue), + ) + .arg(backup_control::arguments::suffix()) + .arg( + Arg::new(options::TARGET_DIRECTORY) + .short('t') + .long(options::TARGET_DIRECTORY) + .help("specify the DIRECTORY in which to create the links") + .value_name("DIRECTORY") + .value_hint(clap::ValueHint::DirPath) + .value_parser(clap::value_parser!(OsString)) + .conflicts_with(options::NO_TARGET_DIRECTORY), + ) + .arg( + Arg::new(options::NO_TARGET_DIRECTORY) + .short('T') + .long(options::NO_TARGET_DIRECTORY) + .help("treat LINK_NAME as a normal file always") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::RELATIVE) + .short('r') + .long(options::RELATIVE) + .help("create symbolic links relative to link location") + .requires(options::SYMBOLIC) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::VERBOSE) + .short('v') + .long(options::VERBOSE) + .help("print name of each linked file") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(ARG_FILES) + .action(ArgAction::Append) + .value_hint(clap::ValueHint::AnyPath) + .value_parser(clap::value_parser!(OsString)) + .required(true) + .num_args(1..), + ) +} + +fn exec(files: &[PathBuf], settings: &Settings) -> UResult<()> { + // Handle cases where we create links in a directory first. + if let Some(target_path) = &settings.target_dir { + // 4th form: a directory is specified by -t. + return link_files_in_dir(files, target_path, settings); + } + if !settings.no_target_dir { + if files.len() == 1 { + // 2nd form: the target directory is the current directory. + return link_files_in_dir(files, &PathBuf::from("."), settings); + } + let last_file = &PathBuf::from(files.last().unwrap()); + // pi-uutils: probe the destination via the resolved path. + if files.len() > 2 || pi_uutils_ctx::resolve(last_file).is_dir() { + // 3rd form: create links in the last argument. + return link_files_in_dir(&files[0..files.len() - 1], last_file, settings); + } + } + + // 1st form. Now there should be only two operands, but if -T is + // specified we may have a wrong number of operands. + if files.len() == 1 { + return Err(LnError::MissingDestination(files[0].clone()).into()); + } + if files.len() > 2 { + // pi-uutils: `uucore::execution_phrase()` reads the process argv, + // which is the host shell's; the builtin is always invoked as "ln". + return Err(LnError::ExtraOperand(files[2].clone().into(), "ln".to_string()).into()); + } + assert!(!files.is_empty()); + + link(&files[0], &files[1], settings) +} + +#[allow(clippy::cognitive_complexity)] +fn link_files_in_dir(files: &[PathBuf], target_dir: &Path, settings: &Settings) -> UResult<()> { + // pi-uutils: resolved target directory for every syscall below; the + // operand keeps its as-typed spelling for display and link-name building. + let target_dir_fs = pi_uutils_ctx::resolve(target_dir); + if !target_dir_fs.is_dir() { + return Err(LnError::TargetIsNotADirectory(target_dir.to_owned()).into()); + } + // remember the linked destinations for further usage + let mut linked_destinations: HashSet = HashSet::with_capacity(files.len()); + + let mut all_successful = true; + for srcpath in files { + let targetpath = if settings.no_dereference && target_dir_fs.is_symlink() { + let remove_target = || { + // In that case, we don't want to do link resolution + // We need to clean the target + if target_dir_fs.is_file() + && let Err(e) = fs::remove_file(&target_dir_fs) + { + show_error(format_args!("Could not update {}: {e}", target_dir.quote())); + } + #[cfg(windows)] + if target_dir_fs.is_dir() { + // Not sure why but on Windows, the symlink can be + // considered as a dir + // See test_ln::test_symlink_no_deref_dir + if let Err(e) = fs::remove_dir(&target_dir_fs) { + show_error(format_args!("Could not update {}: {e}", target_dir.quote())); + } + } + }; + match settings.overwrite { + OverwriteMode::NoClobber => {}, + OverwriteMode::Interactive => { + if prompt_yes(format_args!("replace {}?", target_dir.quote())) { + remove_target(); + } + }, + OverwriteMode::Force => { + remove_target(); + }, + } + target_dir.to_path_buf() + } else if let Some(name) = srcpath.as_os_str().to_str() { + match Path::new(name).file_name() { + Some(basename) => target_dir.join(basename), + // This can be None only for "." or "..". Trying + // to create a link with such name will fail with + // EEXIST, which agrees with the behavior of GNU + // coreutils. + None => target_dir.join(name), + } + } else { + show_error(format_args!("cannot stat {}: No such file or directory", srcpath.quote())); + all_successful = false; + continue; + }; + + if linked_destinations.contains(&targetpath) { + // If the target file was already created in this ln call, do not overwrite + show_error(format_args!( + "will not overwrite just-created {} with {}", + targetpath.quote(), + srcpath.quote() + )); + all_successful = false; + } else if let Err(e) = link(srcpath, &targetpath, settings) { + show_error(format_args!("{e}")); + all_successful = false; + } + + linked_destinations.insert(targetpath.clone()); + } + if all_successful { + Ok(()) + } else { + Err(LnError::SomeLinksFailed.into()) + } +} + +fn relative_path<'a>(src: &'a Path, dst: &Path) -> Cow<'a, Path> { + // pi-uutils: canonicalize from the resolved operands so `-r` computes the + // link text against the shell working directory (uucore's canonicalize + // would otherwise fall back to the process cwd for relative paths). + if let Ok(src_abs) = + canonicalize(pi_uutils_ctx::resolve(src), MissingHandling::Missing, ResolveMode::Physical) + && let Ok(dst_abs) = canonicalize( + pi_uutils_ctx::resolve(dst.parent().unwrap()), + MissingHandling::Missing, + ResolveMode::Physical, + ) { + return make_path_relative_to(src_abs, dst_abs).into(); + } + src.into() +} + +#[allow(clippy::cognitive_complexity)] +fn link(src: &Path, dst: &Path, settings: &Settings) -> UResult<()> { + let mut backup_path = None; + let source: Cow<'_, Path> = if settings.relative { + relative_path(src, dst) + } else { + src.into() + }; + + // pi-uutils: resolved counterparts of both operands for every filesystem + // syscall below. `src`/`dst`/`source` keep the as-typed spelling for + // display — and `source` is what gets stored as the symlink CONTENT, so it + // must never be resolved. + let src_fs = pi_uutils_ctx::resolve(src); + let dst_fs = pi_uutils_ctx::resolve(dst); + + if dst_fs.is_symlink() || dst_fs.exists() { + // pi-uutils: probe numbered backups from the resolved destination so + // the directory scan hits the shell's working directory. + backup_path = backup_control::get_backup_path(settings.backup, &dst_fs, &settings.suffix); + if settings.backup == BackupMode::Existing && !settings.symbolic { + // when ln --backup f f, it should detect that it is the same file + if paths_refer_to_same_file(&src_fs, &dst_fs, true) { + return Err(LnError::SameFile(src.to_owned(), dst.to_owned()).into()); + } + } + if let Some(p) = &backup_path { + fs::rename(&dst_fs, p).map_err_context(|| format!("cannot backup {}", dst.quote()))?; + } + match settings.overwrite { + OverwriteMode::NoClobber => {}, + OverwriteMode::Interactive => { + if !prompt_yes(format_args!("replace {}?", dst.quote())) { + return Err(LnError::SomeLinksFailed.into()); + } + + let _ = fs::remove_file(&dst_fs); + // In case of error, don't do anything + }, + OverwriteMode::Force => { + if !dst_fs.is_symlink() && paths_refer_to_same_file(&src_fs, &dst_fs, true) { + // Even in force overwrite mode, verify we are not targeting the same entry and + // return a SameFile error if so + let same_entry = match ( + canonicalize(&src_fs, MissingHandling::Missing, ResolveMode::Physical), + canonicalize(&dst_fs, MissingHandling::Missing, ResolveMode::Physical), + ) { + (Ok(src), Ok(dst)) => src == dst, + _ => true, + }; + if same_entry { + return Err(LnError::SameFile(src.to_owned(), dst.to_owned()).into()); + } + } + let _ = fs::remove_file(&dst_fs); + // In case of error, don't do anything + }, + } + } + + let res: UResult<()> = if settings.symbolic { + // pi-uutils: the link is created at the resolved location, but its + // content (`source`) stays exactly as typed, like GNU ln. uucore's + // io-error conversion renders EEXIST as "Already exists"; format the + // GNU-style diagnostic ("failed to create symbolic link 'x': File + // exists") from the raw OS error instead. + symlink(&source, &dst_fs).map_err(|e| { + USimpleError::new( + 1, + format!("failed to create symbolic link {}: {}", dst.quote(), strip_errno(&e)), + ) + }) + } else { + // pi-uutils: hard links dereference their target, so the resolved + // source is what the syscalls get. + let source_fs = pi_uutils_ctx::resolve(&source); + let p = if settings.logical && source_fs.is_symlink() { + fs::canonicalize(&source_fs) + .map_err_context(|| format!("failed to access {}", source.quote()))? + } else { + source_fs + }; + match fs::hard_link(&p, &dst_fs) { + Ok(()) => Ok(()), + Err(_) if p.is_dir() => { + Err(LnError::FailedToCreateHardLinkDir(source.to_path_buf()).into()) + }, + // pi-uutils: same GNU-style rendering as the symlink arm (uucore + // would print "Already exists" for EEXIST). + Err(e) => Err(USimpleError::new( + 1, + format!( + "failed to create hard link {} => {}: {}", + source.quote(), + dst.quote(), + strip_errno(&e) + ), + )), + } + }; + + if let Err(e) = res { + if let Some(p) = &backup_path { + fs::rename(p, &dst_fs).map_err_context(|| format!("cannot backup {}", dst.quote()))?; + } + return Err(e); + } + + if settings.verbose { + // pi-uutils: verbose output goes to the context stdout. + let mut out = pi_uutils_ctx::stdout(); + write!(out, "{} -> {}", dst.quote(), source.quote())?; + match backup_path { + Some(path) => { + // pi-uutils: `path` derives from the resolved (absolute) + // destination; rebuild a display path from the operand for + // the verbose message. + let backup_display = match (dst.parent(), path.file_name()) { + (Some(parent), Some(name)) if !parent.as_os_str().is_empty() => parent.join(name), + (_, Some(name)) => PathBuf::from(name), + _ => path.clone(), + }; + writeln!(out, " (backup: {})", backup_display.quote())?; + }, + None => writeln!(out)?, + } + } + Ok(()) +} + +#[cfg(windows)] +pub fn symlink, P2: AsRef>(src: P1, dst: P2) -> std::io::Result<()> { + // pi-uutils: the dir/file probe resolves the target against the shell + // working directory (upstream consults the process cwd); the stored link + // content is still the caller's as-typed `src`. + if pi_uutils_ctx::resolve(src.as_ref()).is_dir() { + symlink_dir(src, dst) + } else { + symlink_file(src, dst) + } +} + +#[cfg(target_os = "wasi")] +fn symlink, P2: AsRef>(_src: P1, _dst: P2) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "symlinks not supported on this platform", + )) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_with_stdin(cwd: PathBuf, args: Vec<&str>, stdin: &[u8]) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::Cursor::new(stdin.to_vec())), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("ln") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + run_with_stdin(cwd, args, b"") + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var, + /// which canonicalizing code paths would otherwise expand mid-assertion). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + #[cfg(unix)] + #[test] + fn symlink_relative_operands_create_in_scope_cwd_with_literal_content() { + let (_dir, root) = canonical_tempdir(); + + // Relative operands + scope cwd differing from the process cwd: only + // the call-site `pi_uutils_ctx::resolve` patch places the link in the + // tempdir — while the CONTENT must stay exactly as typed. + let (code, stdout, stderr) = run_in(root.clone(), vec!["-s", "target", "link"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + + let link = root.join("link"); + assert!(link.is_symlink(), "link must be created inside the scope cwd"); + assert_eq!(fs::read_link(&link).unwrap(), PathBuf::from("target")); + } + + #[cfg(unix)] + #[test] + fn hard_link_shares_inode() { + use std::os::unix::fs::MetadataExt; + + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("a"), b"payload").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["a", "b"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + + assert_eq!(fs::read(root.join("b")).unwrap(), b"payload"); + assert_eq!(fs::metadata(root.join("a")).unwrap().nlink(), 2); + assert_eq!( + fs::metadata(root.join("a")).unwrap().ino(), + fs::metadata(root.join("b")).unwrap().ino() + ); + } + + #[cfg(unix)] + #[test] + fn existing_destination_without_force_fails_with_file_exists() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("link"), b"old").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-s", "target", "link"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "ln: failed to create symbolic link 'link': File exists\n"); + assert_eq!(fs::read(root.join("link")).unwrap(), b"old", "destination must be untouched"); + } + + #[cfg(unix)] + #[test] + fn force_overwrites_existing_destination() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("link"), b"old").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-sf", "target", "link"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + assert_eq!(fs::read_link(root.join("link")).unwrap(), PathBuf::from("target")); + } + + #[cfg(unix)] + #[test] + fn verbose_symlink_prints_mapping_to_stdout() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root, vec!["-sv", "target", "link"]); + assert_eq!(code, 0); + assert_eq!(stdout, "'link' -> 'target'\n"); + assert_eq!(stderr, ""); + } + + #[cfg(unix)] + #[test] + fn interactive_prompt_reads_ctx_stdin() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("link"), b"old").unwrap(); + + // Decline: destination untouched, some-links-failed exit code, no + // dangling "ln: " diagnostic beyond the prompt itself. + let (code, stdout, stderr) = + run_with_stdin(root.clone(), vec!["-si", "target", "link"], b"n\n"); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "ln: replace 'link'? "); + assert!(!root.join("link").is_symlink()); + + // Accept: existing file is replaced by the symlink. + let (code, _, stderr) = run_with_stdin(root.clone(), vec!["-si", "target", "link"], b"y\n"); + assert_eq!(code, 0); + assert_eq!(stderr, "ln: replace 'link'? "); + assert_eq!(fs::read_link(root.join("link")).unwrap(), PathBuf::from("target")); + } + + #[cfg(unix)] + #[test] + fn relative_flag_computes_link_text_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("target"), b"x").unwrap(); + fs::create_dir(root.join("sub")).unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-sr", "target", "sub/link"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + assert_eq!(fs::read_link(root.join("sub").join("link")).unwrap(), PathBuf::from("../target")); + } + + #[cfg(unix)] + #[test] + fn target_directory_flag_places_links_in_directory() { + let (_dir, root) = canonical_tempdir(); + fs::create_dir(root.join("d")).unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-s", "-t", "d", "x"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + assert_eq!(fs::read_link(root.join("d").join("x")).unwrap(), PathBuf::from("x")); + } + + #[test] + fn missing_destination_is_an_error() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root, vec!["-T", "only"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!( + stderr.contains("missing destination file operand after 'only'"), + "stderr was: {stderr:?}" + ); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("Make links between files.")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-mktemp/Cargo.toml b/crates/vendor/uu-mktemp/Cargo.toml new file mode 100644 index 000000000..dae4ba329 --- /dev/null +++ b/crates/vendor/uu-mktemp/Cargo.toml @@ -0,0 +1,24 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/mktemp), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/mktemp.rs +# for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_mktemp" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "mktemp ~ (uutils) create and display a temporary file or directory from TEMPLATE (vendored + patched for in-process embedding)" + +[lib] +path = "src/mktemp.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +rand = { version = "0.10.0", features = ["std_rng"] } +tempfile = "3.15.0" +thiserror = "2.0.3" +uucore = "0.8.0" +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-mktemp/LICENSE b/crates/vendor/uu-mktemp/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-mktemp/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-mktemp/src/mktemp.rs b/crates/vendor/uu-mktemp/src/mktemp.rs new file mode 100644 index 000000000..7e1769b53 --- /dev/null +++ b/crates/vendor/uu-mktemp/src/mktemp.rs @@ -0,0 +1,877 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (paths) GPGHome findxs + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. The temp-file parent directory (from `-p`/`--tmpdir` or a +// relative TEMPLATE prefix) is resolved against the shell working directory via +// `pi_uutils_ctx::resolve` at the creation sites, so the printed path is the +// path actually created. TMPDIR/POSIXLY_CORRECT are read from the scope +// environment, all stdio goes through `pi_uutils_ctx`, `translate!` strings are +// literalized, and the entry point no longer calls `std::process::exit`. + +#[cfg(unix)] +use std::fs; +#[cfg(unix)] +use std::os::unix::prelude::PermissionsExt; +use std::{ + env, + ffi::{OsStr, OsString}, + io::{ErrorKind, Write}, + iter, + path::{MAIN_SEPARATOR, Path, PathBuf}, +}; + +use clap::{ + Arg, ArgAction, ArgMatches, Command, + builder::{TypedValueParser, ValueParserFactory}, +}; +use pi_uutils_ctx::format_usage; +use rand::{ + RngExt as _, SeedableRng as _, + rngs::{self, SmallRng}, +}; +use tempfile::Builder; +use thiserror::Error; +use uucore::{ + display::Quotable, + error::{FromIo, UError, UResult}, +}; + +static DEFAULT_TEMPLATE: &str = "tmp.XXXXXXXXXX"; + +static OPT_DIRECTORY: &str = "directory"; +static OPT_DRY_RUN: &str = "dry-run"; +static OPT_QUIET: &str = "quiet"; +static OPT_SUFFIX: &str = "suffix"; +static OPT_TMPDIR: &str = "tmpdir"; +static OPT_P: &str = "p"; +static OPT_T: &str = "t"; + +static ARG_TEMPLATE: &str = "template"; + +#[cfg(not(windows))] +const TMPDIR_ENV_VAR: &str = "TMPDIR"; +#[cfg(windows)] +const TMPDIR_ENV_VAR: &str = "TMP"; + +const FALLBACK_TMPDIR: &str = "/tmp"; + +// pi-uutils: `translate!` error strings literalized from locales/en-US.ftl. +#[derive(Error, Debug)] +enum MkTempError { + #[error("could not persist file {}", .0.quote())] + PersistError(PathBuf), + + #[error("with --suffix, template {} must end in X", .0.quote())] + MustEndInX(String), + + #[error("too few X's in template {}", .0.quote())] + TooFewXs(String), + + #[error("invalid template, {}, contains directory separator", .0.quote())] + PrefixContainsDirSeparator(String), + + #[error("invalid suffix {}, contains directory separator", .0.quote())] + SuffixContainsDirSeparator(String), + + #[error("invalid template, {}; with --tmpdir, it may not be absolute", .0.quote())] + InvalidTemplate(OsString), + + #[error("too many templates")] + TooManyTemplates, + + #[error("failed to create {} via template {}: No such file or directory", .0, .1.quote())] + NotFound(String, PathBuf), +} + +impl UError for MkTempError { + fn usage(&self) -> bool { + matches!(self, Self::TooManyTemplates) + } +} + +/// Options parsed from the command-line. +/// +/// This provides a layer of indirection between the application logic +/// and the argument parsing library `clap`, allowing each to vary +/// independently. +#[derive(Clone)] +pub struct Options { + /// Whether to create a temporary directory instead of a file. + pub directory: bool, + + /// Whether to just print the name of a file that would have been created. + pub dry_run: bool, + + /// Whether to suppress file creation error messages. + pub quiet: bool, + + /// The directory in which to create the temporary file. + /// + /// If `None`, the file will be created in the current directory. + pub tmpdir: Option, + + /// The suffix to append to the temporary file, if any. + pub suffix: Option, + + /// Whether to treat the template argument as a single file path component. + pub treat_as_template: bool, + + /// The template to use for the name of the temporary file. + pub template: OsString, +} + +impl Options { + fn from(matches: &ArgMatches) -> Self { + let tmpdir = matches + .get_one::>(OPT_TMPDIR) + .or_else(|| matches.get_one::>(OPT_P)) + .map(|dir| match dir { + // If the argument of -p/--tmpdir is non-empty, use it as the + // tmpdir. + Some(d) => d.clone(), + // Otherwise use $TMPDIR if set, else use the system's default + // temporary directory. + None => get_tmpdir_env_or_default(), + }); + let (tmpdir, template) = match matches.get_one::(ARG_TEMPLATE) { + // If no template argument is given, `--tmpdir` is implied. + None => { + let tmpdir = Some(tmpdir.unwrap_or_else(get_tmpdir_env_or_default)); + let template = DEFAULT_TEMPLATE; + (tmpdir, OsString::from(template)) + }, + Some(template) => { + // pi-uutils: TMPDIR comes from the scope environment, not the + // host process environment. + let tmpdir = if let Some(tmpdir) = pi_uutils_ctx::var(TMPDIR_ENV_VAR) + && matches.get_flag(OPT_T) + { + Some(PathBuf::from(tmpdir)) + } else if tmpdir.is_some() { + tmpdir + } else if matches.get_flag(OPT_T) || matches.contains_id(OPT_TMPDIR) { + // If --tmpdir is given without an argument, or -t is given + // export in TMPDIR + Some(env::temp_dir()) + } else { + None + }; + (tmpdir, template.clone()) + }, + }; + Self { + directory: matches.get_flag(OPT_DIRECTORY), + dry_run: matches.get_flag(OPT_DRY_RUN), + quiet: matches.get_flag(OPT_QUIET), + tmpdir, + suffix: matches.get_one::(OPT_SUFFIX).cloned(), + treat_as_template: matches.get_flag(OPT_T), + template, + } + } +} + +/// Parameters that control the path to and name of the temporary file. +/// +/// The temporary file will be created at +/// +/// ```text +/// {directory}/{prefix}{XXX}{suffix} +/// ``` +/// +/// where `{XXX}` is a sequence of random characters whose length is +/// `num_rand_chars`. +struct Params { + /// The directory that will contain the temporary file. + directory: PathBuf, + + /// The (non-random) prefix of the temporary file. + prefix: String, + + /// The number of random characters in the name of the temporary file. + num_rand_chars: usize, + + /// The (non-random) suffix of the temporary file. + suffix: String, +} + +/// Find the start and end indices of the last contiguous block of Xs. +/// +/// If no contiguous block of at least three Xs could be found, this +/// function returns `None`. +/// +/// # Examples +/// +/// ```rust,ignore +/// assert_eq!(find_last_contiguous_block_of_xs("XXX_XXX"), Some((4, 7))); +/// assert_eq!(find_last_contiguous_block_of_xs("aXbXcX"), None); +/// ``` +fn find_last_contiguous_block_of_xs(s: &str) -> Option<(usize, usize)> { + let bytes = s.as_bytes(); + + // Find the index of the last 'X'. + let end = bytes.iter().rposition(|&b| b == b'X')?; + + // Walk left to find the start of the run of Xs that ends at `end`. + let mut start = end; + while start > 0 && bytes[start - 1] == b'X' { + start -= 1; + } + + if end + 1 - start >= 3 { + Some((start, end + 1)) + } else { + None + } +} + +impl Params { + fn from(options: Options) -> Result { + // Convert OsString template to string for processing + // When using -t flag, be permissive with invalid UTF-8 like GNU mktemp + // Otherwise, maintain strict UTF-8 validation (existing behavior) + let template_str = if options.treat_as_template { + // For -t templates, use lossy conversion for GNU compatibility + options.template.to_string_lossy().into_owned() + } else { + // For regular templates, maintain strict validation + match options.template.to_str() { + Some(s) => s.to_string(), + None => { + return Err(MkTempError::InvalidTemplate("template contains invalid UTF-8".into())); + }, + } + }; + + // The template argument must end in 'X' if a suffix option is given. + if options.suffix.is_some() && !template_str.ends_with('X') { + return Err(MkTempError::MustEndInX(template_str.clone())); + } + + // Get the start and end indices of the randomized part of the template. + // + // For example, if the template is "abcXXXXyz", then `i` is 3 and `j` is 7. + let Some((i, j)) = find_last_contiguous_block_of_xs(&template_str) else { + let s = match options.suffix { + // If a suffix is specified, the error message includes the template without the suffix. + Some(_) => template_str + .chars() + .take(template_str.len()) + .collect::(), + None => template_str.clone(), + }; + return Err(MkTempError::TooFewXs(s)); + }; + + // Combine the directory given as an option and the prefix of the template. + // + // For example, if `tmpdir` is "a/b" and the template is "c/dXXX", + // then `prefix` is "a/b/c/d". + let tmpdir = options.tmpdir; + let prefix_from_option = tmpdir.clone().unwrap_or_default(); + let prefix_from_template = &template_str[..i]; + let prefix_path = Path::new(&prefix_from_option).join(prefix_from_template); + if options.treat_as_template && prefix_from_template.contains(MAIN_SEPARATOR) { + return Err(MkTempError::PrefixContainsDirSeparator(template_str.clone())); + } + if tmpdir.is_some() && Path::new(prefix_from_template).is_absolute() { + return Err(MkTempError::InvalidTemplate(template_str.clone().into())); + } + + // Split the parent directory from the file part of the prefix. + // + // For example, if `prefix_path` is "a/b/c/d", then `directory` is + // "a/b/c" and `prefix` gets reassigned to "d". + let (directory, prefix) = { + let prefix_str = prefix_path.to_string_lossy(); + if prefix_str.ends_with(MAIN_SEPARATOR) { + (prefix_path, String::new()) + } else { + let directory = match prefix_path.parent() { + None => PathBuf::new(), + Some(d) => d.to_path_buf(), + }; + let prefix = match prefix_path.file_name() { + None => String::new(), + Some(f) => f.to_string_lossy().to_string(), + }; + (directory, prefix) + } + }; + + // Combine the suffix from the template with the suffix given as an option. + // + // For example, if the suffix command-line argument is ".txt" and + // the template is "XXXabc", then `suffix` is "abc.txt". + let suffix_from_option = options + .suffix + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + let suffix_from_template = &template_str[j..]; + let suffix = format!("{suffix_from_template}{suffix_from_option}"); + if suffix.contains(MAIN_SEPARATOR) { + return Err(MkTempError::SuffixContainsDirSeparator(suffix)); + } + + // The number of random characters in the template. + // + // For example, if the template is "abcXXXXyz", then the number of + // random characters is four. + let num_rand_chars = j - i; + + Ok(Self { directory, prefix, num_rand_chars, suffix }) + } +} + +/// Custom parser that converts empty string to `None`, and non-empty string to +/// `Some(PathBuf)`. +/// +/// This parser is used for the `-p` and `--tmpdir` options where an empty +/// string argument should be treated as "not provided", causing mktemp to fall +/// back to using the `$TMPDIR` environment variable or the system's default +/// temporary directory. +/// +/// # Examples +/// +/// - Empty string `""` -> `None` +/// - Non-empty string `"/tmp"` -> `Some(PathBuf::from("/tmp"))` +/// +/// This handles the special case where users can pass an empty directory name +/// to explicitly request fallback behavior. +#[derive(Clone, Debug)] +struct OptionalPathBufParser; + +impl TypedValueParser for OptionalPathBufParser { + type Value = Option; + + fn parse_ref( + &self, + _cmd: &Command, + _arg: Option<&Arg>, + value: &OsStr, + ) -> Result { + if value.is_empty() { + Ok(None) + } else { + Ok(Some(PathBuf::from(value))) + } + } +} + +impl ValueParserFactory for OptionalPathBufParser { + type Parser = Self; + + fn value_parser() -> Self::Parser { + Self + } +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(&argv) { + Ok(matches) => matches, + Err(err) => { + // pi-uutils: upstream maps a too-many-values clap error on the + // TEMPLATE argument to the GNU "too many templates" usage error. + if err.kind() == clap::error::ErrorKind::TooManyValues + && err.context().any(|(kind, val)| { + kind == clap::error::ContextKind::InvalidArg + && val == &clap::error::ContextValue::String("[template]".into()) + }) { + let _ = writeln!(pi_uutils_ctx::stderr(), "mktemp: too many templates"); + return 1; + } + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match mktemp_main(&argv, &matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + // pi-uutils: --quiet failures surface as bare exit-code errors + // that render to an empty message; don't emit a dangling + // "mktemp: " prefix. + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "mktemp: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn mktemp_main(args: &[OsString], matches: &ArgMatches) -> UResult<()> { + // Parse command-line options into a format suitable for the + // application logic. + let options = Options::from(matches); + + // pi-uutils: POSIXLY_CORRECT comes from the scope environment. + if pi_uutils_ctx::var("POSIXLY_CORRECT").is_some() { + // If POSIXLY_CORRECT was set, template MUST be the last argument. + if matches.contains_id(ARG_TEMPLATE) { + // Template argument was provided, check if was the last one. + if args.last().unwrap() != &options.template { + return Err(Box::new(MkTempError::TooManyTemplates)); + } + } + } + + let dry_run = options.dry_run; + let suppress_file_err = options.quiet; + let make_dir = options.directory; + + // Parse file path parameters from the command-line options. + let Params { directory: tmpdir, prefix, num_rand_chars: rand, suffix } = Params::from(options)?; + + // Create the temporary file or directory, or simulate creating it. + let res = if dry_run { + Ok(dry_exec(&tmpdir, &prefix, rand, &suffix)) + } else { + exec(&tmpdir, &prefix, rand, &suffix, make_dir) + }; + + let res = if suppress_file_err { + // Mapping all UErrors to ExitCodes prevents the errors from being printed + res.map_err(|e| e.code().into()) + } else { + res + }; + + // pi-uutils: replacement for upstream's `println_verbatim` — writes the + // created path's bytes verbatim to the context stdout instead of the + // process stdout. + let path = res?; + let print = || -> std::io::Result<()> { + let mut out = pi_uutils_ctx::stdout(); + out.write_all(uucore::os_str_as_bytes(path.as_os_str()).map_err(std::io::Error::other)?)?; + out.write_all(b"\n")?; + out.flush() + }; + print().map_err_context(|| "failed to print directory name".to_string())?; + Ok(()) +} + +pub fn uu_app() -> Command { + Command::new("mktemp") + .version(uucore::crate_version!()) + .about("Create a temporary file or directory.") + .override_usage(format_usage("mktemp [OPTION]... [TEMPLATE]")) + .infer_long_args(true) + .arg( + Arg::new(OPT_DIRECTORY) + .short('d') + .long(OPT_DIRECTORY) + .help("Make a directory instead of a file") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_DRY_RUN) + .short('u') + .long(OPT_DRY_RUN) + .help("do not create anything; merely print a name (unsafe)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_QUIET) + .short('q') + .long("quiet") + .help("Fail silently if an error occurs.") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_SUFFIX) + .long(OPT_SUFFIX) + .help( + "append SUFFIX to TEMPLATE; SUFFIX must not contain a path separator. This option \ + is implied if TEMPLATE does not end with X.", + ) + .value_name("SUFFIX") + .value_parser(clap::value_parser!(OsString)), + ) + .arg( + Arg::new(OPT_P) + .short('p') + .help("short form of --tmpdir") + .value_name("DIR") + .num_args(1) + .value_parser(OptionalPathBufParser) + .value_hint(clap::ValueHint::DirPath), + ) + .arg( + Arg::new(OPT_TMPDIR) + .long(OPT_TMPDIR) + .help( + "interpret TEMPLATE relative to DIR; if DIR is not specified, use $TMPDIR ($TMP on \ + windows) if set, else /tmp. With this option, TEMPLATE must not be an absolute \ + name; unlike with -t, TEMPLATE may contain slashes, but mktemp creates only the \ + final component", + ) + .value_name("DIR") + // Allows use of default argument just by setting --tmpdir. Else, + // use provided input to generate tmpdir + .num_args(0..=1) + // Require an equals to avoid ambiguity if no tmpdir is supplied + .require_equals(true) + .overrides_with(OPT_P) + .value_parser(OptionalPathBufParser) + .value_hint(clap::ValueHint::DirPath), + ) + .arg( + Arg::new(OPT_T) + .short('t') + .help( + "Generate a template (using the supplied prefix and TMPDIR (TMP on windows) if \ + set) to create a filename template [deprecated]", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(ARG_TEMPLATE) + .num_args(..=1) + .value_parser(clap::value_parser!(OsString)), + ) +} + +fn dry_exec(tmpdir: &Path, prefix: &str, rand: usize, suffix: &str) -> PathBuf { + // pi-uutils: resolve the parent directory against the shell working + // directory so the printed candidate matches where creation would occur. + let tmpdir = pi_uutils_ctx::resolve(tmpdir); + let len = prefix.len() + suffix.len() + rand; + let mut buf = Vec::with_capacity(len); + buf.extend(prefix.as_bytes()); + buf.extend(iter::repeat_n(b'X', rand)); + buf.extend(suffix.as_bytes()); + + // Randomize. + let bytes = &mut buf[prefix.len()..prefix.len() + rand]; + SmallRng::try_from_rng(&mut rngs::SysRng) + .unwrap_or_else(|_| { + //rand::rng panics if getrandom failed + SmallRng::seed_from_u64(bytes.as_ptr() as usize as u64) + }) + .fill(bytes); + for byte in bytes { + *byte = match *byte % 62 { + v @ 0..=9 => v + b'0', + v @ 10..=35 => v - 10 + b'a', + v @ 36..=61 => v - 36 + b'A', + _ => unreachable!(), + } + } + // We guarantee utf8. + let buf = String::from_utf8(buf).unwrap(); + tmpdir.join(buf) +} + +/// Create a temporary directory with the given parameters. +/// +/// This function creates a temporary directory as a subdirectory of +/// `dir`. The name of the directory is the concatenation of `prefix`, +/// a string of `rand` random characters, and `suffix`. The +/// permissions of the directory are set to `u+rwx` +/// +/// # Errors +/// +/// If the temporary directory could not be written to disk or if the +/// given directory `dir` does not exist. +fn make_temp_dir(dir: &Path, prefix: &str, rand: usize, suffix: &str) -> UResult { + let mut builder = Builder::new(); + builder.prefix(prefix).rand_bytes(rand).suffix(suffix); + + // On *nix platforms grant read-write-execute for owner only. + // The directory is created with these permission at creation time, using + // mkdir(3) syscall. This is not relevant on Windows systems. See: https://docs.rs/tempfile/latest/tempfile/#security + // `fs` is not imported on Windows anyways. + #[cfg(not(windows))] + builder.permissions(fs::Permissions::from_mode(0o700)); + + match builder.tempdir_in(dir) { + Ok(d) => { + // `keep` consumes the TempDir without removing it + let path = d.keep(); + Ok(path) + }, + Err(e) if e.kind() == ErrorKind::NotFound => { + let filename = format!("{prefix}{}{suffix}", "X".repeat(rand)); + let path = Path::new(dir).join(filename); + Err(MkTempError::NotFound("directory".to_string(), path).into()) + }, + Err(e) => Err(e.into()), + } +} + +/// Create a temporary file with the given parameters. +/// +/// This function creates a temporary file in the directory `dir`. The +/// name of the file is the concatenation of `prefix`, a string of +/// `rand` random characters, and `suffix`. The permissions of the +/// file are set to `u+rw`. +/// +/// # Errors +/// +/// If the file could not be written to disk or if the directory does +/// not exist. +fn make_temp_file(dir: &Path, prefix: &str, rand: usize, suffix: &str) -> UResult { + let mut builder = Builder::new(); + builder.prefix(prefix).rand_bytes(rand).suffix(suffix); + match builder.tempfile_in(dir) { + // `keep` ensures that the file is not deleted + Ok(named_tempfile) => match named_tempfile.keep() { + Ok((_, pathbuf)) => Ok(pathbuf), + Err(e) => Err(MkTempError::PersistError(e.file.path().to_path_buf()).into()), + }, + Err(e) if e.kind() == ErrorKind::NotFound => { + let filename = format!("{prefix}{}{suffix}", "X".repeat(rand)); + let path = Path::new(dir).join(filename); + Err(MkTempError::NotFound("file".to_string(), path).into()) + }, + Err(e) => Err(e.into()), + } +} + +fn exec(dir: &Path, prefix: &str, rand: usize, suffix: &str, make_dir: bool) -> UResult { + // pi-uutils: resolve the parent directory against the shell working + // directory at the creation site; the resolved form is also what gets + // printed, so the printed path is the path actually created. + let dir = pi_uutils_ctx::resolve(dir); + let path = if make_dir { + make_temp_dir(&dir, prefix, rand, suffix)? + } else { + make_temp_file(&dir, prefix, rand, suffix)? + }; + + // Get just the last component of the path to the created + // temporary file or directory. + let filename = path.file_name(); + let filename = filename.unwrap().to_str().unwrap(); + + // Join the directory to the path to get the path to print. + // pi-uutils: unlike upstream (which re-joins the operand as typed), join + // the resolved directory so the printed path names the created entry even + // when the shell cwd differs from the process cwd. + let path = dir.join(filename); + + Ok(path) +} + +/// Reads from `TMPDIR_ENV_VAR` but defaults to /tmp if value is set to empty +/// string. +fn get_tmpdir_env_or_default() -> PathBuf { + // pi-uutils: read TMPDIR from the scope environment; when it is unset + // there, fall back to the host default temp dir as upstream does. + match pi_uutils_ctx::var(TMPDIR_ENV_VAR) { + Some(val) if val.is_empty() => PathBuf::from(FALLBACK_TMPDIR), + Some(val) => PathBuf::from(val), + None => env::temp_dir(), + } +} + +/// Create a temporary file or directory +/// +/// Behavior is determined by the `options` parameter, see [`Options`] for +/// details. +pub fn mktemp(options: &Options) -> UResult { + // Parse file path parameters from the command-line options. + let Params { directory: tmpdir, prefix, num_rand_chars: rand, suffix } = + Params::from(options.clone())?; + + // Create the temporary file or directory, or simulate creating it. + if options.dry_run { + Ok(dry_exec(&tmpdir, &prefix, rand, &suffix)) + } else { + exec(&tmpdir, &prefix, rand, &suffix, options.directory) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(cwd: PathBuf, env: HashMap, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env, + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("mktemp") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var, + /// which would otherwise break printed-path assertions). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = std::fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + fn tmpdir_env(dir: &Path) -> HashMap { + HashMap::from([("TMPDIR".to_string(), dir.display().to_string())]) + } + + #[test] + fn default_invocation_creates_file_at_printed_path() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root.clone(), tmpdir_env(&root), vec![]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let printed = PathBuf::from(stdout.trim_end_matches('\n')); + assert!(printed.is_file(), "printed path {printed:?} must be a regular file"); + // Scope TMPDIR is honored for the default template. + assert_eq!(printed.parent(), Some(root.as_path())); + assert!( + printed + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("tmp.") + ); + } + + #[test] + fn directory_flag_creates_directory() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root.clone(), tmpdir_env(&root), vec!["-d"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let printed = PathBuf::from(stdout.trim_end_matches('\n')); + assert!(printed.is_dir(), "printed path {printed:?} must be a directory"); + assert_eq!(printed.parent(), Some(root.as_path())); + } + + #[test] + fn relative_tmpdir_resolves_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + std::fs::create_dir(root.join("sub")).unwrap(); + + // Relative -p operand + scope cwd differing from the process cwd: only + // the creation-site `pi_uutils_ctx::resolve` patch makes this land in + // the scope cwd's subdir. + let (code, stdout, stderr) = + run_in(root.clone(), HashMap::new(), vec!["-p", "sub", "foo.XXXX"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let printed = PathBuf::from(stdout.trim_end_matches('\n')); + assert!(printed.is_file(), "printed path {printed:?} must exist"); + assert_eq!(printed.parent(), Some(root.join("sub").as_path())); + assert!( + printed + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("foo.") + ); + } + + #[test] + fn too_few_xs_is_an_error() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root, HashMap::new(), vec!["foo.XX"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "mktemp: too few X's in template 'foo.XX'\n"); + } + + #[test] + fn dry_run_prints_nonexistent_path() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root.clone(), tmpdir_env(&root), vec!["-u"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let printed = PathBuf::from(stdout.trim_end_matches('\n')); + assert_eq!(printed.parent(), Some(root.as_path())); + assert!(!printed.exists(), "dry-run path {printed:?} must not be created"); + } + + #[test] + fn suffix_is_appended_after_random_block() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = + run_in(root.clone(), HashMap::new(), vec!["--suffix=.txt", "-p", ".", "fooXXXX"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let printed = PathBuf::from(stdout.trim_end_matches('\n')); + assert!(printed.is_file()); + let name = printed.file_name().unwrap().to_str().unwrap().to_string(); + assert!(name.starts_with("foo") && name.ends_with(".txt"), "unexpected name {name}"); + } + + #[test] + fn quiet_suppresses_creation_error_message_but_not_exit_code() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = + run_in(root, HashMap::new(), vec!["-q", "-p", "missing-dir", "foo.XXXX"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "", "--quiet must suppress the creation error message"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), HashMap::new(), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("temporary file or directory")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-nproc/Cargo.toml b/crates/vendor/uu-nproc/Cargo.toml new file mode 100644 index 000000000..a075046ad --- /dev/null +++ b/crates/vendor/uu-nproc/Cargo.toml @@ -0,0 +1,22 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/nproc), patched to read the +# OMP_NUM_THREADS/OMP_THREAD_LIMIT environment variables from the shell scope +# environment and route I/O through pi-uutils-ctx so it can run in-process as a +# shell builtin. See src/nproc.rs for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_nproc" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "nproc ~ (uutils) display the number of processing units available (vendored + patched for in-process embedding)" + +[lib] +path = "src/nproc.rs" + +[dependencies] +libc = "0.2.172" +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +uucore = { version = "0.8.0", features = ["fs"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-nproc/LICENSE b/crates/vendor/uu-nproc/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-nproc/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-nproc/src/nproc.rs b/crates/vendor/uu-nproc/src/nproc.rs new file mode 100644 index 000000000..5b6cfed2b --- /dev/null +++ b/crates/vendor/uu-nproc/src/nproc.rs @@ -0,0 +1,287 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) NPROCESSORS nprocs numstr sysconf + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. The OMP_NUM_THREADS and OMP_THREAD_LIMIT environment +// variables are read from the scope environment via `pi_uutils_ctx::var` (the +// shell's exported variables), not the host process environment. All output is +// routed through the context stdout, `translate!` strings are literalized, and +// the entry point no longer calls `std::process::exit`. + +use std::{ffi::OsString, io::Write, thread}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use pi_uutils_ctx::format_usage; +use uucore::{ + display::Quotable, + error::{UResult, USimpleError}, +}; + +static OPT_ALL: &str = "all"; +static OPT_IGNORE: &str = "ignore"; + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match nproc_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "nproc: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn nproc_main(matches: &ArgMatches) -> UResult<()> { + let ignore = match matches.get_one::(OPT_IGNORE) { + Some(numstr) => match numstr.trim().parse::() { + Ok(num) => num, + Err(e) => { + return Err(USimpleError::new( + 1, + // pi-uutils: literalized translate!("nproc-error-invalid-number") + format!("{} is not a valid number: {e}", numstr.quote()), + )); + }, + }, + None => 0, + }; + + // pi-uutils: OMP_THREAD_LIMIT comes from the scope environment (the + // shell's exported variables), not the host process environment. + let limit = match pi_uutils_ctx::var("OMP_THREAD_LIMIT") { + // Uses the OpenMP variable to limit the number of threads + // If the parsing fails, returns the max size (so, no impact) + // If OMP_THREAD_LIMIT=0, rejects the value + Some(threads) => match threads.parse() { + Ok(0) | Err(_) => usize::MAX, + Ok(n) => n, + }, + // the variable 'OMP_THREAD_LIMIT' doesn't exist + // fallback to the max + None => usize::MAX, + }; + + let mut cores = if matches.get_flag(OPT_ALL) { + num_cpus_all() + } else { + // OMP_NUM_THREADS doesn't have an impact on --all + // pi-uutils: OMP_NUM_THREADS comes from the scope environment. + match pi_uutils_ctx::var("OMP_NUM_THREADS") { + // Uses the OpenMP variable to force the number of threads + // If the parsing fails, returns the number of CPU + Some(threads) => { + // In some cases, OMP_NUM_THREADS can be "x,y,z" + // In this case, only take the first one (like GNU) + // If OMP_NUM_THREADS=0, rejects the value + match threads.split_terminator(',').next() { + None => available_parallelism(), + Some(s) => match s.trim().parse() { + Ok(0) | Err(_) => available_parallelism(), + Ok(n) => n, + }, + } + }, + // the variable 'OMP_NUM_THREADS' doesn't exist + // fallback to the regular CPU detection + None => available_parallelism(), + } + }; + + cores = std::cmp::min(limit, cores); + if cores <= ignore { + cores = 1; + } else { + cores -= ignore; + } + // pi-uutils: write to the context stdout instead of the process stdout. + pi_uutils_ctx::stdout() + .write_all(format!("{cores}\n").as_bytes()) + .map_err(|e| USimpleError::new(1, e.to_string()))?; + Ok(()) +} + +pub fn uu_app() -> Command { + Command::new("nproc") + .version(uucore::crate_version!()) + .about( + "Print the number of cores available to the current process.\nIf the OMP_NUM_THREADS or \ + OMP_THREAD_LIMIT environment variables are set, then\nthey will determine the minimum \ + and maximum returned value respectively.", + ) + .override_usage(format_usage("nproc [OPTIONS]...")) + .infer_long_args(true) + .arg( + Arg::new(OPT_ALL) + .long(OPT_ALL) + .help("print the number of cores available to the system") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_IGNORE) + .long(OPT_IGNORE) + .value_name("N") + .help("ignore up to N cores"), + ) +} + +#[cfg(unix)] +fn num_cpus_all() -> usize { + // In some situation, /proc and /sys are not mounted, and sysconf returns 1. + // However, we want to guarantee that `nproc --all` >= `nproc`. + unsafe { libc::sysconf(libc::_SC_NPROCESSORS_CONF) } + .try_into() + .ok() + .filter(|&n: &isize| n > 1) + .map_or_else(available_parallelism, |n| n as usize) +} + +// Other platforms (e.g., windows), available_parallelism() directly. +#[cfg(not(unix))] +fn num_cpus_all() -> usize { + available_parallelism() +} + +/// In some cases, [`thread::available_parallelism`]() may return an Err +/// In this case, we will return 1 (like GNU) +fn available_parallelism() -> usize { + thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(env: HashMap, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd: PathBuf::from("."), + env, + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("nproc") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + #[test] + fn scope_env_omp_num_threads_forces_count() { + let env = HashMap::from([("OMP_NUM_THREADS".to_string(), "3".to_string())]); + let (code, stdout, stderr) = run_in(env, vec![]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "3\n", "")); + } + + #[test] + fn omp_thread_limit_caps_omp_num_threads() { + let env = HashMap::from([ + ("OMP_NUM_THREADS".to_string(), "64".to_string()), + ("OMP_THREAD_LIMIT".to_string(), "2".to_string()), + ]); + let (code, stdout, stderr) = run_in(env, vec![]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "2\n", "")); + } + + #[test] + fn all_prints_positive_integer_and_ignores_omp_num_threads() { + // --all reports hardware CPUs; OMP_NUM_THREADS must not force it. + let env = HashMap::from([("OMP_NUM_THREADS".to_string(), "0".to_string())]); + let (code, stdout, stderr) = run_in(env, vec!["--all"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let n: usize = stdout + .trim_end() + .parse() + .expect("--all output is an integer"); + assert!(n >= 1); + } + + #[test] + fn process_environment_is_not_consulted() { + // The variable exists only in the host process environment, not the + // scope map: only the un-patched `std::env::var` path would see it. + unsafe { std::env::set_var("OMP_NUM_THREADS", "1234") }; + let (code, stdout, stderr) = run_in(HashMap::new(), vec![]); + unsafe { std::env::remove_var("OMP_NUM_THREADS") }; + assert_eq!((code, stderr.as_str()), (0, "")); + assert_ne!(stdout, "1234\n"); + let n: usize = stdout.trim_end().parse().expect("output is an integer"); + assert!(n >= 1); + } + + #[test] + fn ignore_subtracts_and_floors_at_one() { + let env = HashMap::from([("OMP_NUM_THREADS".to_string(), "8".to_string())]); + let (code, stdout, _) = run_in(env, vec!["--ignore=3"]); + assert_eq!((code, stdout.as_str()), (0, "5\n")); + + let env = HashMap::from([("OMP_NUM_THREADS".to_string(), "2".to_string())]); + let (code, stdout, _) = run_in(env, vec!["--ignore=5"]); + assert_eq!((code, stdout.as_str()), (0, "1\n")); + } + + #[test] + fn invalid_ignore_value_is_an_error() { + let (code, stdout, stderr) = run_in(HashMap::new(), vec!["--ignore=bogus"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(stderr.contains("is not a valid number"), "stderr: {stderr}"); + } +} diff --git a/crates/vendor/uu-printenv/Cargo.toml b/crates/vendor/uu-printenv/Cargo.toml new file mode 100644 index 000000000..db7e8c291 --- /dev/null +++ b/crates/vendor/uu-printenv/Cargo.toml @@ -0,0 +1,21 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/printenv), patched to read +# the environment from the shell scope and route I/O through pi-uutils-ctx so +# it can run in-process as a shell builtin. See src/printenv.rs for the patch +# markers (`pi-uutils:` comments). +[package] +name = "uu_printenv" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "printenv ~ (uutils) display value of environment VAR (vendored + patched for in-process embedding)" + +[lib] +path = "src/printenv.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +uucore = { version = "0.8.0" } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-printenv/LICENSE b/crates/vendor/uu-printenv/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-printenv/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-printenv/src/printenv.rs b/crates/vendor/uu-printenv/src/printenv.rs new file mode 100644 index 000000000..5e0eac263 --- /dev/null +++ b/crates/vendor/uu-printenv/src/printenv.rs @@ -0,0 +1,243 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. The environment comes from the SCOPE, not the process: +// the no-argument dump iterates `pi_uutils_ctx::env_snapshot()` and named +// lookups go through `pi_uutils_ctx::var`, because the embedding shell's +// exported variables are not present in the host process environment. All +// output is routed through the context stdout, `translate!` strings are +// literalized, and the entry point no longer calls `std::process::exit`. + +use std::{ffi::OsString, io::Write}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use pi_uutils_ctx::format_usage; +use uucore::{error::UResult, line_ending::LineEnding}; + +static OPT_NULL: &str = "null"; + +static ARG_VARIABLES: &str = "variables"; + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match printenv_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + // pi-uutils: unset-variable failures surface as bare exit-code + // errors that render to an empty message (upstream prints nothing + // for them); don't emit a dangling "printenv: " prefix. + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "printenv: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn printenv_main(matches: &ArgMatches) -> UResult<()> { + let variables: Vec = matches + .get_many::(ARG_VARIABLES) + .map(|v| v.map(ToString::to_string).collect()) + .unwrap_or_default(); + + let separator = LineEnding::from_zero_flag(matches.get_flag(OPT_NULL)); + + if variables.is_empty() { + // pi-uutils: replacement for `uucore::display::print_all_env_vars` — + // dumps the scope environment map to the context stdout. + let mut stdout = pi_uutils_ctx::stdout(); + for (key, value) in pi_uutils_ctx::env_snapshot() { + write!(stdout, "{key}={value}{separator}")?; + } + stdout.flush()?; + return Ok(()); + } + + let mut error_found = false; + for env_var in variables { + // we silently ignore a=b as variable but we trigger an error + if env_var.contains('=') { + error_found = true; + continue; + } + // pi-uutils: look the variable up in the scope environment (upstream + // uses `std::env::var_os`) and write it to the context stdout. + if let Some(var) = pi_uutils_ctx::var(&env_var) { + let mut stdout = pi_uutils_ctx::stdout(); + write!(stdout, "{var}{separator}")?; + stdout.flush()?; + } else { + error_found = true; + } + } + + if error_found { Err(1.into()) } else { Ok(()) } +} + +pub fn uu_app() -> Command { + Command::new("printenv") + .version(uucore::crate_version!()) + .about( + "Display the values of the specified environment VARIABLE(s), or (with no VARIABLE) \ + display name and value pairs for them all.", + ) + .override_usage(format_usage("printenv [OPTION]... [VARIABLE]...")) + .infer_long_args(true) + .arg( + Arg::new(OPT_NULL) + .short('0') + .long(OPT_NULL) + .help("end each output line with 0 byte rather than newline") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(ARG_VARIABLES) + .action(ArgAction::Append) + .num_args(1..), + ) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_with_env(env: HashMap, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd: std::path::PathBuf::from("."), + env, + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("printenv") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + fn scope_env() -> HashMap { + HashMap::from([ + ("FOO".to_string(), "bar".to_string()), + ("BAZ".to_string(), "qux".to_string()), + ]) + } + + #[test] + fn named_variable_prints_scope_value() { + let (code, stdout, stderr) = run_with_env(scope_env(), vec!["FOO"]); + assert_eq!(code, 0); + assert_eq!(stdout, "bar\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn unset_variable_is_silent_failure() { + let (code, stdout, stderr) = run_with_env(scope_env(), vec!["NOPE"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "", "unset variables fail without a message"); + } + + #[test] + fn mixed_set_and_unset_prints_set_ones_and_fails() { + let (code, stdout, stderr) = run_with_env(scope_env(), vec!["FOO", "NOPE", "BAZ"]); + assert_eq!(code, 1); + assert_eq!(stdout, "bar\nqux\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn no_args_dumps_scope_env_not_process_env() { + // The host process certainly has PATH set; the scope env deliberately + // does not, so its absence proves the dump reads the scope map. + assert!(std::env::var_os("PATH").is_some()); + let (code, stdout, stderr) = run_with_env(scope_env(), vec![]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!(lines.len(), 2); + assert!(lines.contains(&"FOO=bar")); + assert!(lines.contains(&"BAZ=qux")); + assert!(!lines.iter().any(|l| l.starts_with("PATH="))); + } + + #[test] + fn null_flag_terminates_with_nul() { + let (code, stdout, _) = run_with_env(scope_env(), vec!["-0", "FOO"]); + assert_eq!((code, stdout.as_str()), (0, "bar\0")); + + let (code, stdout, _) = run_with_env(scope_env(), vec!["--null", "FOO", "BAZ"]); + assert_eq!((code, stdout.as_str()), (0, "bar\0qux\0")); + } + + #[test] + fn name_containing_equals_is_ignored_but_fails() { + let (code, stdout, stderr) = run_with_env(scope_env(), vec!["FOO=bar", "BAZ"]); + assert_eq!(code, 1); + assert_eq!(stdout, "qux\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_with_env(HashMap::new(), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("environment VARIABLE")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-readlink/Cargo.toml b/crates/vendor/uu-readlink/Cargo.toml new file mode 100644 index 000000000..e54321c8b --- /dev/null +++ b/crates/vendor/uu-readlink/Cargo.toml @@ -0,0 +1,22 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/readlink), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/readlink.rs +# for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_readlink" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "readlink ~ (uutils) display resolved path of PATHNAME (vendored + patched for in-process embedding)" + +[lib] +path = "src/readlink.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +uucore = { version = "0.8.0", features = ["fs"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-readlink/LICENSE b/crates/vendor/uu-readlink/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-readlink/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-readlink/src/readlink.rs b/crates/vendor/uu-readlink/src/readlink.rs new file mode 100644 index 000000000..813d4a458 --- /dev/null +++ b/crates/vendor/uu-readlink/src/readlink.rs @@ -0,0 +1,422 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) errno + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Every filesystem syscall resolves its path operand +// against the shell working directory via `pi_uutils_ctx::resolve` AT THE CALL +// SITE, while the original operands are kept for display/error messages (GNU +// prints operands as typed). All process-global stdio is routed through +// `pi_uutils_ctx`, `translate!` strings are literalized, POSIXLY_CORRECT is +// read from the scope environment, and the entry point no longer calls +// `std::process::exit`. + +use std::{ + ffi::OsString, + fs, + io::Write, + path::{Path, PathBuf}, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use pi_uutils_ctx::format_usage; +use uucore::{ + display::Quotable, + error::{FromIo, UResult, UUsageError}, + fs::{MissingHandling, ResolveMode, canonicalize}, + libc::EINVAL, + line_ending::LineEnding, +}; + +const OPT_CANONICALIZE: &str = "canonicalize"; +const OPT_CANONICALIZE_MISSING: &str = "canonicalize-missing"; +const OPT_CANONICALIZE_EXISTING: &str = "canonicalize-existing"; +const OPT_NO_NEWLINE: &str = "no-newline"; +const OPT_QUIET: &str = "quiet"; +const OPT_SILENT: &str = "silent"; +const OPT_VERBOSE: &str = "verbose"; +const OPT_ZERO: &str = "zero"; + +const ARG_FILES: &str = "files"; + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match readlink_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + // pi-uutils: silent failures surface as bare exit-code errors that + // render to an empty message (upstream prints nothing for them); + // don't emit a dangling "readlink: " prefix. + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "readlink: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn readlink_main(matches: &ArgMatches) -> UResult<()> { + let mut no_trailing_delimiter = matches.get_flag(OPT_NO_NEWLINE); + let use_zero = matches.get_flag(OPT_ZERO); + // pi-uutils: POSIXLY_CORRECT comes from the scope environment (the shell's + // exported variables), not the host process environment. + let verbose = matches.get_flag(OPT_VERBOSE) || pi_uutils_ctx::var("POSIXLY_CORRECT").is_some(); + + // GNU readlink -f/-e/-m follows symlinks first and then applies `..` (physical + // resolution). ResolveMode::Logical collapses `..` before following links, + // which yields the opposite order, so we choose Physical here for GNU + // compatibility. + let res_mode = if matches.get_flag(OPT_CANONICALIZE) + || matches.get_flag(OPT_CANONICALIZE_EXISTING) + || matches.get_flag(OPT_CANONICALIZE_MISSING) + { + ResolveMode::Physical + } else { + ResolveMode::None + }; + + let can_mode = if matches.get_flag(OPT_CANONICALIZE_EXISTING) { + MissingHandling::Existing + } else if matches.get_flag(OPT_CANONICALIZE_MISSING) { + MissingHandling::Missing + } else { + MissingHandling::Normal + }; + + let files: Vec = matches + .get_many::(ARG_FILES) + .map(|v| v.map(PathBuf::from).collect()) + .unwrap_or_default(); + + if files.is_empty() { + return Err(UUsageError::new(1, "missing operand".to_string())); + } + + if no_trailing_delimiter && files.len() > 1 { + let _ = writeln!( + pi_uutils_ctx::stderr(), + "readlink: ignoring --no-newline with multiple arguments" + ); + no_trailing_delimiter = false; + } + + let line_ending = if no_trailing_delimiter { + None + } else { + Some(LineEnding::from_zero_flag(use_zero)) + }; + + for p in &files { + // pi-uutils: resolve the operand against the shell working directory; + // `p` is kept for display. Resolving before `canonicalize` also keeps + // uucore's internal `env::current_dir()` fallback from being consulted. + let resolved = pi_uutils_ctx::resolve(p); + let path_result = if res_mode == ResolveMode::None { + fs::read_link(&resolved) + } else { + canonicalize(&resolved, can_mode, res_mode) + }; + + match path_result { + Ok(path) => { + show(&path, line_ending)?; + }, + Err(err) => { + if !verbose { + return Err(1.into()); + } + + let message = if err.raw_os_error() == Some(EINVAL) { + format!("{}: Invalid argument", p.maybe_quote()) + } else { + err.map_err_context(|| p.maybe_quote().to_string()) + .to_string() + }; + let _ = writeln!(pi_uutils_ctx::stderr(), "readlink: {message}"); + return Err(1.into()); + }, + } + } + Ok(()) +} + +pub fn uu_app() -> Command { + Command::new("readlink") + .version(uucore::crate_version!()) + .about("Print value of a symbolic link or canonical file name.") + .override_usage(format_usage("readlink [OPTION]... [FILE]...")) + .infer_long_args(true) + .arg( + Arg::new(OPT_CANONICALIZE) + .short('f') + .long(OPT_CANONICALIZE) + .help( + "canonicalize by following every symlink in every component of the given name \ + recursively; all but the last component must exist", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_CANONICALIZE_EXISTING) + .short('e') + .long("canonicalize-existing") + .help( + "canonicalize by following every symlink in every component of the given name \ + recursively, all components must exist", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_CANONICALIZE_MISSING) + .short('m') + .long(OPT_CANONICALIZE_MISSING) + .help( + "canonicalize by following every symlink in every component of the given name \ + recursively, without requirements on components existence", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_NO_NEWLINE) + .short('n') + .long(OPT_NO_NEWLINE) + .help("do not output the trailing delimiter") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_QUIET) + .short('q') + .long(OPT_QUIET) + .help("suppress most error messages") + .overrides_with_all([OPT_QUIET, OPT_SILENT, OPT_VERBOSE]) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_SILENT) + .short('s') + .long(OPT_SILENT) + .help("suppress most error messages") + .overrides_with_all([OPT_QUIET, OPT_SILENT, OPT_VERBOSE]) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_VERBOSE) + .short('v') + .long(OPT_VERBOSE) + .help("report error message") + .overrides_with_all([OPT_QUIET, OPT_SILENT, OPT_VERBOSE]) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_ZERO) + .short('z') + .long(OPT_ZERO) + .help("separate output with NUL rather than newline") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(ARG_FILES) + .action(ArgAction::Append) + .value_parser(clap::value_parser!(OsString)) + .value_hint(clap::ValueHint::AnyPath), + ) +} + +/// pi-uutils: replacement for upstream's `show` — writes the resolved path +/// bytes verbatim to the context stdout instead of the process stdout. +fn show(path: &Path, line_ending: Option) -> UResult<()> { + let mut out = pi_uutils_ctx::stdout(); + out.write_all(uucore::os_str_as_bytes(path.as_os_str())?)?; + if let Some(line_ending) = line_ending { + write!(out, "{line_ending}")?; + } + out.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("readlink") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var, + /// which -f/-e/-m resolution would otherwise expand mid-assertion). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + #[cfg(unix)] + #[test] + fn resolves_relative_operand_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + std::os::unix::fs::symlink("target-file", root.join("link")).unwrap(); + + // Relative operand + scope cwd differing from the process cwd: only the + // call-site `pi_uutils_ctx::resolve` patch makes this find the link. + let (code, stdout, stderr) = run_in(root, vec!["link"]); + assert_eq!(code, 0); + assert_eq!(stdout, "target-file\n"); + assert_eq!(stderr, ""); + } + + #[cfg(unix)] + #[test] + fn canonicalize_follows_symlink_to_absolute_path() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("target"), b"x").unwrap(); + std::os::unix::fs::symlink("target", root.join("link")).unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-f", "link"]); + assert_eq!(code, 0); + assert_eq!(stdout, format!("{}\n", root.join("target").display())); + assert_eq!(stderr, ""); + } + + #[test] + fn canonicalize_missing_builds_path_from_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-m", "missing/sub"]); + assert_eq!(code, 0); + assert_eq!(stdout, format!("{}\n", root.join("missing").join("sub").display())); + assert_eq!(stderr, ""); + } + + #[cfg(unix)] + #[test] + fn canonicalize_existing_fails_silently_on_missing_final_component() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root, vec!["-e", "missing"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "", "non-verbose failures print nothing"); + } + + #[cfg(unix)] + #[test] + fn non_symlink_is_silent_failure_by_default_and_einval_with_verbose() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("plain"), b"x").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["plain"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (1, "", "")); + + let (code, stdout, stderr) = run_in(root, vec!["-v", "plain"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "readlink: plain: Invalid argument\n"); + } + + #[cfg(unix)] + #[test] + fn no_newline_with_multiple_args_warns_and_keeps_delimiter() { + let (_dir, root) = canonical_tempdir(); + std::os::unix::fs::symlink("a", root.join("l1")).unwrap(); + std::os::unix::fs::symlink("b", root.join("l2")).unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["-n", "l1", "l2"]); + assert_eq!(code, 0); + assert_eq!(stdout, "a\nb\n"); + assert_eq!(stderr, "readlink: ignoring --no-newline with multiple arguments\n"); + } + + #[cfg(unix)] + #[test] + fn zero_terminates_with_nul_and_no_newline_drops_delimiter() { + let (_dir, root) = canonical_tempdir(); + std::os::unix::fs::symlink("a", root.join("l1")).unwrap(); + + let (code, stdout, _) = run_in(root.clone(), vec!["-z", "l1"]); + assert_eq!((code, stdout.as_str()), (0, "a\0")); + + let (code, stdout, _) = run_in(root, vec!["-n", "l1"]); + assert_eq!((code, stdout.as_str()), (0, "a")); + } + + #[test] + fn missing_operand_is_usage_error() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec![]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(stderr.contains("missing operand")); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("canonical file name")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-realpath/Cargo.toml b/crates/vendor/uu-realpath/Cargo.toml new file mode 100644 index 000000000..0a3954649 --- /dev/null +++ b/crates/vendor/uu-realpath/Cargo.toml @@ -0,0 +1,22 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/realpath), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/realpath.rs +# for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_realpath" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "realpath ~ (uutils) display resolved absolute path of PATHNAME (vendored + patched for in-process embedding)" + +[lib] +path = "src/realpath.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +uucore = { version = "0.8.0", features = ["fs"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-realpath/LICENSE b/crates/vendor/uu-realpath/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-realpath/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-realpath/src/realpath.rs b/crates/vendor/uu-realpath/src/realpath.rs new file mode 100644 index 000000000..061f67dfe --- /dev/null +++ b/crates/vendor/uu-realpath/src/realpath.rs @@ -0,0 +1,568 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) retcode + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Every filesystem syscall resolves its path operand +// against the shell working directory via `pi_uutils_ctx::resolve` AT THE CALL +// SITE (FILE operands and the --relative-to/--relative-base option paths), +// while the original operands are kept for display/error messages (GNU prints +// operands as typed). All process-global stdio is routed through +// `pi_uutils_ctx`, `translate!` strings are literalized, `show_if_err!` is +// replaced by a context-stderr write plus `pi_uutils_ctx::set_exit_code`, and +// the entry point no longer calls `std::process::exit`. + +use std::{ + ffi::{OsStr, OsString}, + io::Write, + path::{Path, PathBuf}, +}; + +use clap::{ + Arg, ArgAction, ArgMatches, Command, + builder::{TypedValueParser, ValueParserFactory}, +}; +use pi_uutils_ctx::format_usage; +use uucore::{ + display::Quotable, + error::{FromIo, UResult}, + fs::{MissingHandling, ResolveMode, canonicalize, make_path_relative_to}, + line_ending::LineEnding, +}; + +const OPT_QUIET: &str = "quiet"; +const OPT_STRIP: &str = "strip"; +const OPT_ZERO: &str = "zero"; +const OPT_PHYSICAL: &str = "physical"; +const OPT_LOGICAL: &str = "logical"; +const OPT_CANONICALIZE_MISSING: &str = "canonicalize-missing"; +const OPT_CANONICALIZE: &str = "canonicalize"; +const OPT_CANONICALIZE_EXISTING: &str = "canonicalize-existing"; +const OPT_RELATIVE_TO: &str = "relative-to"; +const OPT_RELATIVE_BASE: &str = "relative-base"; + +const ARG_FILES: &str = "files"; + +/// Custom parser that validates `OsString` is not empty +#[derive(Clone, Debug)] +struct NonEmptyOsStringParser; + +impl TypedValueParser for NonEmptyOsStringParser { + type Value = OsString; + + fn parse_ref( + &self, + _cmd: &Command, + _arg: Option<&Arg>, + value: &OsStr, + ) -> Result { + if value.is_empty() { + let mut err = clap::Error::new(clap::error::ErrorKind::ValueValidation); + err.insert( + clap::error::ContextKind::Custom, + // pi-uutils: literalized `translate!("realpath-invalid-empty-operand")` + clap::error::ContextValue::String("invalid operand: empty string".to_string()), + ); + return Err(err); + } + Ok(value.to_os_string()) + } +} + +impl ValueParserFactory for NonEmptyOsStringParser { + type Parser = Self; + + fn value_parser() -> Self::Parser { + Self + } +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match realpath_main(&matches) { + // pi-uutils: per-file failures accumulate their exit code via + // `pi_uutils_ctx::set_exit_code` (upstream's `show!` machinery). + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "realpath: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn realpath_main(matches: &ArgMatches) -> UResult<()> { + /* the list of files */ + + let paths: Vec = matches + .get_many::(ARG_FILES) + .unwrap() + .map(PathBuf::from) + .collect(); + + let strip = matches.get_flag(OPT_STRIP); + let line_ending = LineEnding::from_zero_flag(matches.get_flag(OPT_ZERO)); + let quiet = matches.get_flag(OPT_QUIET); + let logical = matches.get_flag(OPT_LOGICAL); + let can_mode = if matches.get_flag(OPT_CANONICALIZE_MISSING) { + MissingHandling::Missing + } else if matches.get_flag(OPT_CANONICALIZE_EXISTING) { + // -e: all components must exist + // Despite the name, MissingHandling::Existing requires all components to exist + MissingHandling::Existing + } else { + // Default behavior (same as -E): all but last component must exist + // MissingHandling::Normal allows the final component to not exist + MissingHandling::Normal + }; + let resolve_mode = if strip { + ResolveMode::None + } else if logical { + ResolveMode::Logical + } else { + ResolveMode::Physical + }; + let (relative_to, relative_base) = prepare_relative_options(matches, can_mode, resolve_mode)?; + for path in &paths { + let result = resolve_path( + path, + line_ending, + resolve_mode, + can_mode, + relative_to.as_deref(), + relative_base.as_deref(), + ); + if !quiet { + // pi-uutils: replacement for `show_if_err!` — report the error on + // the context stderr and record the exit code, then keep + // processing the remaining operands (upstream continue semantics). + if let Err(err) = result.map_err_context(|| path.maybe_quote().to_string()) { + let _ = writeln!(pi_uutils_ctx::stderr(), "realpath: {err}"); + pi_uutils_ctx::set_exit_code(err.code()); + } + } + } + // Although we return `Ok`, it is possible that a call to + // `show!()` above has set the exit code for the program to a + // non-zero integer. + Ok(()) +} + +pub fn uu_app() -> Command { + Command::new("realpath") + .version(uucore::crate_version!()) + .about("Print the resolved path") + .override_usage(format_usage("realpath [OPTION]... FILE...")) + .infer_long_args(true) + .arg( + Arg::new(OPT_QUIET) + .short('q') + .long(OPT_QUIET) + .help("Do not print warnings for invalid paths") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_STRIP) + .short('s') + .long(OPT_STRIP) + .visible_alias("no-symlinks") + .help("Only strip '.' and '..' components, but don't resolve symbolic links") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_ZERO) + .short('z') + .long(OPT_ZERO) + .help("Separate output filenames with \\0 rather than newline") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_LOGICAL) + .short('L') + .long(OPT_LOGICAL) + .help("resolve '..' components before symlinks") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_PHYSICAL) + .short('P') + .long(OPT_PHYSICAL) + .overrides_with_all([OPT_STRIP, OPT_LOGICAL]) + .help("resolve symlinks as encountered (default)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_CANONICALIZE) + .short('E') + .long(OPT_CANONICALIZE) + .overrides_with_all([OPT_CANONICALIZE_EXISTING, OPT_CANONICALIZE_MISSING]) + .help("all but the last component must exist (default)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_CANONICALIZE_EXISTING) + .short('e') + .long(OPT_CANONICALIZE_EXISTING) + .overrides_with_all([OPT_CANONICALIZE, OPT_CANONICALIZE_MISSING]) + .help( + "canonicalize by following every symlink in every component of the given name \ + recursively, all components must exist", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_CANONICALIZE_MISSING) + .short('m') + .long(OPT_CANONICALIZE_MISSING) + .overrides_with_all([OPT_CANONICALIZE, OPT_CANONICALIZE_EXISTING]) + .help( + "canonicalize by following every symlink in every component of the given name \ + recursively, without requirements on components existence", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_RELATIVE_TO) + .long(OPT_RELATIVE_TO) + .value_name("DIR") + .value_parser(NonEmptyOsStringParser) + .help("print the resolved path relative to DIR"), + ) + .arg( + Arg::new(OPT_RELATIVE_BASE) + .long(OPT_RELATIVE_BASE) + .value_name("DIR") + .value_parser(NonEmptyOsStringParser) + .help("print absolute paths unless paths below DIR"), + ) + .arg( + Arg::new(ARG_FILES) + .action(ArgAction::Append) + .required(true) + .value_parser(NonEmptyOsStringParser) + .value_hint(clap::ValueHint::AnyPath), + ) +} + +/// Prepare `--relative-to` and `--relative-base` options. +/// Convert them to their absolute values. +/// Check if `--relative-to` is a descendant of `--relative-base`, +/// otherwise nullify their value. +fn prepare_relative_options( + matches: &ArgMatches, + can_mode: MissingHandling, + resolve_mode: ResolveMode, +) -> UResult<(Option, Option)> { + let relative_to = matches + .get_one::(OPT_RELATIVE_TO) + .map(PathBuf::from); + let relative_base = matches + .get_one::(OPT_RELATIVE_BASE) + .map(PathBuf::from); + let relative_to = canonicalize_relative_option(relative_to, can_mode, resolve_mode)?; + let relative_base = canonicalize_relative_option(relative_base, can_mode, resolve_mode)?; + if let (Some(base), Some(to)) = (relative_base.as_deref(), relative_to.as_deref()) + && !to.starts_with(base) + { + return Ok((None, None)); + } + Ok((relative_to, relative_base)) +} + +/// Prepare single `relative-*` option. +fn canonicalize_relative_option( + relative: Option, + can_mode: MissingHandling, + resolve_mode: ResolveMode, +) -> UResult> { + Ok(match relative { + None => None, + Some(p) => Some( + canonicalize_relative(&p, can_mode, resolve_mode) + .map_err_context(|| p.maybe_quote().to_string())?, + ), + }) +} + +/// Make `relative-to` or `relative-base` path values absolute. +/// +/// # Errors +/// +/// If the given path is not a directory the function returns an error. +/// If some parts of the file don't exist, or symlinks make loops, or +/// some other IO error happens, the function returns error, too. +fn canonicalize_relative( + r: &Path, + can_mode: MissingHandling, + resolve: ResolveMode, +) -> std::io::Result { + // pi-uutils: resolve the option path against the shell working directory; + // `r` is kept by the caller for display. Resolving before `canonicalize` + // also keeps uucore's internal `env::current_dir()` fallback from being + // consulted. + let abs = canonicalize(pi_uutils_ctx::resolve(r), can_mode, resolve)?; + if can_mode == MissingHandling::Existing && !abs.is_dir() { + abs.read_dir()?; // raise not a directory error + } + Ok(abs) +} + +/// Resolve a path to an absolute form and print it. +/// +/// If `relative_to` and/or `relative_base` is given +/// the path is printed in a relative form to one of this options. +/// See the details in `process_relative` function. +/// If `zero` is `true`, then this function +/// prints the path followed by the null byte (`'\0'`) instead of a +/// newline character (`'\n'`). +/// +/// # Errors +/// +/// This function returns an error if there is a problem resolving +/// symbolic links. +fn resolve_path( + p: &Path, + line_ending: LineEnding, + resolve: ResolveMode, + can_mode: MissingHandling, + relative_to: Option<&Path>, + relative_base: Option<&Path>, +) -> std::io::Result<()> { + // pi-uutils: resolve the operand against the shell working directory; `p` + // is kept by the caller for display. Resolving before `canonicalize` also + // keeps uucore's internal `env::current_dir()` fallback from being + // consulted. + let abs = canonicalize(pi_uutils_ctx::resolve(p), can_mode, resolve)?; + + let abs = process_relative(abs, relative_base, relative_to); + + // pi-uutils: replacement for `print_verbatim` + process stdout — writes + // the resolved path bytes verbatim to the context stdout. + let mut out = pi_uutils_ctx::stdout(); + out.write_all( + uucore::os_str_as_bytes(abs.as_os_str()).map_err(|e| std::io::Error::other(e.to_string()))?, + )?; + out.write_all(&[line_ending.into()])?; + out.flush()?; + Ok(()) +} + +/// Conditionally converts an absolute path to a relative form, +/// according to the rules: +/// 1. if only `relative_to` is given, the result is relative to `relative_to` +/// 2. if only `relative_base` is given, it checks whether given `path` is a +/// descendant of `relative_base`, on success the result is relative to +/// `relative_base`, otherwise the result is the given `path` +/// 3. if both `relative_to` and `relative_base` are given, the result is +/// relative to `relative_to` if `path` is a descendant of `relative_base`, +/// otherwise the result is `path` +/// +/// For more information see +/// +fn process_relative( + path: PathBuf, + relative_base: Option<&Path>, + relative_to: Option<&Path>, +) -> PathBuf { + if let Some(base) = relative_base { + if path.starts_with(base) { + make_path_relative_to(path, relative_to.unwrap_or(base)) + } else { + path + } + } else if let Some(to) = relative_to { + make_path_relative_to(path, to) + } else { + path + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, fs, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("realpath") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var, + /// which canonicalization would otherwise expand mid-assertion). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + #[cfg(unix)] + #[test] + fn resolves_relative_operand_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("target"), b"x").unwrap(); + std::os::unix::fs::symlink("target", root.join("link")).unwrap(); + + // Relative operand + scope cwd differing from the process cwd: only + // the call-site `pi_uutils_ctx::resolve` patch makes this find the + // symlink and print its canonical target. + let (code, stdout, stderr) = run_in(root.clone(), vec!["link"]); + assert_eq!(code, 0); + assert_eq!(stdout, format!("{}\n", root.join("target").display())); + assert_eq!(stderr, ""); + } + + #[test] + fn canonicalize_missing_builds_path_from_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-m", "missing/sub"]); + assert_eq!(code, 0); + assert_eq!(stdout, format!("{}\n", root.join("missing").join("sub").display())); + assert_eq!(stderr, ""); + } + + #[test] + fn relative_to_option_resolves_against_scope_cwd_and_relativizes_output() { + let (_dir, root) = canonical_tempdir(); + fs::create_dir(root.join("sub")).unwrap(); + fs::write(root.join("sub").join("file"), b"x").unwrap(); + + // Both the operand and the (relative) --relative-to directory resolve + // against the scope cwd. + let (code, stdout, stderr) = run_in(root, vec!["--relative-to", "sub", "sub/file"]); + assert_eq!(code, 0); + assert_eq!(stdout, "file\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn zero_flag_terminates_with_nul() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("f"), b"x").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-z", "f"]); + assert_eq!(code, 0); + assert_eq!(stdout, format!("{}\0", root.join("f").display())); + assert_eq!(stderr, ""); + } + + #[test] + fn nonexistent_operand_errors_but_later_operands_still_process() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("f"), b"x").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["missing/x", "f"]); + assert_eq!(code, 1); + assert_eq!(stdout, format!("{}\n", root.join("f").display())); + assert!(stderr.contains("realpath: missing/x"), "stderr: {stderr}"); + assert!(stderr.contains("No such file"), "stderr: {stderr}"); + } + + #[test] + fn quiet_suppresses_error_messages() { + let (_dir, root) = canonical_tempdir(); + + // Upstream drops the per-file result entirely under -q (the error is + // neither printed nor accumulated into the exit code). + let (code, stdout, stderr) = run_in(root, vec!["-q", "missing/x"]); + assert_eq!(code, 0); + assert_eq!(stdout, ""); + assert_eq!(stderr, ""); + } + + #[cfg(unix)] + #[test] + fn strip_keeps_symlinks_unresolved() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("target"), b"x").unwrap(); + std::os::unix::fs::symlink("target", root.join("link")).unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-s", "link"]); + assert_eq!(code, 0); + assert_eq!(stdout, format!("{}\n", root.join("link").display())); + assert_eq!(stderr, ""); + } + + #[test] + fn empty_operand_is_rejected() { + // The NonEmptyOsStringParser turns "" into a clap parse error (rendered + // by clap's default renderer since the uucore localization layer is + // patched out) instead of a filesystem lookup. + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec![""]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(stderr.contains("invalid value"), "stderr: {stderr}"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("Print the resolved path")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-seq/Cargo.toml b/crates/vendor/uu-seq/Cargo.toml new file mode 100644 index 000000000..f301f621d --- /dev/null +++ b/crates/vendor/uu-seq/Cargo.toml @@ -0,0 +1,32 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/seq), patched to route I/O +# through pi-uutils-ctx so it can run in-process as a shell builtin. See +# src/seq.rs for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_seq" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "seq ~ (uutils) display a sequence of numbers (vendored + patched for in-process embedding)" + +[lib] +path = "src/seq.rs" + +[dependencies] +bigdecimal = "0.4" +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +num-bigint = "0.4" +num-traits = "0.2" +thiserror = "2.0.3" +# pi-uutils: upstream also enables "signals" (SIGPIPE probing) — dropped, the +# in-process builtin has no process-global signal handling. +uucore = { version = "0.8.0", features = [ + "extendedbigdecimal", + "fast-inc", + "format", + "parser", + "quoting-style", +] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-seq/LICENSE b/crates/vendor/uu-seq/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-seq/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-seq/src/error.rs b/crates/vendor/uu-seq/src/error.rs new file mode 100644 index 000000000..417dc0960 --- /dev/null +++ b/crates/vendor/uu-seq/src/error.rs @@ -0,0 +1,57 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore numberparse +//! Errors returned by seq. + +// pi-uutils: `translate!` message lookups are literalized with the en-US +// strings from upstream's locales/en-US.ftl. + +use thiserror::Error; +use uucore::{display::Quotable, error::UError}; + +use crate::numberparse::ParseNumberError; + +#[derive(Debug, Error)] +pub enum SeqError { + /// An error parsing the input arguments. + /// + /// The parameters are the [`String`] argument as read from the + /// command line and the underlying parsing error itself. + #[error("invalid {} argument: {}", parse_error_type(.1), .0.quote())] + ParseError(String, ParseNumberError), + + /// The increment argument was zero, which is not allowed. + /// + /// The parameter is the increment argument as a [`String`] as read + /// from the command line. + #[error("invalid Zero increment value: {}", .0.quote())] + ZeroIncrement(String), + + /// No arguments were passed to this function, 1 or more is required + #[error("missing operand")] + NoArguments, + + /// Both a format and equal width where passed to seq + #[error("format string may not be specified when printing equal width strings")] + FormatAndEqualWidth, +} + +fn parse_error_type(e: &ParseNumberError) -> &'static str { + match e { + ParseNumberError::Float => "floating point", + ParseNumberError::Nan => "'not-a-number'", + } +} + +impl UError for SeqError { + /// Always return 1. + fn code(&self) -> i32 { + 1 + } + + fn usage(&self) -> bool { + true + } +} diff --git a/crates/vendor/uu-seq/src/number.rs b/crates/vendor/uu-seq/src/number.rs new file mode 100644 index 000000000..caa530b55 --- /dev/null +++ b/crates/vendor/uu-seq/src/number.rs @@ -0,0 +1,52 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore extendedbigdecimal +use num_traits::Zero; +use uucore::extendedbigdecimal::ExtendedBigDecimal; + +/// A number with a specified number of integer and fractional digits. +/// +/// This struct can be used to represent a number along with information +/// on how many significant digits to use when displaying the number. +/// The [`PreciseNumber::num_integral_digits`] field also includes the width +/// needed to display the "-" character for a negative number. +/// [`PreciseNumber::num_fractional_digits`] provides the number of decimal +/// digits after the decimal point (a.k.a. precision), or None if that number +/// cannot intuitively be obtained (i.e. hexadecimal floats). +/// Note: Those 2 fields should not necessarily be interpreted literally, but as +/// matching GNU `seq` behavior: the exact way of guessing desired precision +/// from user input is a matter of interpretation. +/// +/// You can get an instance of this struct by calling [`str::parse`]. +#[derive(Debug)] +pub struct PreciseNumber { + pub number: ExtendedBigDecimal, + pub num_integral_digits: usize, + pub num_fractional_digits: Option, +} + +impl PreciseNumber { + // pi-uutils: upstream's unused `new` constructor (only reachable from the + // fuzzing harness) is dropped to keep the vendored crate warning-free. + + pub fn one() -> Self { + // We would like to implement `num_traits::One`, but it requires + // a multiplication implementation, and we don't want to + // implement that here. + Self { + number: ExtendedBigDecimal::one(), + num_integral_digits: 1, + num_fractional_digits: Some(0), + } + } + + /// Decide whether this number is zero (either positive or negative). + pub fn is_zero(&self) -> bool { + // We would like to implement `num_traits::Zero`, but it + // requires an addition implementation, and we don't want to + // implement that here. + self.number.is_zero() + } +} diff --git a/crates/vendor/uu-seq/src/numberparse.rs b/crates/vendor/uu-seq/src/numberparse.rs new file mode 100644 index 000000000..74b832d4d --- /dev/null +++ b/crates/vendor/uu-seq/src/numberparse.rs @@ -0,0 +1,351 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore extendedbigdecimal bigdecimal numberparse +// hexadecimalfloat +//! Parsing numbers for use in `seq`. +//! +//! This module provides an implementation of [`FromStr`] for the +//! [`PreciseNumber`] struct. +use std::str::FromStr; + +use uucore::{ + extendedbigdecimal::ExtendedBigDecimal, + parser::num_parser::{ExtendedParser, ExtendedParserError}, +}; + +use crate::number::PreciseNumber; + +/// An error returned when parsing a number fails. +#[derive(Debug, PartialEq, Eq)] +pub enum ParseNumberError { + Float, + Nan, +} + +/// Compute the number of integral and fractional digits in input string, +/// and wrap the result in a PreciseNumber. +/// We know that the string has already been parsed correctly, so we don't +/// need to be too careful. +fn compute_num_digits(input: &str, ebd: ExtendedBigDecimal) -> PreciseNumber { + let input = input.to_lowercase(); + let input = input.trim_start(); + + // Leading + is ignored for this. + let input = input.strip_prefix('+').unwrap_or(input); + + // Integral digits for any hex number is ill-defined (0 is fine as an output) + // Fractional digits for an floating hex number is ill-defined, return None + // as we'll totally ignore that number for precision computations. + // Still return 0 for hex integers though. + if input.starts_with("0x") || input.starts_with("-0x") { + return PreciseNumber { + number: ebd, + num_integral_digits: 0, + num_fractional_digits: if input.contains('.') || input.contains('p') { + None + } else { + Some(0) + }, + }; + } + + // Split the exponent part, if any + let parts: Vec<&str> = input.split('e').collect(); + debug_assert!(parts.len() <= 2); + + // Count all the digits up to `.`, `-` sign is included. + let (mut int_digits, mut frac_digits) = match parts[0].find('.') { + Some(i) => { + // Cover special case .X and -.X where we behave as if there was a leading 0: + // 0.X, -0.X. + let int_digits = match i { + 0 => 1, + 1 if parts[0].starts_with('-') => 2, + _ => i, + }; + + (int_digits, parts[0].len() - i - 1) + }, + None => (parts[0].len(), 0), + }; + + // If there is an exponent, reparse that (yes this is not optimal, + // but we can't necessarily exactly recover that from the parsed number). + if parts.len() == 2 { + let exp = parts[1].parse::().unwrap_or(0); + // For positive exponents, effectively expand the number. Ignore negative + // exponents. Also ignore overflowed exponents (unwrap_or(0)). + if exp > 0 { + int_digits += exp.try_into().unwrap_or(0); + } + frac_digits = if exp < frac_digits as i64 { + // Subtract from i128 to avoid any overflow + (frac_digits as i128 - exp as i128).try_into().unwrap_or(0) + } else { + 0 + } + } + + PreciseNumber { + number: ebd, + num_integral_digits: int_digits, + num_fractional_digits: Some(frac_digits), + } +} + +// Note: We could also have provided an `ExtendedParser` implementation for +// PreciseNumber, but we want a simpler custom error. +impl FromStr for PreciseNumber { + type Err = ParseNumberError; + + fn from_str(input: &str) -> Result { + let ebd = match ExtendedBigDecimal::extended_parse(input) { + Ok(ebd) => match ebd { + // Handle special values + ExtendedBigDecimal::BigDecimal(_) | ExtendedBigDecimal::MinusZero => { + // TODO: GNU `seq` treats small numbers < 1e-4950 as 0, we could do the same + // to avoid printing senselessly small numbers. + ebd + }, + ExtendedBigDecimal::Infinity | ExtendedBigDecimal::MinusInfinity => { + return Ok(Self { + number: ebd, + num_integral_digits: 0, + num_fractional_digits: Some(0), + }); + }, + ExtendedBigDecimal::Nan | ExtendedBigDecimal::MinusNan => { + return Err(ParseNumberError::Nan); + }, + }, + Err(ExtendedParserError::Underflow(ebd)) => ebd, // Treat underflow as 0 + Err(_) => return Err(ParseNumberError::Float), + }; + + Ok(compute_num_digits(input, ebd)) + } +} + +#[cfg(test)] +mod tests { + use bigdecimal::BigDecimal; + use uucore::extendedbigdecimal::ExtendedBigDecimal; + + use crate::{number::PreciseNumber, numberparse::ParseNumberError}; + + /// Convenience function for parsing a [`Number`] and unwrapping. + fn parse(s: &str) -> ExtendedBigDecimal { + s.parse::().unwrap().number + } + + /// Convenience function for getting the number of integral digits. + fn num_integral_digits(s: &str) -> usize { + s.parse::().unwrap().num_integral_digits + } + + /// Convenience function for getting the number of fractional digits. + fn num_fractional_digits(s: &str) -> usize { + s.parse::() + .unwrap() + .num_fractional_digits + .unwrap() + } + + /// Convenience function for making sure the number of fractional digits is + /// "None" + fn num_fractional_digits_is_none(s: &str) -> bool { + s.parse::() + .unwrap() + .num_fractional_digits + .is_none() + } + + #[test] + fn test_parse_minus_zero_int() { + assert_eq!(parse("-0e0"), ExtendedBigDecimal::MinusZero); + assert_eq!(parse("-0e-0"), ExtendedBigDecimal::MinusZero); + assert_eq!(parse("-0e1"), ExtendedBigDecimal::MinusZero); + assert_eq!(parse("-0e+1"), ExtendedBigDecimal::MinusZero); + assert_eq!(parse("-0.0e1"), ExtendedBigDecimal::MinusZero); + assert_eq!(parse("-0x0"), ExtendedBigDecimal::MinusZero); + } + + #[test] + fn test_parse_minus_zero_float() { + assert_eq!(parse("-0.0"), ExtendedBigDecimal::MinusZero); + assert_eq!(parse("-0e-1"), ExtendedBigDecimal::MinusZero); + assert_eq!(parse("-0.0e-1"), ExtendedBigDecimal::MinusZero); + } + + #[test] + fn test_parse_big_int() { + assert_eq!(parse("0"), ExtendedBigDecimal::zero()); + assert_eq!(parse("0.1e1"), ExtendedBigDecimal::one()); + assert_eq!(parse("0.1E1"), ExtendedBigDecimal::one()); + assert_eq!( + parse("1.0e1"), + ExtendedBigDecimal::BigDecimal("10".parse::().unwrap()) + ); + } + + #[test] + fn test_parse_hexadecimal_big_int() { + assert_eq!(parse("0x0"), ExtendedBigDecimal::zero()); + assert_eq!( + parse("0x10"), + ExtendedBigDecimal::BigDecimal("16".parse::().unwrap()) + ); + } + + #[test] + fn test_parse_big_decimal() { + assert_eq!( + parse("0.0"), + ExtendedBigDecimal::BigDecimal("0.0".parse::().unwrap()) + ); + assert_eq!(parse(".0"), ExtendedBigDecimal::BigDecimal("0.0".parse::().unwrap())); + assert_eq!( + parse("1.0"), + ExtendedBigDecimal::BigDecimal("1.0".parse::().unwrap()) + ); + assert_eq!( + parse("10e-1"), + ExtendedBigDecimal::BigDecimal("1.0".parse::().unwrap()) + ); + assert_eq!( + parse("-1e-3"), + ExtendedBigDecimal::BigDecimal("-0.001".parse::().unwrap()) + ); + } + + #[test] + fn test_parse_inf() { + assert_eq!(parse("inf"), ExtendedBigDecimal::Infinity); + assert_eq!(parse("infinity"), ExtendedBigDecimal::Infinity); + assert_eq!(parse("+inf"), ExtendedBigDecimal::Infinity); + assert_eq!(parse("+infinity"), ExtendedBigDecimal::Infinity); + assert_eq!(parse("-inf"), ExtendedBigDecimal::MinusInfinity); + assert_eq!(parse("-infinity"), ExtendedBigDecimal::MinusInfinity); + } + + #[test] + fn test_parse_invalid_float() { + assert_eq!("1.2.3".parse::().unwrap_err(), ParseNumberError::Float); + assert_eq!("1e2e3".parse::().unwrap_err(), ParseNumberError::Float); + assert_eq!("1e2.3".parse::().unwrap_err(), ParseNumberError::Float); + assert_eq!("-+-1".parse::().unwrap_err(), ParseNumberError::Float); + } + + #[test] + fn test_parse_invalid_hex() { + assert_eq!("0xg".parse::().unwrap_err(), ParseNumberError::Float); + } + + #[test] + fn test_parse_invalid_nan() { + assert_eq!("nan".parse::().unwrap_err(), ParseNumberError::Nan); + assert_eq!("NAN".parse::().unwrap_err(), ParseNumberError::Nan); + assert_eq!("NaN".parse::().unwrap_err(), ParseNumberError::Nan); + assert_eq!("nAn".parse::().unwrap_err(), ParseNumberError::Nan); + assert_eq!("-nan".parse::().unwrap_err(), ParseNumberError::Nan); + } + + #[test] + #[allow(clippy::cognitive_complexity)] + fn test_num_integral_digits() { + // no decimal, no exponent + assert_eq!(num_integral_digits("123"), 3); + // decimal, no exponent + assert_eq!(num_integral_digits("123.45"), 3); + assert_eq!(num_integral_digits("-0.1"), 2); + assert_eq!(num_integral_digits("-.1"), 2); + // exponent, no decimal + assert_eq!(num_integral_digits("123e4"), 3 + 4); + assert_eq!(num_integral_digits("123e-4"), 3); + assert_eq!(num_integral_digits("-1e-3"), 2); + // decimal and exponent + assert_eq!(num_integral_digits("123.45e6"), 3 + 6); + assert_eq!(num_integral_digits("123.45e-6"), 3); + assert_eq!(num_integral_digits("123.45e-1"), 3); + assert_eq!(num_integral_digits("-0.1e0"), 2); + assert_eq!(num_integral_digits("-0.1e2"), 4); + assert_eq!(num_integral_digits("-.1e0"), 2); + assert_eq!(num_integral_digits("-.1e2"), 4); + assert_eq!(num_integral_digits("-1.e-3"), 2); + assert_eq!(num_integral_digits("-1.0e-4"), 2); + // minus zero int + assert_eq!(num_integral_digits("-0e0"), 2); + assert_eq!(num_integral_digits("-0e-0"), 2); + assert_eq!(num_integral_digits("-0e1"), 3); + assert_eq!(num_integral_digits("-0e+1"), 3); + assert_eq!(num_integral_digits("-0.0e1"), 3); + // minus zero float + assert_eq!(num_integral_digits("-0.0"), 2); + assert_eq!(num_integral_digits("-0e-1"), 2); + assert_eq!(num_integral_digits("-0.0e-1"), 2); + + // TODO In GNU `seq`, the `-w` option does not seem to work with + // hexadecimal arguments. In order to match that behavior, we + // report the number of integral digits as zero for hexadecimal + // inputs. + assert_eq!(num_integral_digits("0xff"), 0); + } + + #[test] + #[allow(clippy::cognitive_complexity)] + fn test_num_fractional_digits() { + // no decimal, no exponent + assert_eq!(num_fractional_digits("123"), 0); + assert_eq!(num_fractional_digits("0xff"), 0); + // decimal, no exponent + assert_eq!(num_fractional_digits("123.45"), 2); + assert_eq!(num_fractional_digits("-0.1"), 1); + assert_eq!(num_fractional_digits("-.1"), 1); + // exponent, no decimal + assert_eq!(num_fractional_digits("123e4"), 0); + assert_eq!(num_fractional_digits("123e-4"), 4); + assert_eq!(num_fractional_digits("123e-1"), 1); + assert_eq!(num_fractional_digits("-1e-3"), 3); + // decimal and exponent + assert_eq!(num_fractional_digits("123.45e6"), 0); + assert_eq!(num_fractional_digits("123.45e1"), 1); + assert_eq!(num_fractional_digits("123.45e-6"), 8); + assert_eq!(num_fractional_digits("123.45e-1"), 3); + assert_eq!(num_fractional_digits("-0.1e0"), 1); + assert_eq!(num_fractional_digits("-0.1e2"), 0); + assert_eq!(num_fractional_digits("-.1e0"), 1); + assert_eq!(num_fractional_digits("-.1e2"), 0); + assert_eq!(num_fractional_digits("-1.e-3"), 3); + assert_eq!(num_fractional_digits("-1.0e-4"), 5); + // minus zero int + assert_eq!(num_fractional_digits("-0e0"), 0); + assert_eq!(num_fractional_digits("-0e-0"), 0); + assert_eq!(num_fractional_digits("-0e1"), 0); + assert_eq!(num_fractional_digits("-0e+1"), 0); + assert_eq!(num_fractional_digits("-0.0e1"), 0); + // minus zero float + assert_eq!(num_fractional_digits("-0.0"), 1); + assert_eq!(num_fractional_digits("-0e-1"), 1); + assert_eq!(num_fractional_digits("-0.0e-1"), 2); + // Hexadecimal numbers + assert_eq!(num_fractional_digits("0xff"), 0); + assert!(num_fractional_digits_is_none("0xff.1")); + } + + #[test] + fn test_parse_min_exponents() { + // Make sure exponents < i64::MIN do not cause errors + assert!("1e-9223372036854775807".parse::().is_ok()); + assert!("1e-9223372036854775808".parse::().is_ok()); + assert!("1e-92233720368547758080".parse::().is_ok()); + } + + #[test] + fn test_parse_max_exponents() { + // Make sure exponents much bigger than i64::MAX cause errors + assert!("1e9223372036854775807".parse::().is_ok()); + assert!("1e92233720368547758070".parse::().is_err()); + } +} diff --git a/crates/vendor/uu-seq/src/seq.rs b/crates/vendor/uu-seq/src/seq.rs new file mode 100644 index 000000000..82cdfd46c --- /dev/null +++ b/crates/vendor/uu-seq/src/seq.rs @@ -0,0 +1,561 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) bigdecimal extendedbigdecimal numberparse +// hexadecimalfloat biguint + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. seq is pure computation + stdout: all process-global +// stdio is routed through `pi_uutils_ctx` (the emission loops write to a +// `BufWriter` around the context stdout handle and poll +// `pi_uutils_ctx::is_cancelled()` periodically, since seq can generate +// unbounded output), `translate!` strings are literalized, SIGPIPE probing is +// dropped, and the entry point no longer calls `std::process::exit`. + +use std::{ + ffi::{OsStr, OsString}, + io::{BufWriter, Write}, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use num_bigint::BigUint; +use num_traits::{ToPrimitive, Zero}; +use pi_uutils_ctx::format_usage; +use uucore::{ + error::{FromIo, UResult}, + extendedbigdecimal::ExtendedBigDecimal, + fast_inc::fast_inc, + format::{Format, num_format, num_format::FloatVariant}, +}; + +mod error; + +mod number; +mod numberparse; +use crate::{error::SeqError, number::PreciseNumber}; + +const OPT_SEPARATOR: &str = "separator"; +const OPT_TERMINATOR: &str = "terminator"; +const OPT_EQUAL_WIDTH: &str = "equal-width"; +const OPT_FORMAT: &str = "format"; + +const ARG_NUMBERS: &str = "numbers"; + +/// pi-uutils: how many emitted numbers to print between cancellation polls in +/// the (potentially unbounded) emission loops. +const CANCEL_POLL_INTERVAL: u64 = 4096; + +#[derive(Clone)] +struct SeqOptions<'a> { + separator: OsString, + terminator: OsString, + equal_width: bool, + format: Option<&'a str>, +} + +/// A range of floats. +/// +/// The elements are (first, increment, last). +type RangeFloat = (ExtendedBigDecimal, ExtendedBigDecimal, ExtendedBigDecimal); + +/// Turn short args with attached value, for example "-s,", into two args "-s" +/// and "," to make them work with clap. +fn split_short_args_with_value(args: impl uucore::Args) -> impl uucore::Args { + let mut v: Vec = Vec::new(); + + for arg in args { + let bytes = arg.as_encoded_bytes(); + + if bytes.len() > 2 + && (bytes.starts_with(b"-f") || bytes.starts_with(b"-s") || bytes.starts_with(b"-t")) + { + let (short_arg, value) = bytes.split_at(2); + // SAFETY: + // Both `short_arg` and `value` only contain content that originated from + // `OsStr::as_encoded_bytes` + v.push(unsafe { OsString::from_encoded_bytes_unchecked(short_arg.to_vec()) }); + v.push(unsafe { OsString::from_encoded_bytes_unchecked(value.to_vec()) }); + } else { + v.push(arg); + } + } + + v.into_iter() +} + +fn select_precision( + first: &PreciseNumber, + increment: &PreciseNumber, + last: &PreciseNumber, +) -> Option { + match (first.num_fractional_digits, increment.num_fractional_digits, last.num_fractional_digits) + { + (Some(0), Some(0), Some(0)) => Some(0), + (Some(f), Some(i), Some(_)) => Some(f.max(i)), + _ => None, + } +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(split_short_args_with_value(argv.into_iter())) + { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match seq_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "seq: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn seq_main(matches: &ArgMatches) -> UResult<()> { + let numbers_option = matches.get_many::(ARG_NUMBERS); + + if numbers_option.is_none() { + return Err(SeqError::NoArguments.into()); + } + + let numbers = numbers_option.unwrap().collect::>(); + + let options = SeqOptions { + separator: matches + .get_one::(OPT_SEPARATOR) + .cloned() + .unwrap_or_else(|| OsString::from("\n")), + terminator: matches + .get_one::(OPT_TERMINATOR) + .cloned() + .unwrap_or_else(|| OsString::from("\n")), + equal_width: matches.get_flag(OPT_EQUAL_WIDTH), + format: matches.get_one::(OPT_FORMAT).map(String::as_str), + }; + + if options.equal_width && options.format.is_some() { + return Err(SeqError::FormatAndEqualWidth.into()); + } + + let first = if numbers.len() > 1 { + match numbers[0].parse() { + Ok(num) => num, + Err(e) => return Err(SeqError::ParseError(numbers[0].to_owned(), e).into()), + } + } else { + PreciseNumber::one() + }; + let increment = if numbers.len() > 2 { + match numbers[1].parse() { + Ok(num) => num, + Err(e) => return Err(SeqError::ParseError(numbers[1].to_owned(), e).into()), + } + } else { + PreciseNumber::one() + }; + if increment.is_zero() { + return Err(SeqError::ZeroIncrement(numbers[1].to_owned()).into()); + } + let last: PreciseNumber = { + // We are guaranteed that `numbers.len()` is greater than zero + // and at most three because of the argument specification in + // `uu_app()`. + let n: usize = numbers.len(); + match numbers[n - 1].parse() { + Ok(num) => num, + Err(e) => return Err(SeqError::ParseError(numbers[n - 1].to_owned(), e).into()), + } + }; + + // If a format was passed on the command line, use that. + // If not, use some default format based on parameters precision. + let (format, padding, fast_allowed) = if let Some(str) = options.format { + (Format::::parse(str)?, 0, false) + } else { + let precision = select_precision(&first, &increment, &last); + + let padding = if options.equal_width { + let precision_value = precision.unwrap_or(0); + first + .num_integral_digits + .max(increment.num_integral_digits) + .max(last.num_integral_digits) + + if precision_value > 0 { + precision_value + 1 + } else { + 0 + } + } else { + 0 + }; + + let formatter = match precision { + // format with precision: decimal floats and integers + Some(precision) => num_format::Float { + variant: FloatVariant::Decimal, + width: padding, + alignment: num_format::NumberAlignment::RightZero, + precision: Some(precision), + ..Default::default() + }, + // format without precision: hexadecimal floats + None => num_format::Float { variant: FloatVariant::Shortest, ..Default::default() }, + }; + // Allow fast printing if precision is 0 (integer inputs), `print_seq` will do + // further checks. + (Format::from_formatter(formatter), padding, precision == Some(0)) + }; + + let result = print_seq( + (first.number, increment.number, last.number), + &options.separator, + &options.terminator, + &format, + fast_allowed, + padding, + ); + + match result { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::BrokenPipe => { + // GNU seq prints the Broken pipe message but still exits with status 0 + // unless SIGPIPE was explicitly ignored, in which case it should fail. + // pi-uutils: the in-process builtin does not manipulate process + // signal dispositions, so the upstream `sigpipe_was_ignored` probe + // is dropped and the message goes to the context stderr. + let err = err.map_err_context(|| "write error".into()); + let _ = writeln!(pi_uutils_ctx::stderr(), "seq: {err}"); + Ok(()) + }, + Err(err) => Err(err.map_err_context(|| "write error".into())), + } +} + +pub fn uu_app() -> Command { + Command::new("seq") + .trailing_var_arg(true) + .infer_long_args(true) + .version(uucore::crate_version!()) + .about("Display numbers from FIRST to LAST, in steps of INCREMENT.") + .override_usage(format_usage( + "seq [OPTION]... LAST\nseq [OPTION]... FIRST LAST\nseq [OPTION]... FIRST INCREMENT LAST", + )) + .arg( + Arg::new(OPT_SEPARATOR) + .short('s') + .long("separator") + .help("Separator character (defaults to \\n)") + .value_parser(clap::value_parser!(OsString)), + ) + .arg( + Arg::new(OPT_TERMINATOR) + .short('t') + .long("terminator") + .help("Terminator character (defaults to \\n)") + .value_parser(clap::value_parser!(OsString)), + ) + .arg( + Arg::new(OPT_EQUAL_WIDTH) + .short('w') + .long("equal-width") + .help("Equalize widths of all numbers by padding with zeros") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(OPT_FORMAT) + .short('f') + .long(OPT_FORMAT) + .help("use printf style floating-point FORMAT"), + ) + .arg( + // we use allow_hyphen_values instead of allow_negative_numbers because clap removed + // the support for "exotic" negative numbers like -.1 (see https://github.com/clap-rs/clap/discussions/5837) + Arg::new(ARG_NUMBERS) + .allow_hyphen_values(true) + .action(ArgAction::Append) + .num_args(1..=3), + ) +} + +/// Integer print, default format, positive increment: fast code path +/// that avoids reformatting digit at all iterations. +fn fast_print_seq( + mut stdout: impl Write, + first: &BigUint, + increment: u64, + last: &BigUint, + separator: &OsStr, + terminator: &OsStr, + padding: usize, +) -> std::io::Result<()> { + // Nothing to do, just return. + if last < first { + return Ok(()); + } + + // Do at most u64::MAX loops. We can print in the order of 1e8 digits per + // second, u64::MAX is 1e19, so it'd take hundreds of years for this to + // complete anyway. TODO: we can move this test to `print_seq` if we care about + // this case. + let loop_cnt = ((last - first) / increment).to_u64().unwrap_or(u64::MAX); + + // Format the first number. + let first_str = first.to_string(); + + // Makeshift log10.ceil + let last_length = last.to_string().len(); + + // Allocate a large u8 buffer, that contains a preformatted string + // of the number followed by the `separator`. + // + // | ... head space ... | number | separator | + // ^0 ^ start ^ num_end ^ size (==buf.len()) + // + // We keep track of start in this buffer, as the number grows. + // When printing, we take a slice between start and end. + let size = last_length.max(padding) + separator.len(); + // Fill with '0', this is needed for equal_width, and harmless otherwise. + let mut buf = vec![b'0'; size]; + let buf = buf.as_mut_slice(); + + let num_end = buf.len() - separator.len(); + let mut start = num_end - first_str.len(); + + // Initialize buf with first and separator. + buf[start..num_end].copy_from_slice(first_str.as_bytes()); + buf[num_end..].copy_from_slice(separator.as_encoded_bytes()); + + // Normally, if padding is > 0, it should be equal to last_length, + // so start would be == 0, but there are corner cases. + start = start.min(num_end - padding); + + // Prepare the number to increment with as a string + let inc_str = increment.to_string(); + let inc_str = inc_str.as_bytes(); + + for i in 0..loop_cnt { + // pi-uutils: seq can generate effectively unbounded output; poll the + // host cancel flag periodically so shell abort/timeout is observed. + if i % CANCEL_POLL_INTERVAL == 0 && pi_uutils_ctx::is_cancelled() { + return Ok(()); + } + stdout.write_all(&buf[start..])?; + fast_inc(buf, &mut start, num_end, inc_str); + } + // Write the last number without separator, but with terminator. + stdout.write_all(&buf[start..num_end])?; + stdout.write_all(terminator.as_encoded_bytes())?; + stdout.flush()?; + Ok(()) +} + +fn done_printing(next: &T, increment: &T, last: &T) -> bool { + if increment >= &T::zero() { + next > last + } else { + next < last + } +} + +/// Arbitrary precision decimal number code path ("slow" path) +fn print_seq( + range: RangeFloat, + separator: &OsStr, + terminator: &OsStr, + format: &Format, + fast_allowed: bool, + padding: usize, // Used by fast path only +) -> std::io::Result<()> { + // pi-uutils: buffer the context stdout handle instead of the (locked) + // process stdout. + let mut stdout = BufWriter::new(pi_uutils_ctx::stdout()); + let (first, increment, last) = range; + + if fast_allowed { + // Test if we can use fast code path. + // First try to convert the range to BigUint (u64 for the increment). + let (first_bui, increment_u64, last_bui) = + (first.to_biguint(), increment.to_biguint().and_then(|x| x.to_u64()), last.to_biguint()); + if let (Some(first_bui), Some(increment_u64), Some(last_bui)) = + (first_bui, increment_u64, last_bui) + { + return fast_print_seq( + stdout, + &first_bui, + increment_u64, + &last_bui, + separator, + terminator, + padding, + ); + } + } + + let mut value = first; + + let mut is_first_iteration = true; + // pi-uutils: iteration counter for periodic cancellation polling. + let mut iterations: u64 = 0; + while !done_printing(&value, &increment, &last) { + // pi-uutils: seq can generate effectively unbounded output; poll the + // host cancel flag periodically so shell abort/timeout is observed. + if iterations.is_multiple_of(CANCEL_POLL_INTERVAL) && pi_uutils_ctx::is_cancelled() { + return Ok(()); + } + iterations += 1; + if !is_first_iteration { + stdout.write_all(separator.as_encoded_bytes())?; + } + format.fmt(&mut stdout, &value)?; + // TODO Implement augmenting addition. + value = value + increment.clone(); + is_first_iteration = false; + } + if !is_first_iteration { + stdout.write_all(terminator.as_encoded_bytes())?; + } + stdout.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_scoped(args: Vec<&str>, cancelled: bool) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd: PathBuf::from("."), + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(cancelled)), + }; + + let argv: Vec = std::iter::once("seq") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + fn run_in(args: Vec<&str>) -> (i32, String, String) { + run_scoped(args, false) + } + + #[test] + fn single_operand_counts_from_one() { + let (code, stdout, stderr) = run_in(vec!["3"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "1\n2\n3\n", "")); + } + + #[test] + fn first_increment_last_arithmetic() { + let (code, stdout, stderr) = run_in(vec!["2", "2", "10"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "2\n4\n6\n8\n10\n", "")); + } + + #[test] + fn separator_joins_values_terminator_ends_them() { + let (code, stdout, stderr) = run_in(vec!["-s", ",", "1", "3"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "1,2,3\n", "")); + + // Attached short-arg value goes through `split_short_args_with_value`. + let (code, stdout, _) = run_in(vec!["-s,", "1", "3"]); + assert_eq!((code, stdout.as_str()), (0, "1,2,3\n")); + } + + #[test] + fn equal_width_pads_with_zeros() { + let (code, stdout, stderr) = run_in(vec!["-w", "8", "10"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "08\n09\n10\n", "")); + } + + #[test] + fn float_increment_selects_widest_precision() { + let (code, stdout, stderr) = run_in(vec!["1", "0.5", "2"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "1.0\n1.5\n2.0\n", "")); + } + + #[test] + fn invalid_operand_reports_error_and_fails() { + let (code, stdout, stderr) = run_in(vec!["foo"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "seq: invalid floating point argument: 'foo'\n"); + } + + #[test] + fn zero_increment_is_rejected() { + let (code, stdout, stderr) = run_in(vec!["1", "0", "5"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, "seq: invalid Zero increment value: '0'\n"); + } + + #[test] + fn cancelled_scope_stops_emission() { + // pi-specific contract: a pre-cancelled scope aborts the (potentially + // unbounded) emission loop instead of printing the full range. + let (code, stdout, stderr) = run_scoped(vec!["1", "1000000"], true); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("steps of INCREMENT")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-stat/Cargo.toml b/crates/vendor/uu-stat/Cargo.toml new file mode 100644 index 000000000..5c7f30de8 --- /dev/null +++ b/crates/vendor/uu-stat/Cargo.toml @@ -0,0 +1,23 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/stat), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/stat.rs +# for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_stat" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "stat ~ (uutils) display FILE status (vendored + patched for in-process embedding)" + +[lib] +path = "src/stat.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +thiserror = "2.0.3" +uucore = { version = "0.8.0", features = ["entries", "libc", "fs", "fsext", "time"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-stat/LICENSE b/crates/vendor/uu-stat/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-stat/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-stat/src/stat.rs b/crates/vendor/uu-stat/src/stat.rs new file mode 100644 index 000000000..6cca57259 --- /dev/null +++ b/crates/vendor/uu-stat/src/stat.rs @@ -0,0 +1,1785 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore datetime + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Every filesystem syscall (stat/lstat/statfs/readlink) +// resolves its path operand against the shell working directory via +// `pi_uutils_ctx::resolve` AT THE CALL SITE, while the original operands are +// kept for display/error messages and `%n` output (GNU prints operands as +// typed). All process-global stdio is routed through `pi_uutils_ctx`, +// `translate!` strings are literalized from locales/en-US.ftl, QUOTING_STYLE is +// read from the scope environment, SELinux support is dropped, and the entry +// point no longer calls `std::process::exit`. The upstream implementation is +// unix-only (it relies on `std::os::unix`), so it lives behind `#[cfg(unix)]`; +// non-unix targets get a stub that reports the builtin as unsupported. + +#[cfg(unix)] +pub use imp::{run, uu_app}; + +/// pi-uutils: non-unix stub — upstream stat cannot be built off unix. +#[cfg(not(unix))] +pub fn run(_argv: Vec) -> i32 { + use std::io::Write; + let _ = writeln!(pi_uutils_ctx::stderr(), "stat: unsupported on this platform"); + 1 +} + +/// pi-uutils: minimal non-unix counterpart of the real `uu_app`. +#[cfg(not(unix))] +pub fn uu_app() -> clap::Command { + clap::Command::new("stat") + .version(uucore::crate_version!()) + .about("Display file or file system status.") + .override_usage(pi_uutils_ctx::format_usage("stat [OPTION]... FILE...")) +} + +#[cfg(unix)] +mod imp { + use std::{ + borrow::Cow, + cell::OnceCell, + ffi::{OsStr, OsString}, + fs::{self, FileType, Metadata}, + io::Write, + os::unix::fs::{FileTypeExt, MetadataExt}, + path::Path, + }; + + use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser}; + use pi_uutils_ctx::format_usage; + use thiserror::Error; + use uucore::{ + display::Quotable, + entries, + error::{UError, UResult, USimpleError}, + fs::{display_permissions, major, minor}, + fsext::{ + FsMeta, MetadataTimeField, StatFs, metadata_get_time, pretty_filetype, pretty_fstype, + read_fs_list, statfs, + }, + libc::mode_t, + time::{FormatSystemTimeFallback, format_system_time, system_time_to_sec}, + }; + + const ABOUT: &str = "Display file or file system status."; + const USAGE: &str = "stat [OPTION]... FILE..."; + // pi-uutils: literalized from locales/en-US.ftl (`stat-after-help`). + const AFTER_HELP: &str = "Valid format sequences for files (without `--file-system`): + +-`%a`: access rights in octal (note '#' and '0' printf flags) +-`%A`: access rights in human readable form +-`%b`: number of blocks allocated (see %B) +-`%B`: the size in bytes of each block reported by %b +-`%C`: SELinux security context string +-`%d`: device number in decimal +-`%D`: device number in hex +-`%f`: raw mode in hex +-`%F`: file type +-`%g`: group ID of owner +-`%G`: group name of owner +-`%h`: number of hard links +-`%i`: inode number +-`%m`: mount point +-`%n`: file name +-`%N`: quoted file name with dereference (follow) if symbolic link +-`%o`: optimal I/O transfer size hint +-`%s`: total size, in bytes +-`%t`: major device type in hex, for character/block device special files +-`%T`: minor device type in hex, for character/block device special files +-`%u`: user ID of owner +-`%U`: user name of owner +-`%w`: time of file birth, human-readable; - if unknown +-`%W`: time of file birth, seconds since Epoch; 0 if unknown +-`%x`: time of last access, human-readable +-`%X`: time of last access, seconds since Epoch +-`%y`: time of last data modification, human-readable + +-`%Y`: time of last data modification, seconds since Epoch +-`%z`: time of last status change, human-readable +-`%Z`: time of last status change, seconds since Epoch + +Valid format sequences for file systems: + +-`%a`: free blocks available to non-superuser +-`%b`: total data blocks in file system +-`%c`: total file nodes in file system +-`%d`: free file nodes in file system +-`%f`: free blocks in file system +-`%i`: file system ID in hex +-`%l`: maximum length of filenames +-`%n`: file name +-`%s`: block size (for faster transfers) +-`%S`: fundamental block size (for block counts) +-`%t`: file system type in hex +-`%T`: file system type in human readable form + +NOTE: your shell may have its own version of stat, which usually supersedes +the version described here. Please refer to your shell's documentation +for details about the options it supports."; + + // pi-uutils: `translate!` error strings literalized from locales/en-US.ftl. + #[derive(Debug, Error)] + enum StatError { + #[error("Invalid quoting style: {style}")] + InvalidQuotingStyle { style: String }, + #[error("missing operand\nTry 'stat --help' for more information.")] + MissingOperand, + #[error("{directive}: invalid directive")] + InvalidDirective { directive: String }, + #[error("cannot read table of mounted file systems: {error}")] + CannotReadFilesystem { error: String }, + #[error("using '-' to denote standard input does not work in file system mode")] + StdinFilesystemMode, + #[error("cannot read file system information for {file}: {error}")] + CannotReadFilesystemInfo { file: String, error: String }, + #[error("cannot stat {file}: {error}")] + CannotStat { file: String, error: String }, + } + + impl UError for StatError { + fn code(&self) -> i32 { + 1 + } + } + + mod options { + pub const DEREFERENCE: &str = "dereference"; + pub const FILE_SYSTEM: &str = "file-system"; + pub const FORMAT: &str = "format"; + pub const PRINTF: &str = "printf"; + pub const TERSE: &str = "terse"; + pub const FILES: &str = "files"; + } + + #[derive(Default, Debug, PartialEq, Eq, Clone, Copy)] + struct Flags { + alter: bool, + zero: bool, + left: bool, + space: bool, + sign: bool, + group: bool, + major: bool, + minor: bool, + } + + /// checks if the string is within the specified bound, + /// if it gets out of bound, error out by printing sub-string from index + /// `beg` to`end`, where `beg` & `end` is the beginning and end index of + /// sub-string, respectively + fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> UResult<()> { + if end >= bound { + return Err(USimpleError::new( + 1, + StatError::InvalidDirective { directive: slice[beg..end].quote().to_string() } + .to_string(), + )); + } + Ok(()) + } + + enum Padding { + Zero, + Space, + } + + /// pads the string with zeroes or spaces and prints it + /// + /// # Example + /// ```ignore + /// uu_stat::pad_and_print("1", false, 5, Padding::Zero) == "00001"; + /// ``` + /// currently only supports '0' & ' ' as the padding character + /// because the format specification of print! does not support general + /// fill characters. + fn pad_and_print(result: &str, left: bool, width: usize, padding: Padding) { + // pi-uutils: write to the context stdout instead of `print!`. + let mut out = pi_uutils_ctx::stdout(); + let _ = match (left, padding) { + (false, Padding::Zero) => write!(out, "{result:0>width$}"), + (false, Padding::Space) => write!(out, "{result:>width$}"), + (true, Padding::Zero) => write!(out, "{result:0 write!(out, "{result:( + mut writer: W, + bytes: &[u8], + left: bool, + width: usize, + precision: Precision, + ) -> Result<(), std::io::Error> { + let display_bytes = match precision { + Precision::Number(p) if p < bytes.len() => &bytes[..p], + _ => bytes, + }; + + let display_len = display_bytes.len(); + let padding_needed = width.saturating_sub(display_len); + + let (left_pad, right_pad) = if left { + (0, padding_needed) + } else { + (padding_needed, 0) + }; + + if left_pad > 0 { + write_padding(&mut writer, left_pad)?; + } + writer.write_all(display_bytes)?; + if right_pad > 0 { + write_padding(&mut writer, right_pad)?; + } + + Ok(()) + } + + /// write padding based on a writer W and n size + /// writer is genric to be any buffer like: `std::io::stdout` + /// n is the calculated padding size + fn write_padding(writer: &mut W, n: usize) -> Result<(), std::io::Error> { + for _ in 0..n { + writer.write_all(b" ")?; + } + Ok(()) + } + + #[derive(Debug)] + pub enum OutputType<'a> { + Str(String), + OsStr(&'a OsString), + Integer(i64), + Unsigned(u64), + UnsignedHex(u64), + UnsignedOct(u32), + Float(f64), + Unknown, + } + + #[derive(Default)] + enum QuotingStyle { + Locale, + Shell, + #[default] + ShellEscapeAlways, + Quote, + } + + impl std::str::FromStr for QuotingStyle { + type Err = StatError; + + fn from_str(s: &str) -> Result { + match s { + "locale" => Ok(Self::Locale), + "shell" => Ok(Self::Shell), + "shell-escape-always" => Ok(Self::ShellEscapeAlways), + // The others aren't exposed to the user + _ => Err(StatError::InvalidQuotingStyle { style: s.to_string() }), + } + } + } + + #[derive(Debug, PartialEq, Eq, Clone, Copy)] + enum Precision { + NotSpecified, + NoNumber, + Number(usize), + } + + #[derive(Debug, PartialEq, Eq)] + enum Token { + Char(char), + Byte(u8), + Directive { flag: Flags, width: usize, precision: Precision, format: char }, + } + + trait ScanUtil { + fn scan_num(&self) -> Option<(F, usize)> + where + F: std::str::FromStr; + fn scan_char(&self, radix: u32) -> Option<(char, usize)>; + } + + impl ScanUtil for str { + /// Scans for a number at the beginning of the string + /// Returns the parsed number and the character count + /// Since we only deal with ASCII characters (+, -, 0-9), character count + /// equals byte count + fn scan_num(&self) -> Option<(F, usize)> + where + F: std::str::FromStr, + { + let mut chars = self.chars(); + let count = chars + .next() + .filter(|&c| c.is_ascii_digit() || c == '-' || c == '+') + .map_or(0, |_| 1 + chars.take_while(char::is_ascii_digit).count()); + + if count > 0 { + F::from_str(&self[..count]).ok().map(|x| (x, count)) + } else { + None + } + } + + fn scan_char(&self, radix: u32) -> Option<(char, usize)> { + let count = match radix { + 8 => 3, + 16 => 2, + _ => return None, + }; + let chars = self.chars().enumerate(); + let mut res = 0; + let mut offset = 0; + for (i, c) in chars { + if i >= count { + break; + } + match c.to_digit(radix) { + Some(digit) => { + let tmp = res * radix + digit; + if tmp < 256 { + res = tmp; + } else { + break; + } + }, + None => break, + } + offset = i + 1; + } + if offset > 0 { + Some((res as u8 as char, offset)) + } else { + None + } + } + } + + fn group_num(s: &str) -> Cow<'_, str> { + let is_negative = s.starts_with('-'); + assert!(is_negative || s.chars().take(1).all(|c| c.is_ascii_digit())); + assert!(s.chars().skip(1).all(|c| c.is_ascii_digit())); + if s.len() < 4 { + return s.into(); + } + let mut res = String::with_capacity((s.len() - 1) / 3); + let s = if is_negative { + res.push('-'); + &s[1..] + } else { + s + }; + let mut alone = (s.len() - 1) % 3 + 1; + res.push_str(&s[..alone]); + while alone != s.len() { + res.push(','); + res.push_str(&s[alone..alone + 3]); + alone += 3; + } + res.into() + } + + struct Stater { + follow: bool, + show_fs: bool, + from_user: bool, + files: Vec, + mount_list: OnceCell>>, + mount_list_needed: bool, + default_tokens: Vec, + default_dev_tokens: Vec, + } + + /// Prints a formatted output based on the provided output type, flags, + /// width, and precision. + /// + /// # Arguments + /// + /// * `output` - A reference to the [`OutputType`] enum containing the value + /// to be printed. + /// * `flags` - A Flags struct containing formatting flags. + /// * `width` - The width of the field for the printed output. + /// * `precision` - How many digits of precision, if any. + /// + /// This function delegates the printing process to more specialized + /// functions depending on the output type. + fn print_it(output: &OutputType, flags: Flags, width: usize, precision: Precision) { + // If the precision is given as just '.', the precision is taken to be zero. + // A negative precision is taken as if the precision were omitted. + // This gives the minimum number of digits to appear for d, i, o, u, x, and X + // conversions, the maximum number of characters to be printed from a string + // for s and S conversions. + + // # + // The value should be converted to an "alternate form". + // For o conversions, the first character of the output string is made zero + // (by prefixing a 0 if it was not zero already). For x and X conversions, a + // nonzero result has the string "0x" (or "0X" for X conversions) prepended to + // it. + + // 0 + // The value should be zero padded. + // For d, i, o, u, x, X, a, A, e, E, f, F, g, and G conversions, the converted + // value is padded on the left with zeros rather than blanks. If the 0 and - + // flags both appear, the 0 flag is ignored. If a precision is given with a + // numeric conversion (d, i, o, u, x, and X), the 0 flag is ignored. For other + // conversions, the behavior is undefined. + + // - + // The converted value is to be left adjusted on the field boundary. (The + // default is right justification.) The converted value is padded on the + // right with blanks, rather than on the left with blanks or zeros. + // A - overrides a 0 if both are given. + + // ' ' (a space) + // A blank should be left before a positive number (or empty string) produced by + // a signed conversion. + + // + + // A sign (+ or -) should always be placed before a number produced by a signed + // conversion. By default, a sign is used only for negative numbers. + // A + overrides a space if both are used. + let padding_char = determine_padding_char(flags); + + match output { + OutputType::Str(s) => print_str(s, flags, width, precision), + OutputType::OsStr(s) => print_os_str(s, flags, width, precision), + OutputType::Integer(num) => print_integer(*num, flags, width, precision, padding_char), + OutputType::Unsigned(num) => { + print_unsigned(*num, flags, width, precision, padding_char); + }, + OutputType::UnsignedOct(num) => { + print_unsigned_oct(*num, flags, width, precision, padding_char); + }, + OutputType::UnsignedHex(num) => { + print_unsigned_hex(*num, flags, width, precision, padding_char); + }, + OutputType::Float(num) => { + print_float(*num, flags, width, precision, padding_char); + }, + // pi-uutils: context stdout instead of `print!`. + OutputType::Unknown => { + let _ = write!(pi_uutils_ctx::stdout(), "?"); + }, + } + } + + /// Determines the padding character based on the provided flags and + /// precision. + /// + /// # Arguments + /// + /// * `flags` - A reference to the Flags struct containing formatting flags. + /// + /// # Returns + /// + /// * Padding - An instance of the Padding enum representing the padding + /// character. + fn determine_padding_char(flags: Flags) -> Padding { + if flags.zero && !flags.left { + Padding::Zero + } else { + Padding::Space + } + } + + /// Prints a string value based on the provided flags, width, and precision. + /// + /// # Arguments + /// + /// * `s` - The string to be printed. + /// * `flags` - A reference to the Flags struct containing formatting flags. + /// * `width` - The width of the field for the printed string. + /// * `precision` - How many digits of precision, if any. + fn print_str(s: &str, flags: Flags, width: usize, precision: Precision) { + let s = match precision { + Precision::Number(p) if p < s.len() => &s[..p], + _ => s, + }; + pad_and_print(s, flags.left, width, Padding::Space); + } + + /// Prints a `OsString` value based on the provided flags, width, and + /// precision. It converts the value to bytes and prints them; if that + /// fails, it prints the lossy string version. + /// + /// # Arguments + /// + /// * `s` - The `OsString` to be printed. + /// * `flags` - A reference to the Flags struct containing formatting flags. + /// * `width` - The width of the field for the printed string. + /// * `precision` - How many digits of precision, if any. + fn print_os_str(s: &OsString, flags: Flags, width: usize, precision: Precision) { + // pi-uutils: this module is unix-only, so upstream's `cfg(not(unix))` + // lossy fallback branch is dropped; bytes go to the context stdout. + use std::os::unix::ffi::OsStrExt; + + let bytes = s.as_bytes(); + + if pad_and_print_bytes(pi_uutils_ctx::stdout(), bytes, flags.left, width, precision).is_err() + { + // if an error occurred while trying to print bytes fall back to normal lossy + // string so it can be printed + let fallback_string = s.to_string_lossy(); + print_str(&fallback_string, flags, width, precision); + } + } + + fn quote_file_name(file_name: &str, quoting_style: &QuotingStyle) -> String { + match quoting_style { + QuotingStyle::Locale | QuotingStyle::Shell => { + let escaped = file_name.replace('\'', r"\'"); + format!("'{escaped}'") + }, + QuotingStyle::ShellEscapeAlways => { + let quote = if file_name.contains('\'') { '"' } else { '\'' }; + format!("{quote}{file_name}{quote}") + }, + QuotingStyle::Quote => file_name.to_string(), + } + } + + fn get_quoted_file_name( + display_name: &str, + // pi-uutils: takes the operand resolved against the shell working + // directory for the `readlink` syscall; `display_name` stays as typed. + resolved: &Path, + file_type: FileType, + from_user: bool, + ) -> Result { + // pi-uutils: QUOTING_STYLE comes from the scope environment (the + // shell's exported variables), not the host process environment. + let quoting_style = pi_uutils_ctx::var("QUOTING_STYLE") + .and_then(|style| style.parse().ok()) + .unwrap_or_default(); + + if file_type.is_symlink() { + let quoted_display_name = quote_file_name(display_name, "ing_style); + match fs::read_link(resolved) { + Ok(dst) => { + let quoted_dst = quote_file_name(&dst.to_string_lossy(), "ing_style); + Ok(format!("{quoted_display_name} -> {quoted_dst}")) + }, + Err(e) => { + // pi-uutils: `show_error!` replaced with a context-stderr write. + let _ = writeln!(pi_uutils_ctx::stderr(), "stat: {e}"); + Err(1) + }, + } + } else { + let style = if from_user { + quoting_style + } else { + QuotingStyle::Quote + }; + Ok(quote_file_name(display_name, &style)) + } + } + + fn process_token_filesystem(t: &Token, meta: &StatFs, display_name: &str) { + match *t { + Token::Byte(byte) => write_raw_byte(byte), + // pi-uutils: context stdout instead of `print!`. + Token::Char(c) => { + let _ = write!(pi_uutils_ctx::stdout(), "{c}"); + }, + Token::Directive { flag, width, precision, format } => { + let output = match format { + // free blocks available to non-superuser + 'a' => OutputType::Unsigned(meta.avail_blocks()), + // total data blocks in file system + 'b' => OutputType::Unsigned(meta.total_blocks()), + // total file nodes in file system + 'c' => OutputType::Unsigned(meta.total_file_nodes()), + // free file nodes in file system + 'd' => OutputType::Unsigned(meta.free_file_nodes()), + // free blocks in file system + 'f' => OutputType::Unsigned(meta.free_blocks()), + // file system ID in hex + 'i' => OutputType::UnsignedHex(meta.fsid()), + // maximum length of filenames + 'l' => OutputType::Unsigned(meta.namelen()), + // file name + 'n' => OutputType::Str(display_name.to_string()), + // block size (for faster transfers) + 's' => OutputType::Unsigned(meta.io_size()), + // fundamental block size (for block counts) + 'S' => OutputType::Integer(meta.block_size()), + // file system type in hex + 't' => OutputType::UnsignedHex(meta.fs_type() as u64), + // file system type in human readable form + 'T' => OutputType::Str(pretty_fstype(meta.fs_type()).into()), + _ => OutputType::Unknown, + }; + + print_it(&output, flag, width, precision); + }, + } + } + + /// Prints an integer value based on the provided flags, width, and + /// precision. + /// + /// # Arguments + /// + /// * `num` - The integer value to be printed. + /// * `flags` - A reference to the Flags struct containing formatting flags. + /// * `width` - The width of the field for the printed integer. + /// * `precision` - How many digits of precision, if any. + /// * `padding_char` - The padding character as determined by + /// `determine_padding_char`. + fn print_integer( + num: i64, + flags: Flags, + width: usize, + precision: Precision, + padding_char: Padding, + ) { + let num = num.to_string(); + let arg = if flags.group { + group_num(&num) + } else { + Cow::Borrowed(num.as_str()) + }; + let prefix = if flags.sign { + "+" + } else if flags.space { + " " + } else { + "" + }; + let extended = match precision { + Precision::NotSpecified => format!("{prefix}{arg}"), + Precision::NoNumber => format!("{prefix}{arg}"), + Precision::Number(p) => format!("{prefix}{arg:0>p$}"), + }; + pad_and_print(&extended, flags.left, width, padding_char); + } + + /// Truncate a float to the given number of digits after the decimal point. + fn precision_trunc(num: f64, precision: Precision) -> String { + // GNU `stat` doesn't round, it just seems to truncate to the + // given precision: + // + // $ stat -c "%.5Y" /dev/pts/ptmx + // 1736344012.76399 + // $ stat -c "%.4Y" /dev/pts/ptmx + // 1736344012.7639 + // $ stat -c "%.3Y" /dev/pts/ptmx + // 1736344012.763 + // + // Contrast this with `printf`, which seems to round the + // numbers: + // + // $ printf "%.5f\n" 1736344012.76399 + // 1736344012.76399 + // $ printf "%.4f\n" 1736344012.76399 + // 1736344012.7640 + // $ printf "%.3f\n" 1736344012.76399 + // 1736344012.764 + // + let num_str = num.to_string(); + let n = num_str.len(); + match (num_str.find('.'), precision) { + (None, Precision::NotSpecified) => num_str, + (None, Precision::NoNumber) => num_str, + (None, Precision::Number(0)) => num_str, + (None, Precision::Number(p)) => format!("{num_str}.{zeros}", zeros = "0".repeat(p)), + (Some(i), Precision::NotSpecified) => num_str[..i].to_string(), + (Some(_), Precision::NoNumber) => num_str, + (Some(i), Precision::Number(0)) => num_str[..i].to_string(), + (Some(i), Precision::Number(p)) if p < n - i => num_str[..i + 1 + p].to_string(), + (Some(i), Precision::Number(p)) => { + format!("{num_str}{zeros}", zeros = "0".repeat(p - (n - i - 1))) + }, + } + } + + fn print_float( + num: f64, + flags: Flags, + width: usize, + precision: Precision, + padding_char: Padding, + ) { + let prefix = if flags.sign { + "+" + } else if flags.space { + " " + } else { + "" + }; + let num_str = precision_trunc(num, precision); + let extended = format!("{prefix}{num_str}"); + pad_and_print(&extended, flags.left, width, padding_char); + } + + /// Prints an unsigned integer value based on the provided flags, width, and + /// precision. + /// + /// # Arguments + /// + /// * `num` - The unsigned integer value to be printed. + /// * `flags` - A reference to the Flags struct containing formatting flags. + /// * `width` - The width of the field for the printed unsigned integer. + /// * `precision` - How many digits of precision, if any. + /// * `padding_char` - The padding character as determined by + /// `determine_padding_char`. + fn print_unsigned( + num: u64, + flags: Flags, + width: usize, + precision: Precision, + padding_char: Padding, + ) { + let num = num.to_string(); + let s = if flags.group { + group_num(&num) + } else { + Cow::Borrowed(num.as_str()) + }; + let s = match precision { + Precision::NotSpecified => s, + Precision::NoNumber => s, + Precision::Number(p) => format!("{s:0>p$}").into(), + }; + pad_and_print(&s, flags.left, width, padding_char); + } + + /// Prints an unsigned octal integer value based on the provided flags, + /// width, and precision. + /// + /// # Arguments + /// + /// * `num` - The unsigned octal integer value to be printed. + /// * `flags` - A reference to the Flags struct containing formatting flags. + /// * `width` - The width of the field for the printed unsigned octal + /// integer. + /// * `precision` - How many digits of precision, if any. + /// * `padding_char` - The padding character as determined by + /// `determine_padding_char`. + fn print_unsigned_oct( + num: u32, + flags: Flags, + width: usize, + precision: Precision, + padding_char: Padding, + ) { + let prefix = if flags.alter { "0" } else { "" }; + let s = match precision { + Precision::NotSpecified => format!("{prefix}{num:o}"), + Precision::NoNumber => format!("{prefix}{num:o}"), + Precision::Number(p) => format!("{prefix}{num:0>p$o}"), + }; + pad_and_print(&s, flags.left, width, padding_char); + } + + /// Prints an unsigned hexadecimal integer value based on the provided flags, + /// width, and precision. + /// + /// # Arguments + /// + /// * `num` - The unsigned hexadecimal integer value to be printed. + /// * `flags` - A reference to the Flags struct containing formatting flags. + /// * `width` - The width of the field for the printed unsigned hexadecimal + /// integer. + /// * `precision` - How many digits of precision, if any. + /// * `padding_char` - The padding character as determined by + /// `determine_padding_char`. + fn print_unsigned_hex( + num: u64, + flags: Flags, + width: usize, + precision: Precision, + padding_char: Padding, + ) { + let prefix = if flags.alter { "0x" } else { "" }; + let s = match precision { + Precision::NotSpecified => format!("{prefix}{num:x}"), + Precision::NoNumber => format!("{prefix}{num:x}"), + Precision::Number(p) => format!("{prefix}{num:0>p$x}"), + }; + pad_and_print(&s, flags.left, width, padding_char); + } + + fn write_raw_byte(byte: u8) { + // pi-uutils: context stdout instead of the process stdout, and no + // `unwrap` — an in-process builtin must not panic on a broken pipe. + let _ = pi_uutils_ctx::stdout().write_all(&[byte]); + } + + impl Stater { + fn process_flags(chars: &[char], i: &mut usize, bound: usize, flag: &mut Flags) { + while *i < bound { + match chars[*i] { + '#' => flag.alter = true, + '0' => flag.zero = true, + '-' => flag.left = true, + ' ' => flag.space = true, + // This is not documented but the behavior seems to be + // the same as a space. For example `stat -c "%I5s" f` + // prints " 0". + 'I' => flag.space = true, + '+' => flag.sign = true, + '\'' => flag.group = true, + _ => break, + } + *i += 1; + } + } + + /// Converts a character index to a byte index in a UTF-8 string + /// This is necessary because Rust strings are UTF-8 encoded, so character + /// positions don't always align with byte positions for multi-byte + /// characters + fn char_index_to_byte_index(format_str: &str, char_index: usize) -> usize { + format_str + .char_indices() + .nth(char_index) + .map_or(format_str.len(), |(byte_idx, _)| byte_idx) + } + + fn handle_percent_case( + chars: &[char], + i: &mut usize, + bound: usize, + format_str: &str, + ) -> UResult { + let old = *i; + + *i += 1; + if *i >= bound { + return Ok(Token::Char('%')); + } + if chars[*i] == '%' { + return Ok(Token::Char('%')); + } + + let mut flag = Flags::default(); + + Self::process_flags(chars, i, bound, &mut flag); + + let mut width = 0; + let mut precision = Precision::NotSpecified; + let mut j = *i; + + let j_byte = Self::char_index_to_byte_index(format_str, j); + if let Some((field_width, offset)) = format_str[j_byte..].scan_num::() { + width = field_width; + j += offset; + + // Reject directives like `%` by checking if width has been parsed. + if j >= bound || chars[j] == '%' { + let invalid_directive: String = chars[old..=j.min(bound - 1)].iter().collect(); + return Err(USimpleError::new( + 1, + StatError::InvalidDirective { directive: invalid_directive.quote().to_string() } + .to_string(), + )); + } + } + check_bound(format_str, bound, old, j)?; + + if chars[j] == '.' { + j += 1; + check_bound(format_str, bound, old, j)?; + + let j_byte = Self::char_index_to_byte_index(format_str, j); + match format_str[j_byte..].scan_num::() { + Some((value, offset)) => { + if value >= 0 { + precision = Precision::Number(value as usize); + } + j += offset; + }, + None => precision = Precision::NoNumber, + } + check_bound(format_str, bound, old, j)?; + } + + *i = j; + + // Check for multi-character specifiers (e.g., `%Hd`, `%Lr`) + if *i + 1 < bound + && let Some(&next_char) = chars.get(*i + 1) + && (chars[*i] == 'H' || chars[*i] == 'L') + && (next_char == 'd' || next_char == 'r') + { + flag.major = chars[*i] == 'H'; + flag.minor = chars[*i] == 'L'; + *i += 1; + return Ok(Token::Directive { flag, width, precision, format: next_char }); + } + + Ok(Token::Directive { flag, width, precision, format: chars[*i] }) + } + + fn handle_escape_sequences( + chars: &[char], + i: &mut usize, + bound: usize, + format_str: &str, + ) -> Token { + *i += 1; + if *i >= bound { + // pi-uutils: `show_warning!` replaced with a context-stderr + // write; message literalized from locales/en-US.ftl. + let _ = writeln!(pi_uutils_ctx::stderr(), "stat: warning: backslash at end of format"); + return Token::Char('\\'); + } + match chars[*i] { + 'a' => Token::Byte(0x07), // BEL + 'b' => Token::Byte(0x08), // Backspace + 'f' => Token::Byte(0x0c), // Form feed + 'n' => Token::Byte(0x0a), // Line feed + 'r' => Token::Byte(0x0d), // Carriage return + 't' => Token::Byte(0x09), // Horizontal tab + '\\' => Token::Byte(b'\\'), // Backslash + '\'' => Token::Byte(b'\''), // Single quote + '"' => Token::Byte(b'"'), // Double quote + '0'..='7' => { + // Parse octal escape sequence (up to 3 digits) + let mut value = 0u8; + let mut count = 0; + while *i < bound && count < 3 { + if let Some(digit) = chars[*i].to_digit(8) { + value = value * 8 + digit as u8; + *i += 1; + count += 1; + } else { + break; + } + } + *i -= 1; // Adjust index to account for the outer loop increment + Token::Byte(value) + }, + 'x' => { + // Parse hexadecimal escape sequence (\xNN format) + // Uses UTF-8 safe byte indexing to handle multi-byte characters properly + if *i + 1 < bound { + let byte_index = Self::char_index_to_byte_index(format_str, *i + 1); + if let Some((c, offset)) = format_str[byte_index..].scan_char(16) { + *i += offset; + Token::Byte(c as u8) + } else { + // pi-uutils: `show_warning!` replaced with a + // context-stderr write. + let _ = writeln!( + pi_uutils_ctx::stderr(), + "stat: warning: unrecognized escape '\\x'" + ); + Token::Byte(b'x') + } + } else { + // pi-uutils: `show_warning!` replaced with a + // context-stderr write. + let _ = writeln!( + pi_uutils_ctx::stderr(), + "stat: warning: incomplete hex escape '\\x'" + ); + Token::Byte(b'x') + } + }, + other => { + // pi-uutils: `show_warning!` replaced with a context-stderr + // write. + let _ = writeln!( + pi_uutils_ctx::stderr(), + "stat: warning: unrecognized escape '\\{other}'" + ); + Token::Byte(other as u8) + }, + } + } + + fn generate_tokens(format_str: &str, use_printf: bool) -> UResult> { + let mut tokens = Vec::new(); + let chars = format_str.chars().collect::>(); + let bound = chars.len(); + let mut i = 0; + while i < bound { + match chars.get(i) { + Some('%') => { + tokens.push(Self::handle_percent_case(&chars, &mut i, bound, format_str)?); + }, + Some('\\') => { + if use_printf { + tokens.push(Self::handle_escape_sequences(&chars, &mut i, bound, format_str)); + } else { + tokens.push(Token::Char('\\')); + } + }, + Some(c) => tokens.push(Token::Char(*c)), + None => break, + } + i += 1; + } + if !use_printf && !format_str.ends_with('\n') { + tokens.push(Token::Char('\n')); + } + Ok(tokens) + } + + fn populate_mount_list() -> UResult> { + let mut mount_list = read_fs_list() + .map_err(|e| { + USimpleError::new( + e.code(), + StatError::CannotReadFilesystem { error: e.to_string() }.to_string(), + ) + })? + .iter() + .map(|mi| mi.mount_dir.clone()) + .collect::>(); + + // Reverse sort. The longer comes first. + mount_list.sort(); + mount_list.reverse(); + + Ok(mount_list) + } + + fn new(matches: &ArgMatches) -> UResult { + let files: Vec = matches + .get_many::(options::FILES) + .map(|v| v.map(OsString::from).collect()) + .unwrap_or_default(); + if files.is_empty() { + return Err(Box::new(StatError::MissingOperand) as Box); + } + let format_str = if matches.contains_id(options::PRINTF) { + matches + .get_one::(options::PRINTF) + .expect("Invalid format string") + } else { + matches + .get_one::(options::FORMAT) + .map_or("", |s| s.as_str()) + }; + + let use_printf = matches.contains_id(options::PRINTF); + let terse = matches.get_flag(options::TERSE); + let show_fs = matches.get_flag(options::FILE_SYSTEM); + + let default_tokens = if format_str.is_empty() { + Self::generate_tokens(&Self::default_format(show_fs, terse, false), use_printf)? + } else { + Self::generate_tokens(format_str, use_printf)? + }; + let default_dev_tokens = + Self::generate_tokens(&Self::default_format(show_fs, terse, true), use_printf)?; + + // mount points aren't displayed when showing filesystem information, or + // whenever the format string does not request the mount point. + let mount_list_needed = !show_fs + && default_tokens + .iter() + .any(|tok| matches!(tok, Token::Directive { format: 'm', .. })); + + Ok(Self { + follow: matches.get_flag(options::DEREFERENCE), + show_fs, + from_user: !format_str.is_empty(), + files, + mount_list: OnceCell::new(), + mount_list_needed, + default_tokens, + default_dev_tokens, + }) + } + + fn find_mount_point>(&self, p: P) -> Option<&OsString> { + if !self.mount_list_needed { + return None; + } + + let mount_list = self.mount_list.get_or_init(|| { + match Self::populate_mount_list() { + Ok(list) => Some(list), + Err(e) => { + // Show warning like GNU does when mount information cannot be read + // pi-uutils: `show_warning!` replaced with a + // context-stderr write. + let _ = writeln!( + pi_uutils_ctx::stderr(), + "stat: warning: cannot read table of mounted file systems: {e}" + ); + None + }, + } + }); + + let path = p.as_ref().canonicalize().ok()?; + mount_list + .as_ref()? + .iter() + .find(|root| path.starts_with(root)) + } + + fn exec(&self) -> i32 { + let mut stdin_is_fifo = false; + if let Ok(md) = fs::metadata("/dev/stdin") { + stdin_is_fifo = md.file_type().is_fifo(); + } + + let mut ret = 0; + for f in &self.files { + ret |= self.do_stat(f, stdin_is_fifo); + } + ret + } + + fn process_token_files( + &self, + t: &Token, + meta: &Metadata, + display_name: &str, + // pi-uutils: takes the operand resolved against the shell working + // directory for the `%m`/`%N` syscalls (upstream passed the raw + // operand); display output keeps `display_name` as typed. The + // SELinux `follow_symbolic_links` parameter is dropped along with + // SELinux support. + resolved: &Path, + file_type: FileType, + from_user: bool, + ) -> Result<(), i32> { + match *t { + Token::Byte(byte) => write_raw_byte(byte), + // pi-uutils: context stdout instead of `print!`. + Token::Char(c) => { + let _ = write!(pi_uutils_ctx::stdout(), "{c}"); + }, + + Token::Directive { flag, width, precision, format } => { + let output = match format { + // access rights in octal + 'a' => OutputType::UnsignedOct(0o7777 & meta.mode()), + // access rights in human readable form + 'A' => OutputType::Str(display_permissions(meta, true)), + // number of blocks allocated (see %B) + 'b' => OutputType::Unsigned(meta.blocks()), + + // the size in bytes of each block reported by %b + // FIXME: blocksize differs on various platform + // See coreutils/gnulib/lib/stat-size.h ST_NBLOCKSIZE // + // spell-checker:disable-line + 'B' => OutputType::Unsigned(512), + // SELinux security context string + // pi-uutils: SELinux support is dropped; this is + // upstream's non-SELinux fallback string. + 'C' => OutputType::Str("unsupported for this operating system".to_string()), + // device number in decimal + 'd' if flag.major => OutputType::Unsigned(major(meta.dev() as _) as u64), + 'd' if flag.minor => OutputType::Unsigned(minor(meta.dev() as _) as u64), + 'd' => OutputType::Unsigned(meta.dev()), + // device number in hex + 'D' => OutputType::UnsignedHex(meta.dev()), + // raw mode in hex + 'f' => OutputType::UnsignedHex(meta.mode() as u64), + // file type + 'F' => OutputType::Str(pretty_filetype(meta.mode() as mode_t, meta.len())), + // group ID of owner + 'g' => OutputType::Unsigned(meta.gid() as u64), + // group name of owner + 'G' => { + let group_name = + entries::gid2grp(meta.gid()).unwrap_or_else(|_| "UNKNOWN".to_owned()); + OutputType::Str(group_name) + }, + // number of hard links + 'h' => OutputType::Unsigned(meta.nlink()), + // inode number + 'i' => OutputType::Unsigned(meta.ino()), + // mount point + 'm' => match self.find_mount_point(resolved) { + Some(s) => OutputType::OsStr(s), + None => OutputType::Str(String::new()), + }, + // file name + 'n' => OutputType::Str(display_name.to_string()), + // quoted file name with dereference if symbolic link + 'N' => { + let file_name = + get_quoted_file_name(display_name, resolved, file_type, from_user)?; + OutputType::Str(file_name) + }, + // optimal I/O transfer size hint + 'o' => OutputType::Unsigned(meta.blksize()), + // total size, in bytes + 's' => OutputType::Integer(meta.len() as i64), + // major device type in hex, for character/block device special + // files + 't' => OutputType::UnsignedHex(major(meta.rdev() as _) as u64), + // minor device type in hex, for character/block device special + // files + 'T' => OutputType::UnsignedHex(minor(meta.rdev() as _) as u64), + // user ID of owner + 'u' => OutputType::Unsigned(meta.uid() as u64), + // user name of owner + 'U' => { + let user_name = + entries::uid2usr(meta.uid()).unwrap_or_else(|_| "UNKNOWN".to_owned()); + OutputType::Str(user_name) + }, + + // time of file birth, human-readable; - if unknown + 'w' => OutputType::Str(pretty_time(meta, MetadataTimeField::Birth)), + + // time of file birth, seconds since Epoch; 0 if unknown + 'W' => OutputType::Integer( + metadata_get_time(meta, MetadataTimeField::Birth) + .map_or(0, |x| system_time_to_sec(x).0), + ), + + // time of last access, human-readable + 'x' => OutputType::Str(pretty_time(meta, MetadataTimeField::Access)), + // time of last access, seconds since Epoch + 'X' => { + let (sec, nsec) = metadata_get_time(meta, MetadataTimeField::Access) + .map_or((0, 0), system_time_to_sec); + OutputType::Float(sec as f64 + nsec as f64 / 1_000_000_000.0) + }, + // time of last data modification, human-readable + 'y' => OutputType::Str(pretty_time(meta, MetadataTimeField::Modification)), + // time of last data modification, seconds since Epoch + 'Y' => { + let (sec, nsec) = metadata_get_time(meta, MetadataTimeField::Modification) + .map_or((0, 0), system_time_to_sec); + OutputType::Float(sec as f64 + nsec as f64 / 1_000_000_000.0) + }, + // time of last status change, human-readable + 'z' => OutputType::Str(pretty_time(meta, MetadataTimeField::Change)), + // time of last status change, seconds since Epoch + 'Z' => { + let (sec, nsec) = metadata_get_time(meta, MetadataTimeField::Change) + .map_or((0, 0), system_time_to_sec); + OutputType::Float(sec as f64 + nsec as f64 / 1_000_000_000.0) + }, + 'R' => OutputType::UnsignedHex(meta.rdev()), + 'r' if flag.major => OutputType::Unsigned(major(meta.rdev() as _) as u64), + 'r' if flag.minor => OutputType::Unsigned(minor(meta.rdev() as _) as u64), + 'r' => OutputType::Unsigned(meta.rdev()), + _ => OutputType::Unknown, + }; + print_it(&output, flag, width, precision); + }, + } + Ok(()) + } + + fn do_stat(&self, file: &OsStr, stdin_is_fifo: bool) -> i32 { + let display_name = file.to_string_lossy(); + let file = if display_name == "-" { + if self.show_fs { + // pi-uutils: `show_error!` replaced with a context-stderr + // write. + let _ = + writeln!(pi_uutils_ctx::stderr(), "stat: {}", StatError::StdinFilesystemMode); + return 1; + } + if let Ok(p) = Path::new("/dev/stdin").canonicalize() { + p.into_os_string() + } else { + OsString::from("/dev/stdin") + } + } else { + OsString::from(file) + }; + // pi-uutils: resolve the operand against the shell working + // directory for every syscall below; `display_name` keeps the + // operand as typed for `%n` and error messages. + let resolved = pi_uutils_ctx::resolve(&file); + if self.show_fs { + match statfs(resolved.as_os_str()) { + Ok(meta) => { + let tokens = &self.default_tokens; + + // Usage + for t in tokens { + process_token_filesystem(t, &meta, &display_name); + } + }, + Err(error) => { + // pi-uutils: `show_error!` replaced with a + // context-stderr write. + let _ = writeln!( + pi_uutils_ctx::stderr(), + "stat: {}", + StatError::CannotReadFilesystemInfo { + file: display_name.quote().to_string(), + error, + } + ); + return 1; + }, + } + } else { + let follow_symbolic_links = self.follow || stdin_is_fifo && display_name == "-"; + let result = if follow_symbolic_links { + fs::metadata(&resolved) + } else { + fs::symlink_metadata(&resolved) + }; + match result { + Ok(meta) => { + let file_type = meta.file_type(); + let tokens = if self.from_user + || !(file_type.is_char_device() || file_type.is_block_device()) + { + &self.default_tokens + } else { + &self.default_dev_tokens + }; + + for t in tokens { + if let Err(code) = self.process_token_files( + t, + &meta, + &display_name, + &resolved, + file_type, + self.from_user, + ) { + return code; + } + } + }, + Err(e) => { + // pi-uutils: `show_error!` replaced with a + // context-stderr write. + let _ = writeln!(pi_uutils_ctx::stderr(), "stat: {}", StatError::CannotStat { + file: display_name.quote().to_string(), + error: e.to_string(), + }); + return 1; + }, + } + } + 0 + } + + fn default_format(show_fs: bool, terse: bool, show_dev_type: bool) -> String { + // SELinux related format is *ignored* + // pi-uutils: `translate!` word lookups literalized from + // locales/en-US.ftl. + + if show_fs { + if terse { + "%n %i %l %t %s %S %b %f %a %c %d\n".into() + } else { + " File: \"%n\"\n ID: %-8i Namelen: %-7l Type: %T\nBlock size: %-10s Fundamental \ + block size: %S\nBlocks: Total: %-10b Free: %-10f Available: %a\nInodes: Total: \ + %-10c Free: %d\n" + .into() + } + } else if terse { + "%n %s %b %f %u %g %D %i %h %t %T %X %Y %Z %W %o\n".into() + } else { + let device_line = if show_dev_type { + "Device: %Hd,%Ld\tInode: %-10i Links: %-5h Device type: %t,%T\n" + } else { + "Device: %Hd,%Ld\tInode: %-10i Links: %h\n" + }; + + format!( + " File: %N\n size: %-10s\tBlocks: %-10b IO Block: %-6o %F\n{device_line}Access: \ + (%04a/%10.10A) Uid: (%5u/%8U) Gid: (%5g/%8G)\nAccess: %x\nModify: %y\nChange: \ + %z\n Birth: %w\n" + ) + } + } + } + + /// In-process builtin entry point. Unlike upstream's `uumain`, this parses + /// the arguments directly (without the uucore clap-localization helper that + /// would terminate the process), renders clap help/usage/version to the + /// context streams, and maps the `UResult` to an exit code, so it is safe + /// to run inside the host shell process. + pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match stat_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + // pi-uutils: `do_stat` already reports its errors to the + // context stderr and surfaces a bare exit-code error that + // renders to an empty message; don't emit a dangling + // "stat: " prefix for it. + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "stat: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } + } + + fn stat_main(matches: &ArgMatches) -> UResult<()> { + let stater = Stater::new(matches)?; + let exit_status = stater.exec(); + if exit_status == 0 { + Ok(()) + } else { + Err(exit_status.into()) + } + } + + pub fn uu_app() -> Command { + Command::new("stat") + .version(uucore::crate_version!()) + .about(ABOUT) + .after_help(AFTER_HELP) + .override_usage(format_usage(USAGE)) + .infer_long_args(true) + .arg( + Arg::new(options::DEREFERENCE) + .short('L') + .long(options::DEREFERENCE) + .help("follow links") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::FILE_SYSTEM) + .short('f') + .long(options::FILE_SYSTEM) + .help("display file system status instead of file status") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::TERSE) + .short('t') + .long(options::TERSE) + .help("print the information in terse form") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::FORMAT) + .short('c') + .long(options::FORMAT) + .help( + "use the specified FORMAT instead of the default;\noutput a newline after each \ + use of FORMAT", + ) + .value_name("FORMAT"), + ) + .arg( + Arg::new(options::PRINTF) + .long(options::PRINTF) + .value_name("FORMAT") + .help( + "like --format, but interpret backslash escapes,\nand do not output a mandatory \ + trailing newline;\nif you want a newline, include \\n in FORMAT", + ), + ) + .arg( + Arg::new(options::FILES) + .action(ArgAction::Append) + .value_parser(ValueParser::os_string()) + .value_hint(clap::ValueHint::FilePath), + ) + } + + const PRETTY_DATETIME_FORMAT: &str = "%Y-%m-%d %H:%M:%S.%N %z"; + + fn pretty_time(meta: &Metadata, md_time_field: MetadataTimeField) -> String { + if let Some(time) = metadata_get_time(meta, md_time_field) { + let mut tmp = Vec::new(); + if format_system_time( + &mut tmp, + time, + PRETTY_DATETIME_FORMAT, + FormatSystemTimeFallback::Float, + ) + .is_ok() + { + return String::from_utf8(tmp).unwrap(); + } + } + "-".to_string() + } + + /// Upstream format-parser unit tests, kept because the token parser is the + /// most intricate part of the utility and the print paths were repatched. + #[cfg(test)] + mod unit_tests { + use super::{Flags, Precision, ScanUtil, Stater, Token, group_num, precision_trunc}; + + #[test] + fn test_scanners() { + assert_eq!(Some((-5, 2)), "-5zxc".scan_num::()); + assert_eq!(Some((51, 2)), "51zxc".scan_num::()); + assert_eq!(Some((192, 4)), "+192zxc".scan_num::()); + assert_eq!(None, "z192zxc".scan_num::()); + + assert_eq!(Some(('a', 3)), "141zxc".scan_char(8)); + assert_eq!(Some(('\n', 2)), "12qzxc".scan_char(8)); // spell-checker:disable-line + assert_eq!(Some(('\r', 1)), "dqzxc".scan_char(16)); // spell-checker:disable-line + assert_eq!(None, "z2qzxc".scan_char(8)); // spell-checker:disable-line + } + + #[test] + fn test_group_num() { + assert_eq!("12,379,821,234", group_num("12379821234")); + assert_eq!("821,234", group_num("821234")); + assert_eq!("1,234", group_num("1234")); + assert_eq!("234", group_num("234")); + assert_eq!("", group_num("")); + assert_eq!("-5", group_num("-5")); + assert_eq!("-1,234", group_num("-1234")); + } + + #[test] + fn normal_format() { + let s = "%'010.2ac%-#5.w\n"; + let expected = vec![ + Token::Directive { + flag: Flags { group: true, zero: true, ..Default::default() }, + width: 10, + precision: Precision::Number(2), + format: 'a', + }, + Token::Char('c'), + Token::Directive { + flag: Flags { left: true, alter: true, ..Default::default() }, + width: 5, + precision: Precision::NoNumber, + format: 'w', + }, + Token::Char('\n'), + ]; + assert_eq!(&expected, &Stater::generate_tokens(s, false).unwrap()); + } + + #[test] + fn printf_format() { + let s = r#"%-# 15a\t\r\"\\\a\b\x1B\f\x0B%+020.-23w\x12\167\132\112\n"#; + let expected = vec![ + Token::Directive { + flag: Flags { left: true, alter: true, space: true, ..Default::default() }, + width: 15, + precision: Precision::NotSpecified, + format: 'a', + }, + Token::Byte(b'\t'), + Token::Byte(b'\r'), + Token::Byte(b'"'), + Token::Byte(b'\\'), + Token::Byte(b'\x07'), + Token::Byte(b'\x08'), + Token::Byte(b'\x1B'), + Token::Byte(b'\x0C'), + Token::Byte(b'\x0B'), + Token::Directive { + flag: Flags { sign: true, zero: true, ..Default::default() }, + width: 20, + precision: Precision::NotSpecified, + format: 'w', + }, + Token::Byte(b'\x12'), + Token::Byte(b'w'), + Token::Byte(b'Z'), + Token::Byte(b'J'), + Token::Byte(b'\n'), + ]; + assert_eq!(&expected, &Stater::generate_tokens(s, true).unwrap()); + } + + #[test] + fn test_precision_trunc() { + assert_eq!(precision_trunc(123.456, Precision::NotSpecified), "123"); + assert_eq!(precision_trunc(123.456, Precision::NoNumber), "123.456"); + assert_eq!(precision_trunc(123.456, Precision::Number(0)), "123"); + assert_eq!(precision_trunc(123.456, Precision::Number(1)), "123.4"); + assert_eq!(precision_trunc(123.456, Precision::Number(5)), "123.45600"); + } + } +} + +#[cfg(all(test, unix))] +mod tests { + use std::{collections::HashMap, ffi::OsString, fs, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::run; + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("stat") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var, + /// which mount-point/canonicalize logic would otherwise expand + /// mid-assertion). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + #[test] + fn resolves_relative_operand_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("data.bin"), b"hello world!").unwrap(); + + // Relative operand + scope cwd differing from the process cwd: only the + // call-site `pi_uutils_ctx::resolve` patch makes this find the file. + let (code, stdout, stderr) = run_in(root, vec!["-c", "%s", "data.bin"]); + assert_eq!(code, 0); + assert_eq!(stdout, "12\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn percent_n_prints_operand_as_typed() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("data.bin"), b"x").unwrap(); + + // GNU prints the file name exactly as typed, not the resolved path. + let (code, stdout, stderr) = run_in(root, vec!["-c", "%n", "data.bin"]); + assert_eq!(code, 0); + assert_eq!(stdout, "data.bin\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn dereference_switches_between_link_and_target() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("target"), b"abc").unwrap(); + std::os::unix::fs::symlink("target", root.join("link")).unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-c", "%F", "link"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "symbolic link\n", "")); + + let (code, stdout, stderr) = run_in(root, vec!["-L", "-c", "%F", "link"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "regular file\n", "")); + } + + #[test] + fn nonexistent_file_reports_cannot_stat() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root, vec!["missing"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(stderr.starts_with("stat: cannot stat 'missing':"), "unexpected stderr: {stderr:?}"); + } + + #[test] + fn file_system_mode_succeeds() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root, vec!["-f", "-c", "%S", "."]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + assert!( + stdout.trim_end().parse::().is_ok(), + "fundamental block size should be numeric: {stdout:?}" + ); + } + + #[test] + fn printf_controls_trailing_newline_and_escapes() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("data.bin"), b"hello world!").unwrap(); + + // --printf emits no mandatory trailing newline... + let (code, stdout, _) = run_in(root.clone(), vec!["--printf", "%s", "data.bin"]); + assert_eq!((code, stdout.as_str()), (0, "12")); + + // ...but interprets backslash escapes. + let (code, stdout, _) = run_in(root, vec!["--printf", r"%s\t%n\n", "data.bin"]); + assert_eq!((code, stdout.as_str()), (0, "12\tdata.bin\n")); + } + + #[test] + fn terse_prints_name_as_typed_and_size() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("data.bin"), b"hello world!").unwrap(); + + let (code, stdout, stderr) = run_in(root, vec!["-t", "data.bin"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let fields: Vec<&str> = stdout.split_whitespace().collect(); + assert_eq!(fields[0], "data.bin"); + assert_eq!(fields[1], "12"); + assert_eq!(fields.len(), 16, "terse format has 16 fields: {stdout:?}"); + } + + #[test] + fn missing_operand_is_error() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec![]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(stderr.contains("stat: missing operand"), "unexpected stderr: {stderr:?}"); + assert!(stderr.contains("Try 'stat --help'"), "unexpected stderr: {stderr:?}"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("file system status")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-tac/Cargo.toml b/crates/vendor/uu-tac/Cargo.toml new file mode 100644 index 000000000..c7bd7c526 --- /dev/null +++ b/crates/vendor/uu-tac/Cargo.toml @@ -0,0 +1,26 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/tac), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/tac.rs +# for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_tac" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "tac ~ (uutils) concatenate and display input lines in reverse order (vendored + patched for in-process embedding)" + +[lib] +path = "src/tac.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +memchr = "2.7.4" +memmap2 = "0.9" +regex = "1.11" +thiserror = "2.0.3" +uucore = "0.8.0" +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-tac/LICENSE b/crates/vendor/uu-tac/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-tac/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-tac/src/error.rs b/crates/vendor/uu-tac/src/error.rs new file mode 100644 index 000000000..cc58fd224 --- /dev/null +++ b/crates/vendor/uu-tac/src/error.rs @@ -0,0 +1,47 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +//! Errors returned by tac during processing of a file. + +// pi-uutils: vendored from uutils/coreutils 0.8.0; `translate!` strings are +// literalized with the en-US locale text. + +use std::ffi::OsString; + +use thiserror::Error; +use uucore::{ + display::Quotable, + error::{UError, strip_errno}, +}; + +#[derive(Debug, Error)] +pub enum TacError { + /// A regular expression given by the user is invalid. + #[error("invalid regular expression: {0}")] + InvalidRegex(regex::Error), + /// An error opening a file for reading. + /// + /// The parameters are the name of the file and the underlying + /// [`std::io::Error`] that caused this error. + #[error("failed to open {} for reading: {}", .0.quote(), strip_errno(.1))] + OpenError(OsString, std::io::Error), + /// An error reading the contents of a file or stdin. + /// + /// The parameters are the name of the file and the underlying + /// [`std::io::Error`] that caused this error. + #[error("{}: read error: {}", .0.maybe_quote(), strip_errno(.1))] + ReadError(OsString, std::io::Error), + /// An error writing the (reversed) contents of a file or stdin. + /// + /// The parameter is the underlying [`std::io::Error`] that caused + /// this error. + #[error("failed to write to stdout: {}", strip_errno(.0))] + WriteError(std::io::Error), +} + +impl UError for TacError { + fn code(&self) -> i32 { + 1 + } +} diff --git a/crates/vendor/uu-tac/src/tac.rs b/crates/vendor/uu-tac/src/tac.rs new file mode 100644 index 000000000..90c124e61 --- /dev/null +++ b/crates/vendor/uu-tac/src/tac.rs @@ -0,0 +1,634 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) sbytes slen dlen memmem memmap Mmap mmap SIGBUS + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. FILE operands resolve against the shell working directory +// via `pi_uutils_ctx::resolve` at the open/mmap call site (the original operand +// is kept for error messages), `-`/no-operand read the context stdin, output is +// written through the context stdout, recoverable per-file errors go to the +// context stderr with `pi_uutils_ctx::set_exit_code` (upstream `show!`), the +// `translate!` strings are literalized, and the process-global signal handling +// plus the stdin mmap/tempfile buffering (which target the process stdin fd) +// are removed. + +mod error; + +use std::{ + ffi::{OsStr, OsString}, + fs::File, + io::{BufWriter, Read, Write}, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use memchr::memmem; +use memmap2::Mmap; +use pi_uutils_ctx::format_usage; +use uucore::error::UResult; + +use crate::error::TacError; + +mod options { + pub static BEFORE: &str = "before"; + pub static REGEX: &str = "regex"; + pub static SEPARATOR: &str = "separator"; + pub static FILE: &str = "file"; +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match tac_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "tac: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn tac_main(matches: &ArgMatches) -> UResult<()> { + let before = matches.get_flag(options::BEFORE); + let regex = matches.get_flag(options::REGEX); + let raw_separator = matches + .get_one::(options::SEPARATOR) + .map_or(OsStr::new("\n"), |s| s.as_os_str()); + + let separator = if raw_separator.is_empty() { + OsStr::new("\0") + } else { + raw_separator + }; + + let files: Vec = match matches.get_many::(options::FILE) { + Some(v) => v.cloned().collect(), + None => vec![OsString::from("-")], + }; + + tac(&files, before, regex, separator) +} + +pub fn uu_app() -> Command { + Command::new("tac") + .version(uucore::crate_version!()) + .override_usage(format_usage("tac [OPTION]... [FILE]...")) + .about("Write each file to standard output, last line first.") + .infer_long_args(true) + .arg( + Arg::new(options::BEFORE) + .short('b') + .long(options::BEFORE) + .help("attach the separator before instead of after") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::REGEX) + .short('r') + .long(options::REGEX) + .help("interpret the sequence as a regular expression") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::SEPARATOR) + .short('s') + .long(options::SEPARATOR) + .help("use STRING as the separator instead of newline") + .value_parser(clap::value_parser!(OsString)) + .value_name("STRING"), + ) + .arg( + Arg::new(options::FILE) + .hide(true) + .action(ArgAction::Append) + .value_parser(clap::value_parser!(OsString)) + .value_hint(clap::ValueHint::FilePath), + ) +} + +/// pi-uutils: replacement for upstream's `show!` — reports a recoverable +/// per-file error to the context stderr and accumulates a non-zero exit code +/// while processing continues with the next operand. +fn show(err: &TacError) { + let _ = writeln!(pi_uutils_ctx::stderr(), "tac: {err}"); + pi_uutils_ctx::set_exit_code(1); +} + +/// Print lines of a buffer in reverse, with line separator given as a regex. +/// +/// `data` contains the bytes of the file. +/// +/// `pattern` is the regular expression given as a +/// [`regex::bytes::Regex`] (not a [`regex::Regex`], since the input is +/// given as a slice of bytes). If `before` is `true`, then each match +/// of this pattern in `data` is interpreted as the start of a line. If +/// `before` is `false`, then each match of this pattern is interpreted +/// as the end of a line. +/// +/// This function writes each line in `data` to the context stdout in +/// reverse. +/// +/// # Errors +/// +/// If there is a problem writing to stdout, then this function +/// returns [`std::io::Error`]. +fn buffer_tac_regex( + data: &[u8], + pattern: ®ex::bytes::Regex, + before: bool, +) -> std::io::Result<()> { + // pi-uutils: write through the context stdout instead of the process stdout. + let mut out = BufWriter::new(pi_uutils_ctx::stdout()); + + // The index of the line separator for the current line. + // + // As we scan through the `data` from right to left, we update this + // variable each time we find a new line separator. We restrict our + // regular expression search to only those bytes up to the line + // separator. + let mut this_line_end = data.len(); + + // The index of the start of the next line in the `data`. + // + // As we scan through the `data` from right to left, we update this + // variable each time we find a new line. + // + // If `before` is `true`, then each line starts immediately before + // the line separator. Otherwise, each line starts immediately after + // the line separator. + let mut following_line_start = data.len(); + + // Iterate over each byte in the buffer in reverse. When we find a + // line separator, write the line to stdout. + // + // The `before` flag controls whether the line separator appears at + // the end of the line (as in "abc\ndef\n") or at the beginning of + // the line (as in "/abc/def"). + for i in (0..data.len()).rev() { + // Determine if there is a match for `pattern` starting at index + // `i` in `data`. Only search up to the line ending that was + // found previously. + if let Some(match_) = pattern.find_at(&data[..this_line_end], i) + && match_.start() == i + { + // Record this index as the ending of the current line. + this_line_end = i; + + // The length of the match (that is, the line separator), in bytes. + let slen = match_.end() - match_.start(); + + if before { + out.write_all(&data[i..following_line_start])?; + following_line_start = i; + } else { + out.write_all(&data[i + slen..following_line_start])?; + following_line_start = i + slen; + } + } + } + + // After the loop terminates, write whatever bytes are remaining at + // the beginning of the buffer. + out.write_all(&data[0..following_line_start])?; + out.flush()?; + Ok(()) +} + +/// Write lines from `data` to stdout in reverse. +/// +/// This function writes to the context stdout each line appearing in `data`, +/// starting with the last line and ending with the first line. The +/// `separator` parameter defines what characters to use as a line +/// separator. +/// +/// If `before` is `false`, then this function assumes that the +/// `separator` appears at the end of each line, as in `"abc\ndef\n"`. +/// If `before` is `true`, then this function assumes that the +/// `separator` appears at the beginning of each line, as in +/// `"/abc/def"`. +fn buffer_tac(data: &[u8], before: bool, separator: &OsStr) -> std::io::Result<()> { + // pi-uutils: write through the context stdout instead of the process stdout. + let mut out = BufWriter::new(pi_uutils_ctx::stdout()); + + // The number of bytes in the line separator. + let slen = separator.len(); + + // The index of the start of the next line in the `data`. + // + // As we scan through the `data` from right to left, we update this + // variable each time we find a new line. + // + // If `before` is `true`, then each line starts immediately before + // the line separator. Otherwise, each line starts immediately after + // the line separator. + let mut following_line_start = data.len(); + + // Iterate over each byte in the buffer in reverse. When we find a + // line separator, write the line to stdout. + // + // The `before` flag controls whether the line separator appears at + // the end of the line (as in "abc\ndef\n") or at the beginning of + // the line (as in "/abc/def"). + for i in memmem::rfind_iter(data, separator.as_encoded_bytes()) { + if before { + out.write_all(&data[i..following_line_start])?; + following_line_start = i; + } else { + out.write_all(&data[i + slen..following_line_start])?; + following_line_start = i + slen; + } + } + + // After the loop terminates, write whatever bytes are remaining at + // the beginning of the buffer. + out.write_all(&data[0..following_line_start])?; + out.flush()?; + Ok(()) +} + +/// Make the regex flavor compatible with `regex` crate +/// +/// Concretely: +/// - Toggle escaping of (), |, {} +/// - Escape ^ and $ when not at edges +/// - Leave only ASCII bytes inside [] +/// - Escape non-ASCII bytes as `(?-u:\xFF)` outside [] +fn translate_regex_flavor(bytes: &[u8]) -> String { + let mut result = Vec::new(); + let mut i = 0; + let mut inside_brackets = false; + let mut prev_was_backslash = false; + let mut last_byte: Option = None; + + while let Some(b) = bytes.get(i) { + let is_escaped = prev_was_backslash; + prev_was_backslash = false; + + match b { + _ if inside_brackets && !b.is_ascii() => { + i += 1; + continue; + }, + // Unescape escaped (), |, {} when not inside brackets + b'\\' if !inside_brackets && !is_escaped => { + if let Some(next) = bytes.get(i + 1) + && matches!(next, b'(' | b')' | b'|' | b'{' | b'}') + { + result.push(*next); + last_byte = Some(*next); + i += 2; + continue; + } + + result.push(b'\\'); + last_byte = Some(b'\\'); + prev_was_backslash = true; + }, + // Bracket tracking + b'[' => { + inside_brackets = true; + result.push(*b); + last_byte = Some(*b); + }, + b']' => { + inside_brackets = false; + result.push(*b); + last_byte = Some(*b); + }, + // Escape (), |, {} when not escaped and outside brackets + b'(' | b')' | b'|' | b'{' | b'}' if !inside_brackets && !is_escaped => { + result.push(b'\\'); + result.push(*b); + last_byte = Some(*b); + }, + b'^' if !inside_brackets && !is_escaped => { + let is_anchor_position = result.is_empty() || matches!(last_byte, Some(b'(' | b'|')); + if !is_anchor_position { + result.push(b'\\'); + } + result.push(*b); + last_byte = Some(*b); + }, + b'$' if !inside_brackets && !is_escaped => { + let next_is_anchor_position = match bytes.get(i + 1) { + None => true, + Some(b')' | b'|') => true, + Some(b'\\') => { + // Peek two ahead to see if it's \) or \| + matches!(bytes.get(i + 2), Some(b')' | b'|')) + }, + _ => false, + }; + if !next_is_anchor_position { + result.push(b'\\'); + } + result.push(*b); + last_byte = Some(*b); + }, + _ if !b.is_ascii() => { + let _ = write!(result, r"(?-u:\x{b:02x})"); + last_byte = None; + }, + _ => { + result.push(*b); + last_byte = Some(*b); + }, + } + + i += 1; + } + + String::from_utf8(result).expect("produces ASCII bytes") +} + +#[allow(clippy::cognitive_complexity)] +fn tac(filenames: &[OsString], before: bool, regex: bool, separator: &OsStr) -> UResult<()> { + // Compile the regular expression pattern if it is provided. + let maybe_pattern = if regex { + match regex::bytes::RegexBuilder::new(&translate_regex_flavor(separator.as_encoded_bytes())) + .multi_line(true) + .build() + { + Ok(p) => Some(p), + Err(e) => return Err(TacError::InvalidRegex(e).into()), + } + } else { + None + }; + + for filename in filenames { + let mmap; + let buf; + + let data: &[u8] = if filename == "-" { + // pi-uutils: in-process stdin is a context stream, not the process + // stdin fd; upstream's stdin mmap / tempfile buffering and the + // `stdin_was_closed` signal check do not apply. Read it fully. + let mut contents = Vec::new(); + match pi_uutils_ctx::stdin().read_to_end(&mut contents) { + Ok(_) => { + buf = contents; + &buf + }, + Err(e) => { + show(&TacError::ReadError(OsString::from("stdin"), e)); + continue; + }, + } + } else { + // pi-uutils: resolve the operand against the shell working + // directory at the open site; `filename` is kept for errors. + let path = pi_uutils_ctx::resolve(filename); + let mut file = match File::open(&path) { + Ok(f) => f, + Err(e) => { + show(&TacError::OpenError(filename.clone(), e)); + continue; + }, + }; + + if let Some(mmap1) = try_mmap_file(&file) { + mmap = mmap1; + &mmap + } else { + let mut contents = Vec::new(); + match file.read_to_end(&mut contents) { + Ok(_) => { + buf = contents; + &buf + }, + Err(e) => { + show(&TacError::ReadError(filename.clone(), e)); + continue; + }, + } + } + }; + + // Select the appropriate `tac` algorithm based on whether the + // separator is given as a regular expression or a fixed string. + // pi-uutils: match ergonomics instead of upstream's `Some(ref pattern)`. + let result = match &maybe_pattern { + Some(pattern) => buffer_tac_regex(data, pattern, before), + None => buffer_tac(data, before, separator), + }; + + // If there is any error in writing the output, terminate immediately. + if let Err(e) = result { + return Err(TacError::WriteError(e).into()); + } + } + Ok(()) +} + +fn try_mmap_file(file: &File) -> Option { + // SAFETY: If the file is truncated while we map it, SIGBUS will be raised + // and our process will be terminated, thus preventing access of invalid memory. + unsafe { Mmap::map(file).ok() } +} + +#[cfg(test)] +mod tests_hybrid_flavor { + use super::translate_regex_flavor; + + #[test] + fn test_grouping_and_alternation() { + assert_eq!(translate_regex_flavor(br"\(abc\)"), r"(abc)"); + + assert_eq!(translate_regex_flavor(br"(abc)"), r"\(abc\)"); + + assert_eq!(translate_regex_flavor(br"a\|b"), r"a|b"); + + assert_eq!(translate_regex_flavor(br"a|b"), r"a\|b"); + } + + #[test] + fn test_anchors_context() { + assert_eq!(translate_regex_flavor(br"^abc$"), r"^abc$"); + + assert_eq!(translate_regex_flavor(br"a^b"), r"a\^b"); + assert_eq!(translate_regex_flavor(br"a$b"), r"a\$b"); + + // Anchors inside groups (reset by \(...\) regardless of position) + assert_eq!(translate_regex_flavor(br"\(^abc\)"), r"(^abc)"); + assert_eq!(translate_regex_flavor(br"\(abc$\)"), r"(abc$)"); + + // Anchors inside alternation (reset by \| regardless of position) + assert_eq!(translate_regex_flavor(br"^a\|^b"), r"^a|^b"); + assert_eq!(translate_regex_flavor(br"a$\|b$"), r"a$|b$"); + } + + #[test] + fn test_character_classes() { + assert_eq!(translate_regex_flavor(br"[a-z]"), r"[a-z]"); + + assert_eq!(translate_regex_flavor(br"[.]"), r"[.]"); + + assert_eq!(translate_regex_flavor(br"[]abc]"), r"[]abc]"); + + assert_eq!(translate_regex_flavor(br"[^]abc]"), r"[^]abc]"); + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, fs, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_with(cwd: PathBuf, stdin: &[u8], args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::Cursor::new(stdin.to_vec())), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("tac") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + #[test] + fn resolves_relative_operand_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("input.txt"), b"a\nb\nc\n").unwrap(); + + // Relative operand + scope cwd differing from the process cwd: only the + // call-site `pi_uutils_ctx::resolve` patch makes this find the file. + let (code, stdout, stderr) = run_with(root, b"", vec!["input.txt"]); + assert_eq!(code, 0); + assert_eq!(stdout, "c\nb\na\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn no_operand_reads_context_stdin() { + let (code, stdout, stderr) = run_with(PathBuf::from("."), b"one\ntwo\nthree\n", vec![]); + assert_eq!(code, 0); + assert_eq!(stdout, "three\ntwo\none\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn dash_operand_reads_context_stdin() { + let (code, stdout, stderr) = run_with(PathBuf::from("."), b"x\ny\n", vec!["-"]); + assert_eq!(code, 0); + assert_eq!(stdout, "y\nx\n"); + assert_eq!(stderr, ""); + } + + #[test] + fn custom_separator_reverses_fields() { + let (code, stdout, stderr) = run_with(PathBuf::from("."), b"a,b,c,", vec!["-s", ","]); + assert_eq!(code, 0); + assert_eq!(stdout, "c,b,a,"); + assert_eq!(stderr, ""); + } + + #[test] + fn before_flag_attaches_separator_before_each_line() { + let (code, stdout, stderr) = run_with(PathBuf::from("."), b"/abc/def", vec!["-b", "-s", "/"]); + assert_eq!(code, 0); + assert_eq!(stdout, "/def/abc"); + assert_eq!(stderr, ""); + } + + #[test] + fn regex_separator_splits_on_character_class() { + // `[,;]` treats either byte as a separator; records are emitted in + // reverse with each separator kept attached to its preceding record. + let (code, stdout, stderr) = run_with(PathBuf::from("."), b"a,b;c", vec!["-r", "-s", "[,;]"]); + assert_eq!(code, 0); + assert_eq!(stdout, "cb;a,"); + assert_eq!(stderr, ""); + } + + #[test] + fn invalid_regex_is_fatal_error() { + let (code, stdout, stderr) = run_with(PathBuf::from("."), b"abc", vec!["-r", "-s", "["]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(stderr.starts_with("tac: invalid regular expression:"), "stderr: {stderr}"); + } + + #[test] + fn missing_file_continues_with_next_operand_and_exits_nonzero() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("good.txt"), b"1\n2\n").unwrap(); + + let (code, stdout, stderr) = run_with(root, b"", vec!["nope.txt", "good.txt"]); + assert_eq!(code, 1); + assert_eq!(stdout, "2\n1\n", "valid operand still printed after the failure"); + assert!(stderr.contains("tac: failed to open 'nope.txt' for reading:"), "stderr: {stderr}"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_with(PathBuf::from("."), b"", vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("last line first")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-touch/Cargo.toml b/crates/vendor/uu-touch/Cargo.toml new file mode 100644 index 000000000..679bfcb87 --- /dev/null +++ b/crates/vendor/uu-touch/Cargo.toml @@ -0,0 +1,36 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/touch), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/touch.rs +# for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_touch" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "touch ~ (uutils) change FILE timestamps (vendored + patched for in-process embedding)" + +[lib] +path = "src/touch.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +filetime = "0.2.23" +jiff = "0.2.18" +parse_datetime = "0.14.0" +thiserror = "2.0.3" +uucore = { version = "0.8.0", features = ["libc", "parser"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[target.'cfg(unix)'.dependencies] +libc = "0.2.172" +rustix = { version = "1.1.4", features = ["fs"] } + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61.0", default-features = false, features = [ + "Win32_Storage_FileSystem", + "Win32_Foundation", +] } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-touch/LICENSE b/crates/vendor/uu-touch/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-touch/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-touch/src/touch.rs b/crates/vendor/uu-touch/src/touch.rs new file mode 100644 index 000000000..4b874d06e --- /dev/null +++ b/crates/vendor/uu-touch/src/touch.rs @@ -0,0 +1,1170 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) datelike datetime filetime lpszfilepath mktime +// strtime timelike utime DATETIME UTIME futimens spell-checker:ignore (FORMATS) +// MMDDhhmm YYYYMMDDHHMM YYMMDDHHMM YYYYMMDDHHMMS + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Every filesystem syscall resolves its path operand +// against the shell working directory via `pi_uutils_ctx::resolve` AT THE CALL +// SITE, while the original operands are kept for display/error messages (GNU +// prints operands as typed). All process-global stdio is routed through +// `pi_uutils_ctx`, `translate!` strings are literalized, `_POSIX2_VERSION` is +// read from the scope environment, `show!` accumulation goes through +// `pi_uutils_ctx::set_exit_code`, and the entry point no longer calls +// `std::process::exit`. Upstream's `src/error.rs` is inlined below as +// `pub mod error`. jiff's `TimeZone::system()` (and thus `TZ`) intentionally +// stays process-global. + +#[cfg(unix)] +use std::fs::OpenOptions; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::{ + borrow::Cow, + ffi::{OsStr, OsString}, + fs::{self, File}, + io::{Error, ErrorKind, Write}, + path::{Path, PathBuf}, + time::SystemTime, +}; + +use clap::{ + Arg, ArgAction, ArgGroup, ArgMatches, Command, + builder::{PossibleValue, ValueParser}, +}; +use filetime::{FileTime, set_file_times, set_symlink_file_times}; +use jiff::{Timestamp, ToSpan, Zoned, civil::Time, fmt::strtime, tz::TimeZone}; +#[cfg(unix)] +use libc::O_NONBLOCK; +use pi_uutils_ctx::format_usage; +#[cfg(unix)] +use rustix::fs::Timestamps; +#[cfg(unix)] +use rustix::fs::futimens; +#[cfg(target_os = "linux")] +use uucore::libc; +use uucore::{ + display::Quotable, + error::{FromIo, UError, UResult, USimpleError}, + parser::shortcut_value_parser::ShortcutValueParser, +}; + +use crate::error::TouchError; + +// pi-uutils: upstream `src/error.rs`, inlined so the vendored crate is a +// single source file. `translate!` message templates are literalized with the +// en-US strings. +pub mod error { + use std::path::PathBuf; + + use filetime::FileTime; + use thiserror::Error; + use uucore::{ + display::Quotable, + error::{UError, UIoError}, + }; + + #[derive(Debug, Error)] + pub enum TouchError { + #[error("Unable to parse date: {0}")] + InvalidDateFormat(String), + + /// The source time couldn't be converted to a [`jiff::Zoned`] + #[error("Source has invalid access or modification time: {0}")] + InvalidFiletime(FileTime), + + /// The reference file's attributes could not be found or read + #[error("failed to get attributes of {}: {}", .0.quote(), to_uioerror(.1))] + ReferenceFileInaccessible(PathBuf, std::io::Error), + + /// An error getting a path to stdout on Windows + #[error("GetFinalPathNameByHandleW failed with code {0}")] + WindowsStdoutPathError(String), + + /// An error encountered on a specific file + #[error("{error}")] + TouchFileError { path: PathBuf, index: usize, error: Box }, + } + + fn to_uioerror(err: &std::io::Error) -> UIoError { + let copy = if let Some(code) = err.raw_os_error() { + std::io::Error::from_raw_os_error(code) + } else { + std::io::Error::from(err.kind()) + }; + UIoError::from(copy) + } + + impl UError for TouchError {} +} + +/// Options contains all the possible behaviors and flags for touch. +/// +/// All options are public so that the options can be programmatically +/// constructed by other crates, such as nushell. That means that this struct is +/// part of our public API. It should therefore not be changed without good +/// reason. +/// +/// The fields are documented with the arguments that determine their value. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Options { + /// Do not create any files. Set by `-c`/`--no-create`. + pub no_create: bool, + + /// Affect each symbolic link instead of any referenced file. Set by + /// `-h`/`--no-dereference`. + pub no_deref: bool, + + /// Where to get access and modification times from + pub source: Source, + + /// If given, uses time from `source` but on given date + pub date: Option, + + /// Whether to change access time only, modification time only, or both + pub change_times: ChangeTimes, + + /// When true, error when file doesn't exist and either `--no-dereference` + /// was passed or the file couldn't be created + pub strict: bool, +} + +pub enum InputFile { + /// A regular file + Path(PathBuf), + /// Touch stdout. `--no-dereference` will be ignored in this case. + Stdout, +} + +/// Whether to set access time only, modification time only, or both +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum ChangeTimes { + /// Change only access time + AtimeOnly, + /// Change only modification time + MtimeOnly, + /// Change both access and modification times + Both, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum Source { + /// Use access/modification times of given file + Reference(PathBuf), + Timestamp(FileTime), + /// Use current time + Now, +} + +pub mod options { + // Both SOURCES and sources are needed as we need to be able to refer to the + // ArgGroup. + pub static SOURCES: &str = "sources"; + pub mod sources { + pub static DATE: &str = "date"; + pub static REFERENCE: &str = "reference"; + pub static TIMESTAMP: &str = "timestamp"; + } + pub static HELP: &str = "help"; + pub static ACCESS: &str = "access"; + pub static MODIFICATION: &str = "modification"; + pub static NO_CREATE: &str = "no-create"; + pub static NO_DEREF: &str = "no-dereference"; + pub static TIME: &str = "time"; + pub static FORCE: &str = "force"; +} + +static ARG_FILES: &str = "files"; + +mod format { + pub(crate) const POSIX_LOCALE: &str = "%a %b %e %H:%M:%S %Y"; + pub(crate) const ISO_8601: &str = "%Y-%m-%d"; + // "%Y%m%d%H%M.%S" 15 chars + pub(crate) const YYYYMMDDHHMM_DOT_SS: &str = "%Y%m%d%H%M.%S"; + // "%Y-%m-%d %H:%M:%S.%SS" 12 chars + pub(crate) const YYYYMMDDHHMMSS: &str = "%Y-%m-%d %H:%M:%S.%f"; + // "%Y-%m-%d %H:%M:%S" 12 chars + pub(crate) const YYYYMMDDHHMMS: &str = "%Y-%m-%d %H:%M:%S"; + // "%Y-%m-%d %H:%M" 12 chars + // Used for example in tests/touch/no-rights.sh + pub(crate) const YYYY_MM_DD_HH_MM: &str = "%Y-%m-%d %H:%M"; + // "%Y%m%d%H%M" 12 chars + pub(crate) const YYYYMMDDHHMM: &str = "%Y%m%d%H%M"; + // "%Y-%m-%d %H:%M +offset" + // Used for example in tests/touch/relative.sh + pub(crate) const YYYYMMDDHHMM_OFFSET: &str = "%Y-%m-%d %H:%M %z"; +} + +fn timestamp_to_filetime(ts: Timestamp) -> FileTime { + FileTime::from_system_time(SystemTime::from(ts)) +} + +fn filetime_to_zoned(ft: &FileTime) -> Option { + let ts = Timestamp::new(ft.unix_seconds(), ft.nanoseconds() as i32).ok()?; + Some(Zoned::new(ts, TimeZone::system())) +} + +/// Whether all characters in the string are digits. +fn all_digits(s: &str) -> bool { + s.as_bytes().iter().all(u8::is_ascii_digit) +} + +/// Convert a two-digit year string to the corresponding number. +/// +/// `s` must be of length two or more. The last two bytes of `s` are +/// assumed to be the two digits of the year. +fn get_year(s: &str) -> u8 { + let bytes = s.as_bytes(); + let n = bytes.len(); + let y1 = bytes[n - 2] - b'0'; + let y2 = bytes[n - 1] - b'0'; + 10 * y1 + y2 +} + +/// Whether the first filename should be interpreted as a timestamp. +fn is_first_filename_timestamp( + reference: Option<&OsString>, + date: Option<&str>, + timestamp: Option<&str>, + files: &[&OsString], +) -> bool { + timestamp.is_none() + && reference.is_none() + && date.is_none() + && files.len() >= 2 + // pi-uutils: `_POSIX2_VERSION` comes from the scope environment (the + // shell's exported variables), not the host process environment. + // env check is last as the slowest op + && pi_uutils_ctx::var("_POSIX2_VERSION").as_deref() == Some("199209") + && files[0].to_str().is_some_and(is_timestamp) +} + +// Check if string is a valid POSIX timestamp (8 digits or 10 digits with valid +// year range) +fn is_timestamp(s: &str) -> bool { + all_digits(s) && (s.len() == 8 || (s.len() == 10 && (69..=99).contains(&get_year(s)))) +} + +/// Cycle the last two characters to the beginning of the string. +/// +/// `s` must have length at least two. +fn shr2(s: &str) -> String { + let n = s.len(); + let (a, b) = s.split_at(n - 2); + let mut result = String::with_capacity(n); + result.push_str(b); + result.push_str(a); + result +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match touch_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "touch: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn touch_main(matches: &ArgMatches) -> UResult<()> { + let mut filenames: Vec<&OsString> = matches + .get_many::(ARG_FILES) + .ok_or_else(|| { + // pi-uutils: literalized; `uucore::execution_phrase()` is "touch" + // when running as a builtin. + USimpleError::new(1, "missing file operand\nTry 'touch --help' for more information.") + })? + .collect(); + + let no_deref = matches.get_flag(options::NO_DEREF); + + let reference = matches.get_one::(options::sources::REFERENCE); + let date = matches + .get_one::(options::sources::DATE) + .map(ToOwned::to_owned); + + let mut timestamp = matches + .get_one::(options::sources::TIMESTAMP) + .map(ToOwned::to_owned); + + if is_first_filename_timestamp(reference, date.as_deref(), timestamp.as_deref(), &filenames) { + let first_file = filenames[0].to_str().unwrap(); + timestamp = if first_file.len() == 10 { + Some(shr2(first_file)) + } else { + Some(first_file.to_string()) + }; + filenames = filenames[1..].to_vec(); + } + + let source = if let Some(reference) = reference { + Source::Reference(PathBuf::from(reference)) + } else if let Some(ts) = timestamp { + Source::Timestamp(parse_timestamp(&ts)?) + } else { + Source::Now + }; + + let files: Vec = filenames + .into_iter() + .map(|filename| { + if filename == "-" { + InputFile::Stdout + } else { + InputFile::Path(PathBuf::from(filename)) + } + }) + .collect(); + + let opts = Options { + no_create: matches.get_flag(options::NO_CREATE), + no_deref, + source, + date, + change_times: determine_atime_mtime_change(matches), + strict: false, + }; + + touch(&files, &opts)?; + + Ok(()) +} + +pub fn uu_app() -> Command { + Command::new("touch") + .version(uucore::crate_version!()) + .about("Update the access and modification times of each FILE to the current time.") + .override_usage(format_usage("touch [OPTION]... [FILE]...")) + .infer_long_args(true) + .disable_help_flag(true) + .arg( + Arg::new(options::HELP) + .long(options::HELP) + .help("Print help information.") + .action(ArgAction::Help), + ) + .arg( + Arg::new(options::ACCESS) + .short('a') + .help("change only the access time") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::sources::TIMESTAMP) + .short('t') + .help("use [[CC]YY]MMDDhhmm[.ss] instead of the current time") + .value_name("STAMP"), + ) + .arg( + Arg::new(options::sources::DATE) + .short('d') + .long(options::sources::DATE) + .allow_hyphen_values(true) + .help("parse argument and use it instead of current time") + .value_name("STRING") + .conflicts_with(options::sources::TIMESTAMP), + ) + .arg( + Arg::new(options::FORCE) + .short('f') + .help("(ignored)") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::MODIFICATION) + .short('m') + .help("change only the modification time") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::NO_CREATE) + .short('c') + .long(options::NO_CREATE) + .help("do not create any files") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::NO_DEREF) + .short('h') + .long(options::NO_DEREF) + .help( + "affect each symbolic link instead of any referenced file (only for systems that \ + can change the timestamps of a symlink)", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::sources::REFERENCE) + .short('r') + .long(options::sources::REFERENCE) + .help("use this file's times instead of the current time") + .value_name("FILE") + .value_parser(ValueParser::os_string()) + .value_hint(clap::ValueHint::AnyPath) + .conflicts_with(options::sources::TIMESTAMP), + ) + .arg( + Arg::new(options::TIME) + .long(options::TIME) + .help( + "change only the specified time: \"access\", \"atime\", or \"use\" are equivalent \ + to -a; \"modify\" or \"mtime\" are equivalent to -m", + ) + .value_name("WORD") + .value_parser(ShortcutValueParser::new([ + PossibleValue::new("atime").alias("access").alias("use"), + PossibleValue::new("mtime").alias("modify"), + ])), + ) + .arg( + Arg::new(ARG_FILES) + .action(ArgAction::Append) + .num_args(1..) + .value_parser(clap::value_parser!(OsString)) + .value_hint(clap::ValueHint::AnyPath), + ) + .group( + ArgGroup::new(options::SOURCES) + .args([ + options::sources::TIMESTAMP, + options::sources::DATE, + options::sources::REFERENCE, + ]) + .multiple(true), + ) +} + +/// Execute the touch command. +/// +/// # Errors +/// +/// Possible causes: +/// - The user doesn't have permission to access the file +/// - One of the directory components of the file path doesn't exist. +/// - Dangling symlink is given and -r/--reference is used. +/// +/// It will return an `Err` on the first error. However, for any of the files, +/// if all of the following are true, it will print the error and continue +/// touching the rest of the files. +/// - `opts.strict` is `false` +/// - The file doesn't already exist +/// - `-c`/`--no-create` was passed (`opts.no_create`) +/// - Either `-h`/`--no-dereference` was passed (`opts.no_deref`) or the file +/// couldn't be created +pub fn touch(files: &[InputFile], opts: &Options) -> Result<(), TouchError> { + let (atime, mtime) = match &opts.source { + Source::Reference(reference) => { + // pi-uutils: resolve the reference operand against the shell + // working directory at the syscall site; the original operand is + // kept for the error message. + let (atime, mtime) = stat(&pi_uutils_ctx::resolve(reference), !opts.no_deref) + .map_err(|e| TouchError::ReferenceFileInaccessible(reference.to_owned(), e))?; + + (atime, mtime) + }, + Source::Now => { + let now: FileTime; + #[cfg(target_os = "linux")] + { + if opts.date.is_none() { + now = FileTime::from_unix_time(0, libc::UTIME_NOW as u32); + } else { + now = timestamp_to_filetime(Timestamp::now()); + } + } + #[cfg(not(target_os = "linux"))] + { + now = timestamp_to_filetime(Timestamp::now()); + } + (now, now) + }, + &Source::Timestamp(ts) => (ts, ts), + }; + + let (atime, mtime) = if let Some(date) = &opts.date { + ( + parse_date( + filetime_to_zoned(&atime).ok_or_else(|| TouchError::InvalidFiletime(atime))?, + date, + )?, + parse_date( + filetime_to_zoned(&mtime).ok_or_else(|| TouchError::InvalidFiletime(mtime))?, + date, + )?, + ) + } else { + (atime, mtime) + }; + + for (ind, file) in files.iter().enumerate() { + let (path, is_stdout) = match file { + InputFile::Stdout => (Cow::Owned(pathbuf_from_stdout()?), true), + InputFile::Path(path) => (Cow::Borrowed(path), false), + }; + touch_file(&path, is_stdout, opts, atime, mtime).map_err(|e| TouchError::TouchFileError { + path: path.into_owned(), + index: ind, + error: e, + })?; + } + + Ok(()) +} + +/// Create or update the timestamp for a single file. +/// +/// # Arguments +/// +/// - `path` - The path to the file to create/update timestamp for +/// - `is_stdout` - Stdout is handled specially, see [`update_times`] for more +/// info +/// - `atime` - Access time to set for the file +/// - `mtime` - Modification time to set for the file +fn touch_file( + path: &Path, + is_stdout: bool, + opts: &Options, + atime: FileTime, + mtime: FileTime, +) -> UResult<()> { + let filename = if is_stdout { + OsStr::new("-") + } else { + path.as_os_str() + }; + + // pi-uutils: resolve the operand against the shell working directory for + // every syscall below; `path`/`filename` keep the operand as typed for + // error messages. + let resolved = pi_uutils_ctx::resolve(path); + + let metadata_result = if opts.no_deref { + resolved.symlink_metadata() + } else { + resolved.metadata() + }; + + if let Err(e) = metadata_result { + if e.kind() != ErrorKind::NotFound { + return Err(e.map_err_context(|| format!("setting times of {}", filename.quote()))); + } + + if opts.no_create { + return Ok(()); + } + + if opts.no_deref { + let e = USimpleError::new( + 1, + format!("setting times of {}: No such file or directory", filename.quote()), + ); + if opts.strict { + return Err(e); + } + // pi-uutils: upstream `show!` — print the error and accumulate the + // exit code in the scope instead of process-global state. + let _ = writeln!(pi_uutils_ctx::stderr(), "touch: {e}"); + pi_uutils_ctx::set_exit_code(e.code()); + return Ok(()); + } + + if let Err(e) = File::create(&resolved) { + // we need to check if the path is the path to a directory (ends with a + // separator) we can't use File::create to create a directory + // we cannot use path.is_dir() because it calls fs::metadata which we already + // called when stable, we can change to use e.kind() == + // std::io::ErrorKind::IsADirectory + let is_directory = if let Some(last_char) = path.to_string_lossy().chars().last() { + last_char == std::path::MAIN_SEPARATOR + } else { + false + }; + if is_directory { + let custom_err = Error::other("No such file or directory"); + return Err( + custom_err.map_err_context(|| format!("cannot touch {}", filename.quote())), + ); + } + let e = e.map_err_context(|| format!("cannot touch {}", path.quote())); + if opts.strict { + return Err(e); + } + // pi-uutils: upstream `show!` — see above. + let _ = writeln!(pi_uutils_ctx::stderr(), "touch: {e}"); + pi_uutils_ctx::set_exit_code(e.code()); + return Ok(()); + } + + // Minor optimization: if no reference time, timestamp, or date was specified, + // we're done. + if opts.source == Source::Now && opts.date.is_none() { + return Ok(()); + } + } + + update_times(path, is_stdout, opts, atime, mtime) +} + +/// Returns which of the times (access, modification) are to be changed. +/// +/// Note that "-a" and "-m" may be passed together; this is not an xor. +/// - If `-a` is passed but not `-m`, only access time is changed +/// - If `-m` is passed but not `-a`, only modification time is changed +/// - If neither or both are passed, both times are changed +fn determine_atime_mtime_change(matches: &ArgMatches) -> ChangeTimes { + // If `--time` is given, Some(true) if equivalent to `-a`, Some(false) if + // equivalent to `-m` If `--time` not given, None + let time_access_only = if matches.contains_id(options::TIME) { + matches + .get_one::(options::TIME) + .map(|time| time.contains("access") || time.contains("atime") || time.contains("use")) + } else { + None + }; + + let atime_only = matches.get_flag(options::ACCESS) || time_access_only.unwrap_or_default(); + let mtime_only = matches.get_flag(options::MODIFICATION) || !time_access_only.unwrap_or(true); + + if atime_only && !mtime_only { + ChangeTimes::AtimeOnly + } else if mtime_only && !atime_only { + ChangeTimes::MtimeOnly + } else { + ChangeTimes::Both + } +} + +/// Updating file access and modification times based on user-specified options +/// +/// If the file is not stdout (`!is_stdout`) and `-h`/`--no-dereference` was +/// passed, then, if the given file is a symlink, its own times will be updated, +/// rather than the file it points to. +fn update_times( + path: &Path, + is_stdout: bool, + opts: &Options, + atime: FileTime, + mtime: FileTime, +) -> UResult<()> { + // pi-uutils: resolve the operand against the shell working directory for + // every syscall below; `path` keeps the operand as typed for error + // messages. + let resolved = pi_uutils_ctx::resolve(path); + + // If changing "only" atime or mtime, grab the existing value of the other. + let (atime, mtime) = match opts.change_times { + ChangeTimes::AtimeOnly => ( + atime, + stat(&resolved, !opts.no_deref) + .map_err_context(|| format!("failed to get attributes of {}", path.quote()))? + .1, + ), + ChangeTimes::MtimeOnly => ( + stat(&resolved, !opts.no_deref) + .map_err_context(|| format!("failed to get attributes of {}", path.quote()))? + .0, + mtime, + ), + ChangeTimes::Both => (atime, mtime), + }; + + // sets the file access and modification times for a file or a symbolic link. + // The filename, access time (atime), and modification time (mtime) are provided + // as inputs. + + if opts.no_deref && !is_stdout { + return set_symlink_file_times(&resolved, atime, mtime) + .map_err_context(|| format!("setting times of {}", path.quote())); + } + + #[cfg(unix)] + { + // Open write-only and use futimens to trigger IN_CLOSE_WRITE on Linux. + if !is_stdout && try_futimens_via_write_fd(&resolved, atime, mtime).is_ok() { + return Ok(()); + } + } + + set_file_times(&resolved, atime, mtime) + .map_err_context(|| format!("setting times of {}", path.quote())) +} + +#[cfg(unix)] +/// Set file times via file descriptor using `futimens`. +/// +/// This opens the file write-only and uses the POSIX `futimens` call to set +/// access and modification times on the open FD (not by path), which also +/// triggers `IN_CLOSE_WRITE` on Linux when the FD is closed. +fn try_futimens_via_write_fd(path: &Path, atime: FileTime, mtime: FileTime) -> std::io::Result<()> { + let file = OpenOptions::new() + .write(true) + // Avoid blocking on special files (e.g. FIFOs) before we can inspect metadata. + .custom_flags(O_NONBLOCK) + .open(path)?; + + let timestamps = Timestamps { + last_access: rustix::fs::Timespec { + tv_sec: atime.unix_seconds(), + tv_nsec: atime.nanoseconds() as _, + }, + last_modification: rustix::fs::Timespec { + tv_sec: mtime.unix_seconds(), + tv_nsec: mtime.nanoseconds() as _, + }, + }; + + futimens(&file, ×tamps).map_err(|e| Error::from_raw_os_error(e.raw_os_error())) +} + +/// Get metadata of the provided path +/// If `follow` is `true`, the function will try to follow symlinks. Errors if +/// the symlink is dangling, otherwise defaults to symlink metadata. If `follow` +/// is `false`, the function will return metadata of the symlink itself +fn stat(path: &Path, follow: bool) -> std::io::Result<(FileTime, FileTime)> { + let metadata = if follow { + match fs::metadata(path) { + // Successfully followed symlink + Ok(meta) => meta, + // Dangling symlink + Err(e) if e.kind() == ErrorKind::NotFound => return Err(e), + // Other error (?), try to get the symlink metadata + Err(_) => fs::symlink_metadata(path)?, + } + } else { + fs::symlink_metadata(path)? + }; + + Ok(( + FileTime::from_last_access_time(&metadata), + FileTime::from_last_modification_time(&metadata), + )) +} + +fn parse_date(ref_zoned: Zoned, s: &str) -> Result { + // This isn't actually compatible with GNU touch, but there doesn't seem to + // be any simple specification for what format this parameter allows and I'm + // not about to implement GNU parse_datetime. + // http://git.savannah.gnu.org/gitweb/?p=gnulib.git;a=blob_plain;f=lib/parse-datetime.y + + // TODO: match on char count? + + // "The preferred date and time representation for the current locale." + // "(In the POSIX locale this is equivalent to %a %b %e %H:%M:%S %Y.)" + // time 0.1.43 parsed this as 'a b e T Y' + // which is equivalent to the POSIX locale: %a %b %e %H:%M:%S %Y + // Tue Dec 3 ... + // ("%c", POSIX_LOCALE_FORMAT), + // + if let Ok(parsed) = strtime::parse(format::POSIX_LOCALE, s) + .and_then(|tm| tm.to_datetime()) + .and_then(|dt| TimeZone::UTC.to_zoned(dt)) + { + return Ok(timestamp_to_filetime(parsed.timestamp())); + } + + // Also support other formats found in the GNU tests like + // in tests/misc/stat-nanoseconds.sh + // or tests/touch/no-rights.sh + for fmt in [ + format::YYYYMMDDHHMMS, + format::YYYYMMDDHHMMSS, + format::YYYY_MM_DD_HH_MM, + format::YYYYMMDDHHMM_OFFSET, + ] { + if let Ok(parsed) = strtime::parse(fmt, s) + .and_then(|tm| tm.to_datetime()) + .and_then(|dt| TimeZone::UTC.to_zoned(dt)) + { + return Ok(timestamp_to_filetime(parsed.timestamp())); + } + } + + // "Equivalent to %Y-%m-%d (the ISO 8601 date format). (C99)" + // ("%F", ISO_8601_FORMAT), + // pi-uutils: `TimeZone::system()` (and the `TZ` variable it consults) + // intentionally stays process-global; jiff reads it internally. + if let Ok(filetime) = strtime::parse(format::ISO_8601, s) + .and_then(|tm| tm.to_date()) + .and_then(|date| { + TimeZone::system() + .to_ambiguous_zoned(date.to_datetime(Time::midnight())) + .unambiguous() + }) + .map(|zdt| timestamp_to_filetime(zdt.timestamp())) + { + return Ok(filetime); + } + + // "@%s" is "The number of seconds since the Epoch, 1970-01-01 00:00:00 +0000 + // (UTC). (TZ) (Calculated from mktime(tm).)" + if s.bytes().next() == Some(b'@') + && let Ok(ts) = &s[1..].parse::() + { + return Ok(FileTime::from_unix_time(*ts, 0)); + } + + if let Ok(zoned) = parse_datetime::parse_datetime_at_date(ref_zoned, s) { + return Ok(timestamp_to_filetime(zoned.timestamp())); + } + + Err(TouchError::InvalidDateFormat(s.to_owned())) +} + +/// Prepends 19 or 20 to the year if it is a 2 digit year +/// +/// GNU `touch` behavior: +/// +/// - 68 and before is interpreted as 20xx +/// - 69 and after is interpreted as 19xx +fn prepend_century(s: &str) -> UResult { + let first_two_digits = s[..2] + .parse::() + .map_err(|_| USimpleError::new(1, format!("invalid date ts format {}", s.quote())))?; + Ok(format!("{}{s}", if first_two_digits > 68 { 19 } else { 20 })) +} + +/// Parses a timestamp string into a [`FileTime`]. +/// +/// This function attempts to parse a string into a [`FileTime`] +/// As expected by gnu touch -t : `[[cc]yy]mmddhhmm[.ss]` +/// +/// Note that If the year is specified with only two digits, +/// then cc is 20 for years in the range 0 … 68, and 19 for years in 69 … 99. +/// in order to be compatible with GNU `touch`. +fn parse_timestamp(s: &str) -> UResult { + use format::{YYYYMMDDHHMM, YYYYMMDDHHMM_DOT_SS}; + + // pi-uutils: `TimeZone::system()` intentionally stays process-global. + let current_year = || Timestamp::now().to_zoned(TimeZone::system()).year(); + + let (format, ts) = match s.chars().count() { + 15 => (YYYYMMDDHHMM_DOT_SS, s.to_owned()), + 12 => (YYYYMMDDHHMM, s.to_owned()), + // If we don't add "19" or "20", we have insufficient information to parse + 13 => (YYYYMMDDHHMM_DOT_SS, prepend_century(s)?), + 10 => (YYYYMMDDHHMM, prepend_century(s)?), + 11 => (YYYYMMDDHHMM_DOT_SS, format!("{}{s}", current_year())), + 8 => (YYYYMMDDHHMM, format!("{}{s}", current_year())), + _ => { + return Err(USimpleError::new(1, format!("invalid date format {}", s.quote()))); + }, + }; + + let mut dt = strtime::parse(format, &ts) + .and_then(|parsed| parsed.to_datetime()) + .map_err(|_| USimpleError::new(1, format!("invalid date ts format {}", ts.quote())))?; + + // Jiff caps seconds at 59, but 60 is valid. It might be a leap second + // or wrap to the next minute. But that doesn't really matter, because we + // only care about the timestamp anyway. + // Tested in gnu/tests/touch/60-seconds + if dt.second() == 59 && ts.ends_with(".60") { + dt += 1.second(); + } + + // Due to daylight saving time switch, local time can jump from 1:59 AM to + // 3:00 AM, in which case any time between 2:00 AM and 2:59 AM is not valid. + // Jiff's `to_ambiguous_zoned(...).unambiguous()` handles this case. + let local = TimeZone::system() + .to_ambiguous_zoned(dt) + .unambiguous() + .map_err(|_| USimpleError::new(1, format!("invalid date ts format {}", ts.quote())))?; + + Ok(timestamp_to_filetime(local.timestamp())) +} + +// TODO: this may be a good candidate to put in fsext.rs +/// Returns a [`PathBuf`] to stdout. +/// +/// On Windows, uses `GetFinalPathNameByHandleW` to attempt to get the path +/// from the stdout handle. +#[cfg_attr(not(windows), expect(clippy::unnecessary_wraps))] +fn pathbuf_from_stdout() -> Result { + #[cfg(all(unix, not(target_os = "android")))] + { + Ok(PathBuf::from("/dev/stdout")) + } + #[cfg(target_os = "android")] + { + Ok(PathBuf::from("/proc/self/fd/1")) + } + #[cfg(windows)] + { + use std::os::windows::prelude::AsRawHandle; + + use windows_sys::Win32::{ + Foundation::{ + ERROR_INVALID_PARAMETER, ERROR_NOT_ENOUGH_MEMORY, ERROR_PATH_NOT_FOUND, GetLastError, + HANDLE, MAX_PATH, + }, + Storage::FileSystem::{FILE_NAME_OPENED, GetFinalPathNameByHandleW}, + }; + + let handle = std::io::stdout().lock().as_raw_handle() as HANDLE; + let mut file_path_buffer: [u16; MAX_PATH as usize] = [0; MAX_PATH as usize]; + + // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlea#examples + // SAFETY: We transmute the handle to be able to cast *mut c_void into a + // HANDLE (i32) so rustc will let us call GetFinalPathNameByHandleW. The + // reference example code for GetFinalPathNameByHandleW implies that + // it is safe for us to leave lpszfilepath uninitialized, so long as + // the buffer size is correct. We know the buffer size (MAX_PATH) at + // compile time. MAX_PATH is a small number (260) so we can cast it + // to a u32. + let ret = unsafe { + GetFinalPathNameByHandleW( + handle, + file_path_buffer.as_mut_ptr(), + file_path_buffer.len() as u32, + FILE_NAME_OPENED, + ) + }; + + // pi-uutils: literalized error strings; the variant's Display supplies + // the "GetFinalPathNameByHandleW failed with code" prefix, so only the + // code payload is stored. + let buffer_size = match ret { + ERROR_PATH_NOT_FOUND | ERROR_NOT_ENOUGH_MEMORY | ERROR_INVALID_PARAMETER => { + return Err(TouchError::WindowsStdoutPathError(ret.to_string())); + }, + 0 => { + return Err(TouchError::WindowsStdoutPathError(format!( + "{}", + // SAFETY: GetLastError is thread-safe and has no documented memory unsafety. + unsafe { GetLastError() } + ))); + }, + e => e as usize, + }; + + // Don't include the null terminator + Ok(String::from_utf16(&file_path_buffer[0..buffer_size]) + .map_err(|e| TouchError::WindowsStdoutPathError(e.to_string()))? + .into()) + } + #[cfg(target_os = "wasi")] + { + Ok(PathBuf::from("/dev/stdout")) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("touch") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + fn times_of(path: &Path) -> (FileTime, FileTime) { + let metadata = fs::metadata(path).unwrap(); + (FileTime::from_last_access_time(&metadata), FileTime::from_last_modification_time(&metadata)) + } + + #[test] + fn relative_operand_creates_file_in_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + + // Relative operand + scope cwd differing from the process cwd: only + // the call-site `pi_uutils_ctx::resolve` patch makes the file land in + // the scope cwd instead of the process cwd. + let (code, stdout, stderr) = run_in(root.clone(), vec!["created.txt"]); + assert_eq!(code, 0); + assert_eq!(stdout, ""); + assert_eq!(stderr, ""); + assert!(root.join("created.txt").is_file()); + assert!( + !std::env::current_dir() + .unwrap() + .join("created.txt") + .exists() + ); + } + + #[test] + fn no_create_on_missing_file_is_silent_success() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-c", "missing.txt"]); + assert_eq!(code, 0); + assert_eq!(stdout, ""); + assert_eq!(stderr, ""); + assert!(!root.join("missing.txt").exists()); + } + + #[test] + fn reference_copies_times_from_relative_reference() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("ref"), b"x").unwrap(); + let ref_atime = FileTime::from_unix_time(1_000_000, 0); + let ref_mtime = FileTime::from_unix_time(2_000_000, 0); + set_file_times(root.join("ref"), ref_atime, ref_mtime).unwrap(); + + // Both the `-r` reference and the FILE operand are relative: each is + // resolved against the scope cwd at its own syscall site. + let (code, stdout, stderr) = run_in(root.clone(), vec!["-r", "ref", "new"]); + assert_eq!(code, 0); + assert_eq!(stdout, ""); + assert_eq!(stderr, ""); + let (atime, mtime) = times_of(&root.join("new")); + assert_eq!(atime, ref_atime); + assert_eq!(mtime, ref_mtime); + } + + #[test] + fn date_sets_mtime_to_fixed_utc_instant() { + let (_dir, root) = canonical_tempdir(); + + // "%Y-%m-%d %H:%M:%S" dates are interpreted in UTC, so the expected + // epoch is timezone-independent: 2001-02-03T04:05:06Z. + let (code, stdout, stderr) = run_in(root.clone(), vec!["-d", "2001-02-03 04:05:06", "f"]); + assert_eq!(code, 0); + assert_eq!(stdout, ""); + assert_eq!(stderr, ""); + let (atime, mtime) = times_of(&root.join("f")); + assert_eq!(mtime.unix_seconds(), 981_173_106); + assert_eq!(atime.unix_seconds(), 981_173_106); + } + + #[test] + fn modification_only_preserves_existing_atime() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("f"), b"x").unwrap(); + let old_atime = FileTime::from_unix_time(1_111, 0); + let old_mtime = FileTime::from_unix_time(2_222, 0); + set_file_times(root.join("f"), old_atime, old_mtime).unwrap(); + + let (code, _, stderr) = run_in(root.clone(), vec!["-m", "-d", "@981173106", "f"]); + assert_eq!(code, 0); + assert_eq!(stderr, ""); + let (atime, mtime) = times_of(&root.join("f")); + assert_eq!(atime, old_atime, "-m must not change atime"); + assert_eq!(mtime, FileTime::from_unix_time(981_173_106, 0)); + } + + #[test] + fn missing_operand_is_usage_error() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec![]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(stderr.contains("missing file operand"), "stderr: {stderr}"); + assert!(stderr.contains("Try 'touch --help'"), "stderr: {stderr}"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("access and modification times")); + assert_eq!(stderr, ""); + } + + #[test] + fn time_word_and_flags_select_change_times() { + assert_eq!( + ChangeTimes::Both, + determine_atime_mtime_change(&uu_app().try_get_matches_from(vec!["touch", "f"]).unwrap()) + ); + assert_eq!( + ChangeTimes::Both, + determine_atime_mtime_change( + &uu_app() + .try_get_matches_from(vec!["touch", "-a", "-m", "--time", "modify", "f"]) + .unwrap() + ) + ); + assert_eq!( + ChangeTimes::AtimeOnly, + determine_atime_mtime_change( + &uu_app() + .try_get_matches_from(vec!["touch", "--time", "access", "f"]) + .unwrap() + ) + ); + assert_eq!( + ChangeTimes::MtimeOnly, + determine_atime_mtime_change( + &uu_app() + .try_get_matches_from(vec!["touch", "-m", "f"]) + .unwrap() + ) + ); + } +} diff --git a/crates/vendor/uu-truncate/Cargo.toml b/crates/vendor/uu-truncate/Cargo.toml new file mode 100644 index 000000000..4c05a9e67 --- /dev/null +++ b/crates/vendor/uu-truncate/Cargo.toml @@ -0,0 +1,22 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/truncate), patched to resolve +# path arguments against the shell working directory and route I/O through +# pi-uutils-ctx so it can run in-process as a shell builtin. See src/truncate.rs +# for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_truncate" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "truncate ~ (uutils) truncate (or extend) FILE to SIZE (vendored + patched for in-process embedding)" + +[lib] +path = "src/truncate.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +uucore = { version = "0.8.0", features = ["parser-size"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" +tempfile = "3" diff --git a/crates/vendor/uu-truncate/LICENSE b/crates/vendor/uu-truncate/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-truncate/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-truncate/src/truncate.rs b/crates/vendor/uu-truncate/src/truncate.rs new file mode 100644 index 000000000..0b6a35ff7 --- /dev/null +++ b/crates/vendor/uu-truncate/src/truncate.rs @@ -0,0 +1,557 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) RFILE refsize rfilename fsize tsize + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Every filesystem syscall resolves its path operand +// against the shell working directory via `pi_uutils_ctx::resolve` AT THE CALL +// SITE, while the original operands are kept for display/error messages (GNU +// prints operands as typed). All process-global stdio is routed through +// `pi_uutils_ctx`, `translate!` strings are literalized, per-file errors are +// reported through the context stderr with `set_exit_code` (continue-on-error +// like GNU truncate), and the entry point no longer calls `std::process::exit`. + +#[cfg(unix)] +use std::os::unix::fs::FileTypeExt; +use std::{ + ffi::OsString, + fs::{OpenOptions, metadata}, + io::{ErrorKind, Write}, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use pi_uutils_ctx::format_usage; +use uucore::{ + display::Quotable, + error::{FromIo, UResult, USimpleError, UUsageError}, + parser::parse_size::{ParseSizeError, Parser, allow_list_with_all_suffixes}, +}; + +#[derive(Debug, Eq, PartialEq)] +enum TruncateMode { + Absolute(u64), + Extend(u64), + Reduce(u64), + AtMost(u64), + AtLeast(u64), + RoundDown(u64), + RoundUp(u64), +} + +impl TruncateMode { + /// Compute a target size in bytes for this truncate mode. + /// + /// `fsize` is the size of the reference file, in bytes. + /// + /// If the mode is [`TruncateMode::Reduce`] and the value to + /// reduce by is greater than `fsize`, then this function returns + /// 0 (since it cannot return a negative number). + /// + /// # Returns + /// + /// `None` if rounding by 0, else the target size. + fn to_size(&self, fsize: u64) -> Option { + match self { + Self::Absolute(size) => Some(*size), + Self::Extend(size) => Some(fsize + size), + Self::Reduce(size) => Some(fsize.saturating_sub(*size)), + Self::AtMost(size) => Some(fsize.min(*size)), + Self::AtLeast(size) => Some(fsize.max(*size)), + Self::RoundDown(size) => fsize.checked_rem(*size).map(|remainder| fsize - remainder), + Self::RoundUp(size) => fsize.checked_next_multiple_of(*size), + } + } + + /// Determine if mode is absolute + /// + /// # Returns + /// + /// `true` is self matches Self::Absolute(_), `false` otherwise. + fn is_absolute(&self) -> bool { + matches!(self, Self::Absolute(_)) + } +} + +pub mod options { + pub static IO_BLOCKS: &str = "io-blocks"; + pub static NO_CREATE: &str = "no-create"; + pub static REFERENCE: &str = "reference"; + pub static SIZE: &str = "size"; + pub static ARG_FILES: &str = "files"; +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match truncate_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + // pi-uutils: don't emit a dangling "truncate: " prefix when the + // error renders to an empty message. + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "truncate: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn truncate_main(matches: &ArgMatches) -> UResult<()> { + let files: Vec = matches + .get_many::(options::ARG_FILES) + .map(|v| v.cloned().collect()) + .unwrap_or_default(); + + if files.is_empty() { + Err(UUsageError::new(1, "missing file operand".to_string())) + } else { + let io_blocks = matches.get_flag(options::IO_BLOCKS); + let no_create = matches.get_flag(options::NO_CREATE); + let reference = matches + .get_one::(options::REFERENCE) + .map(String::from); + let size = matches.get_one::(options::SIZE).map(String::from); + truncate(no_create, io_blocks, reference, size, &files) + } +} + +pub fn uu_app() -> Command { + Command::new("truncate") + .version(uucore::crate_version!()) + .about("Shrink or extend the size of each file to the specified size.") + .override_usage(format_usage("truncate [OPTION]... [FILE]...")) + .after_help( + "SIZE is an integer with an optional prefix and optional unit.\nThe available units (K, \ + M, G, T, P, E, Z, and Y) use the following format:\n 'KB' => 1000 (kilobytes)\n \ + 'K' => 1024 (kibibytes)\n 'MB' => 1000*1000 (megabytes)\n 'M' => 1024*1024 \ + (mebibytes)\n 'GB' => 1000*1000*1000 (gigabytes)\n 'G' => 1024*1024*1024 \ + (gibibytes)\nSIZE may also be prefixed by one of the following to adjust the size of \ + each\nfile based on its current size:\n '+' => extend by\n '-' => reduce by\n \ + '<' => at most\n '>' => at least\n '/' => round down to multiple of\n '%' => \ + round up to multiple of", + ) + .infer_long_args(true) + .arg( + Arg::new(options::IO_BLOCKS) + .short('o') + .long(options::IO_BLOCKS) + .help( + "treat SIZE as the number of I/O blocks of the file rather than bytes (NOT \ + IMPLEMENTED)", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::NO_CREATE) + .short('c') + .long(options::NO_CREATE) + .help("do not create files that do not exist") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::REFERENCE) + .short('r') + .long(options::REFERENCE) + .required_unless_present(options::SIZE) + .help("base the size of each file on the size of RFILE") + .value_name("RFILE") + .value_hint(clap::ValueHint::FilePath), + ) + .arg( + Arg::new(options::SIZE) + .short('s') + .long(options::SIZE) + .required_unless_present(options::REFERENCE) + .help( + "set or adjust the size of each file according to SIZE, which is in bytes unless \ + --io-blocks is specified", + ) + .allow_hyphen_values(true) + .value_name("SIZE"), + ) + .arg( + Arg::new(options::ARG_FILES) + .value_name("FILE") + .action(ArgAction::Append) + .required(true) + .value_hint(clap::ValueHint::FilePath) + .value_parser(clap::value_parser!(OsString)), + ) +} + +/// Truncate the named file to the specified size. +/// +/// If `create` is true, then the file will be created if it does not +/// already exist. If `size` is larger than the number of bytes in the +/// file, then the file will be padded with zeros. If `size` is smaller +/// than the number of bytes in the file, then the file will be +/// truncated and any bytes beyond `size` will be lost. +/// +/// # Errors +/// +/// If the file could not be opened, or there was a problem setting the +/// size of the file. +fn do_file_truncate(filename: &OsString, create: bool, size: u64) -> UResult<()> { + // pi-uutils: resolve the operand against the shell working directory at + // the open site; `filename` is kept for the error message. + let resolved = pi_uutils_ctx::resolve(filename); + + match OpenOptions::new() + .write(true) + .create(create) + .open(&resolved) + { + Ok(file) => file.set_len(size), + Err(e) if e.kind() == ErrorKind::NotFound && !create => Ok(()), + Err(e) => Err(e), + } + .map_err_context(|| format!("cannot open {} for writing", filename.quote())) +} + +fn file_truncate( + no_create: bool, + reference_size: Option, + mode: &TruncateMode, + filename: &OsString, +) -> UResult<()> { + // pi-uutils: resolve the operand against the shell working directory at + // the metadata site; `filename` is kept for the error message. + let resolved = pi_uutils_ctx::resolve(filename); + + // Get the length of the file. + let file_size = match metadata(&resolved) { + Ok(metadata) => { + // A pipe has no length. Do this check here to avoid duplicate `stat()` syscall. + #[cfg(unix)] + if metadata.file_type().is_fifo() { + return Err(USimpleError::new( + 1, + format!( + "cannot open {} for writing: No such device or address", + filename.to_string_lossy().quote() + ), + )); + } + metadata.len() + }, + Err(_) => 0, + }; + + // The reference size can be either: + // + // 1. The size of a given file + // 2. The size of the file to be truncated if no reference has been provided. + let actual_reference_size = reference_size.unwrap_or(file_size); + + let Some(truncate_size) = mode.to_size(actual_reference_size) else { + return Err(USimpleError::new(1, "division by zero".to_string())); + }; + + do_file_truncate(filename, !no_create, truncate_size) +} + +fn truncate( + no_create: bool, + _: bool, + reference: Option, + size: Option, + filenames: &[OsString], +) -> UResult<()> { + let reference_size = match reference { + Some(reference_path) => { + // pi-uutils: resolve the reference operand against the shell + // working directory; `reference_path` is kept for the message. + let reference_metadata = + metadata(pi_uutils_ctx::resolve(&reference_path)).map_err(|error| { + match error.kind() { + ErrorKind::NotFound => USimpleError::new( + 1, + format!("cannot stat {}: No such file or directory", reference_path.quote()), + ), + _ => error.map_err_context(String::new), + } + })?; + + Some(reference_metadata.len()) + }, + None => None, + }; + + let size_string = size.as_deref(); + + // Omitting the mode is equivalent to extending a file by 0 bytes. + let mode = match size_string { + Some(string) => match parse_mode_and_size(string) { + Err(error) => { + return Err(USimpleError::new(1, format!("Invalid number: {error}"))); + }, + Ok(mode) => mode, + }, + None => TruncateMode::Extend(0), + }; + + // If a reference file has been given, the truncate mode cannot be absolute. + if reference_size.is_some() && mode.is_absolute() { + return Err(USimpleError::new( + 1, + "you must specify a relative '--size' with '--reference'".to_string(), + )); + } + + for filename in filenames { + // pi-uutils: upstream aborts on the first failing file; report the + // error through the context stderr and continue with the remaining + // operands (GNU behavior), accumulating the exit code. + if let Err(err) = file_truncate(no_create, reference_size, &mode, filename) { + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "truncate: {msg}"); + } + pi_uutils_ctx::set_exit_code(if err.code() == 0 { 1 } else { err.code() }); + } + } + + Ok(()) +} + +/// Decide whether a character is one of the size modifiers, like '+' or '<'. +fn is_modifier(c: char) -> bool { + c == '+' || c == '-' || c == '<' || c == '>' || c == '/' || c == '%' +} + +/// Parse a size string with optional modifier symbol as its first character. +/// +/// A size string is as described in [`Parser::parse_u64`]. The first character +/// of `size_string` might be a modifier symbol, like `'+'` or +/// `'<'`. The first element of the pair returned by this function +/// indicates which modifier symbol was present, or +/// [`TruncateMode::Absolute`] if none. +fn parse_mode_and_size(size_string: &str) -> Result { + // Trim any whitespace. + let mut size_string = size_string.trim(); + + // Get the modifier character from the size string, if any. For + // example, if the argument is "+123", then the modifier is '+'. + if let Some(c) = size_string.chars().next() { + if is_modifier(c) { + size_string = &size_string[1..]; + } + let allow_list = allow_list_with_all_suffixes("EgGkKmMPQRtTYZ"); + let allow_list_ref = allow_list.iter().map(AsRef::as_ref).collect::>(); + Parser::default() + .with_allow_list(&allow_list_ref) + .parse_u64(size_string) + .map(match c { + '+' => TruncateMode::Extend, + '-' => TruncateMode::Reduce, + '<' => TruncateMode::AtMost, + '>' => TruncateMode::AtLeast, + '/' => TruncateMode::RoundDown, + '%' => TruncateMode::RoundUp, + _ => TruncateMode::Absolute, + }) + } else { + Err(ParseSizeError::ParseFailure(size_string.to_string())) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, fs, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(cwd: PathBuf, args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd, + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("truncate") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + /// Canonicalized temp dir (macOS tempdirs live behind /var -> /private/var). + fn canonical_tempdir() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let canon = fs::canonicalize(dir.path()).unwrap(); + (dir, canon) + } + + fn len(path: &PathBuf) -> u64 { + fs::metadata(path).unwrap().len() + } + + #[test] + fn resolves_relative_operand_against_scope_cwd() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("f"), b"12345678").unwrap(); + + // Relative operand + scope cwd differing from the process cwd: only the + // call-site `pi_uutils_ctx::resolve` patch makes this find the file. + let (code, stdout, stderr) = run_in(root.clone(), vec!["-s", "5", "f"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + assert_eq!(len(&root.join("f")), 5); + } + + #[test] + fn extend_grows_by_relative_amount() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("f"), b"1234").unwrap(); + + let (code, _, stderr) = run_in(root.clone(), vec!["-s", "+3", "f"]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert_eq!(len(&root.join("f")), 7); + } + + #[test] + fn at_most_caps_only_larger_files() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("big"), vec![0u8; 20]).unwrap(); + fs::write(root.join("small"), b"abc").unwrap(); + + let (code, _, stderr) = run_in(root.clone(), vec!["-s", "<10", "big", "small"]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert_eq!(len(&root.join("big")), 10); + assert_eq!(len(&root.join("small")), 3); + } + + #[test] + fn no_create_skips_missing_file() { + let (_dir, root) = canonical_tempdir(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-c", "-s", "5", "missing"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", "")); + assert!(!root.join("missing").exists()); + } + + #[test] + fn missing_file_without_no_create_is_created_at_size() { + let (_dir, root) = canonical_tempdir(); + + let (code, _, stderr) = run_in(root.clone(), vec!["-s", "9", "fresh"]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert_eq!(len(&root.join("fresh")), 9); + } + + #[test] + fn reference_copies_size_of_rfile() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("ref"), b"123456").unwrap(); + fs::write(root.join("f"), b"x").unwrap(); + + let (code, _, stderr) = run_in(root.clone(), vec!["-r", "ref", "f"]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert_eq!(len(&root.join("f")), 6); + } + + #[test] + fn missing_reference_file_fails_with_stat_error() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("f"), b"x").unwrap(); + + let (code, _, stderr) = run_in(root.clone(), vec!["-r", "nope", "f"]); + assert_eq!(code, 1); + assert!(stderr.contains("cannot stat 'nope': No such file or directory")); + assert_eq!(len(&root.join("f")), 1, "operand must be untouched"); + } + + #[test] + fn invalid_size_reports_error_and_exit_1() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("f"), b"x").unwrap(); + + let (code, stdout, stderr) = run_in(root.clone(), vec!["-s", "bogus", "f"]); + assert_eq!((code, stdout.as_str()), (1, "")); + assert!(stderr.contains("truncate: Invalid number:")); + assert_eq!(len(&root.join("f")), 1, "operand must be untouched"); + } + + #[test] + fn reference_with_absolute_size_is_rejected() { + let (_dir, root) = canonical_tempdir(); + fs::write(root.join("ref"), b"123").unwrap(); + fs::write(root.join("f"), b"x").unwrap(); + + let (code, _, stderr) = run_in(root.clone(), vec!["-r", "ref", "-s", "5", "f"]); + assert_eq!(code, 1); + assert!(stderr.contains("you must specify a relative '--size' with '--reference'")); + } + + #[test] + fn parse_mode_and_size_prefixes() { + assert_eq!(parse_mode_and_size("10"), Ok(TruncateMode::Absolute(10))); + assert_eq!(parse_mode_and_size("+10"), Ok(TruncateMode::Extend(10))); + assert_eq!(parse_mode_and_size("-10"), Ok(TruncateMode::Reduce(10))); + assert_eq!(parse_mode_and_size("<10"), Ok(TruncateMode::AtMost(10))); + assert_eq!(parse_mode_and_size(">10"), Ok(TruncateMode::AtLeast(10))); + assert_eq!(parse_mode_and_size("/10"), Ok(TruncateMode::RoundDown(10))); + assert_eq!(parse_mode_and_size("%10"), Ok(TruncateMode::RoundUp(10))); + assert_eq!(parse_mode_and_size("1kB"), Ok(TruncateMode::Absolute(1000))); + assert!(parse_mode_and_size("1b").is_err()); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(PathBuf::from("."), vec!["--help"]); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("round up to multiple of")); + assert_eq!(stderr, ""); + } +} diff --git a/crates/vendor/uu-uname/Cargo.toml b/crates/vendor/uu-uname/Cargo.toml new file mode 100644 index 000000000..f5884aeee --- /dev/null +++ b/crates/vendor/uu-uname/Cargo.toml @@ -0,0 +1,21 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/uname), patched to route +# output through pi-uutils-ctx so it can run in-process as a shell builtin. See +# src/uname.rs for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_uname" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "uname ~ (uutils) display system information (vendored + patched for in-process embedding)" + +[lib] +path = "src/uname.rs" + +[dependencies] +platform-info = "2.0.3" +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +uucore = "0.8.0" +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-uname/LICENSE b/crates/vendor/uu-uname/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-uname/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-uname/src/uname.rs b/crates/vendor/uu-uname/src/uname.rs new file mode 100644 index 000000000..58bafde0a --- /dev/null +++ b/crates/vendor/uu-uname/src/uname.rs @@ -0,0 +1,342 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (API) nodename osname sysname (options) mnrsv mnrsvo + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Output goes to the context stdout (upstream's +// `println_verbatim` writes to the process stdout), `translate!` strings are +// literalized, and the entry point no longer calls `std::process::exit`. + +use std::{ + ffi::{OsStr, OsString}, + io::Write, +}; + +use clap::{Arg, ArgAction, ArgMatches, Command}; +use pi_uutils_ctx::format_usage; +use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI}; +use uucore::error::{UResult, USimpleError}; + +pub mod options { + pub static ALL: &str = "all"; + pub static KERNEL_NAME: &str = "kernel-name"; + pub static NODENAME: &str = "nodename"; + pub static KERNEL_VERSION: &str = "kernel-version"; + pub static KERNEL_RELEASE: &str = "kernel-release"; + pub static MACHINE: &str = "machine"; + pub static PROCESSOR: &str = "processor"; + pub static HARDWARE_PLATFORM: &str = "hardware-platform"; + pub static OS: &str = "operating-system"; +} + +pub struct UNameOutput { + pub kernel_name: Option, + pub nodename: Option, + pub kernel_release: Option, + pub kernel_version: Option, + pub machine: Option, + pub os: Option, + pub processor: Option, + pub hardware_platform: Option, +} + +impl UNameOutput { + fn display(&self) -> OsString { + [ + self.kernel_name.as_ref(), + self.nodename.as_ref(), + self.kernel_release.as_ref(), + self.kernel_version.as_ref(), + self.machine.as_ref(), + self.processor.as_ref(), + self.hardware_platform.as_ref(), + self.os.as_ref(), + ] + .into_iter() + .flatten() + .map(OsString::as_os_str) + .collect::>() + .join(OsStr::new(" ")) + } + + pub fn new(opts: &Options) -> UResult { + let uname = PlatformInfo::new() + .map_err(|_e| USimpleError::new(1, "cannot get system name".to_string()))?; + let none = !(opts.all + || opts.kernel_name + || opts.nodename + || opts.kernel_release + || opts.kernel_version + || opts.machine + || opts.os + || opts.processor + || opts.hardware_platform); + + let kernel_name = (opts.kernel_name || opts.all || none).then(|| uname.sysname().to_owned()); + + let nodename = (opts.nodename || opts.all).then(|| uname.nodename().to_owned()); + + let kernel_release = (opts.kernel_release || opts.all).then(|| uname.release().to_owned()); + + let kernel_version = (opts.kernel_version || opts.all).then(|| uname.version().to_owned()); + + let machine = (opts.machine || opts.all).then(|| uname.machine().to_owned()); + + let os = (opts.os || opts.all).then(|| uname.osname().to_owned()); + + // This option is unsupported on modern Linux systems + // See: https://lists.gnu.org/archive/html/bug-coreutils/2005-09/msg00063.html + let processor = opts.processor.then(|| "unknown".into()); + + // This option is unsupported on modern Linux systems + // See: https://lists.gnu.org/archive/html/bug-coreutils/2005-09/msg00063.html + let hardware_platform = opts.hardware_platform.then(|| "unknown".into()); + + Ok(Self { + kernel_name, + nodename, + kernel_release, + kernel_version, + machine, + os, + processor, + hardware_platform, + }) + } +} + +pub struct Options { + pub all: bool, + pub kernel_name: bool, + pub nodename: bool, + pub kernel_version: bool, + pub kernel_release: bool, + pub machine: bool, + pub processor: bool, + pub hardware_platform: bool, + pub os: bool, +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + match uname_main(&matches) { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "uname: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn uname_main(matches: &ArgMatches) -> UResult<()> { + let options = Options { + all: matches.get_flag(options::ALL), + kernel_name: matches.get_flag(options::KERNEL_NAME), + nodename: matches.get_flag(options::NODENAME), + kernel_release: matches.get_flag(options::KERNEL_RELEASE), + kernel_version: matches.get_flag(options::KERNEL_VERSION), + machine: matches.get_flag(options::MACHINE), + processor: matches.get_flag(options::PROCESSOR), + hardware_platform: matches.get_flag(options::HARDWARE_PLATFORM), + os: matches.get_flag(options::OS), + }; + let output = UNameOutput::new(&options)?; + // pi-uutils: replacement for upstream's `println_verbatim` — writes the + // output bytes verbatim to the context stdout instead of the process + // stdout. + let mut out = pi_uutils_ctx::stdout(); + out.write_all(uucore::os_str_as_bytes(output.display().as_os_str())?) + .and_then(|()| out.write_all(b"\n")) + .and_then(|()| out.flush()) + .map_err(|e| USimpleError::new(1, e.to_string()))?; + Ok(()) +} + +pub fn uu_app() -> Command { + Command::new("uname") + .version(uucore::crate_version!()) + .about("Print certain system information.\nWith no OPTION, same as -s.") + .override_usage(format_usage("uname [OPTION]...")) + .infer_long_args(true) + .arg( + Arg::new(options::ALL) + .short('a') + .long(options::ALL) + .help("Behave as though all of the options -mnrsvo were specified.") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::KERNEL_NAME) + .short('s') + .long(options::KERNEL_NAME) + .alias("sysname") // Obsolescent option in GNU uname + .help("print the kernel name.") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::NODENAME) + .short('n') + .long(options::NODENAME) + .help( + "print the nodename (the nodename may be a name that the system is known by to a \ + communications network).", + ) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::KERNEL_RELEASE) + .short('r') + .long(options::KERNEL_RELEASE) + .alias("release") // Obsolescent option in GNU uname + .help("print the operating system release.") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::KERNEL_VERSION) + .short('v') + .long(options::KERNEL_VERSION) + .help("print the operating system version.") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::MACHINE) + .short('m') + .long(options::MACHINE) + .help("print the machine hardware name.") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::OS) + .short('o') + .long(options::OS) + .help("print the operating system name.") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new(options::PROCESSOR) + .short('p') + .long(options::PROCESSOR) + .help("print the processor type (non-portable)") + .action(ArgAction::SetTrue) + .hide(true), + ) + .arg( + Arg::new(options::HARDWARE_PLATFORM) + .short('i') + .long(options::HARDWARE_PLATFORM) + .help("print the hardware platform (non-portable)") + .action(ArgAction::SetTrue) + .hide(true), + ) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd: PathBuf::from("."), + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("uname") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + #[test] + fn kernel_name_matches_platform() { + let (code, stdout, stderr) = run_in(vec!["-s"]); + assert_eq!((code, stderr.as_str()), (0, "")); + #[cfg(target_os = "macos")] + assert_eq!(stdout, "Darwin\n"); + #[cfg(target_os = "linux")] + assert_eq!(stdout, "Linux\n"); + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + assert!(stdout.trim_end().len() > 0); + } + + #[test] + fn no_options_defaults_to_kernel_name() { + let (code, bare, _) = run_in(vec![]); + let (_, with_s, _) = run_in(vec!["-s"]); + assert_eq!(code, 0); + assert_eq!(bare, with_s); + } + + #[test] + fn all_contains_kernel_name_and_more() { + let (code, all, stderr) = run_in(vec!["-a"]); + let (_, kernel, _) = run_in(vec!["-s"]); + assert_eq!((code, stderr.as_str()), (0, "")); + let kernel = kernel.trim_end(); + assert!(all.starts_with(kernel), "-a output {all:?} must start with {kernel:?}"); + assert!(all.trim_end().len() > kernel.len(), "-a must print more fields than -s"); + } + + #[test] + fn processor_prints_unknown() { + let (code, stdout, stderr) = run_in(vec!["-p"]); + assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "unknown\n", "")); + } +} diff --git a/crates/vendor/uu-whoami/Cargo.toml b/crates/vendor/uu-whoami/Cargo.toml new file mode 100644 index 000000000..d227d61ae --- /dev/null +++ b/crates/vendor/uu-whoami/Cargo.toml @@ -0,0 +1,27 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/whoami), patched to route +# output through pi-uutils-ctx so it can run in-process as a shell builtin. See +# src/whoami.rs for the patch markers (`pi-uutils:` comments). +[package] +name = "uu_whoami" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "whoami ~ (uutils) display user name of current effective user ID (vendored + patched for in-process embedding)" + +[lib] +path = "src/whoami.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +uucore = { version = "0.8.0", features = ["entries", "process"] } +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[target.'cfg(target_os = "windows")'.dependencies] +windows-sys = { version = "0.61.0", features = [ + "Win32_NetworkManagement_NetManagement", + "Win32_System_WindowsProgramming", + "Win32_Foundation", +] } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-whoami/LICENSE b/crates/vendor/uu-whoami/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-whoami/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-whoami/src/whoami.rs b/crates/vendor/uu-whoami/src/whoami.rs new file mode 100644 index 000000000..564f6c7ab --- /dev/null +++ b/crates/vendor/uu-whoami/src/whoami.rs @@ -0,0 +1,198 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// spell-checker:ignore (ToDO) getusername + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. Output goes to the context stdout (upstream's +// `println_verbatim` writes to the process stdout), `translate!` strings are +// literalized, the `platform` module (upstream +// src/platform/{mod,unix,windows}.rs) is inlined, and the entry point no longer +// calls `std::process::exit`. + +use std::{ffi::OsString, io::Write}; + +use clap::Command; +use uucore::error::{FromIo, UResult, USimpleError}; + +// pi-uutils: inlined from upstream src/platform/{mod,unix,windows}.rs (verbatim +// bodies); the platform user lookup itself is process-global state and needs no +// scope patching. +mod platform { + #[cfg(unix)] + pub use self::unix::get_username; + #[cfg(windows)] + pub use self::windows::get_username; + + #[cfg(unix)] + mod unix { + use std::{ffi::OsString, io}; + + use uucore::{entries::uid2usr, process::geteuid}; + + pub fn get_username() -> io::Result { + // uid2usr should arguably return an OsString but currently doesn't + uid2usr(geteuid()).map(Into::into) + } + } + + #[cfg(windows)] + mod windows { + use std::{ffi::OsString, io, os::windows::ffi::OsStringExt}; + + use windows_sys::Win32::{ + NetworkManagement::NetManagement::UNLEN, System::WindowsProgramming::GetUserNameW, + }; + + pub fn get_username() -> io::Result { + const BUF_LEN: u32 = UNLEN + 1; + let mut buffer = [0_u16; BUF_LEN as usize]; + let mut len = BUF_LEN; + // SAFETY: buffer.len() == len + if unsafe { GetUserNameW(buffer.as_mut_ptr(), &raw mut len) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(OsString::from_wide(&buffer[..len as usize - 1])) + } + } +} + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the `UResult` to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + match uu_app().try_get_matches_from(argv) { + Ok(_matches) => {}, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + } + match whoami_main() { + Ok(()) => pi_uutils_ctx::exit_code(), + Err(err) => { + let code = err.code(); + let msg = err.to_string(); + if !msg.is_empty() { + let _ = writeln!(pi_uutils_ctx::stderr(), "whoami: {msg}"); + } + if code == 0 { 1 } else { code } + }, + } +} + +fn whoami_main() -> UResult<()> { + let username = whoami()?; + // pi-uutils: replacement for upstream's `println_verbatim` — writes the + // username bytes verbatim to the context stdout instead of the process + // stdout. + let mut out = pi_uutils_ctx::stdout(); + out.write_all(uucore::os_str_as_bytes(&username)?) + .and_then(|()| out.write_all(b"\n")) + .and_then(|()| out.flush()) + .map_err(|e| USimpleError::new(1, format!("failed to print username: {e}")))?; + Ok(()) +} + +/// Get the current username +pub fn whoami() -> UResult { + platform::get_username().map_err_context(|| "failed to get username".to_string()) +} + +pub fn uu_app() -> Command { + Command::new("whoami") + .version(uucore::crate_version!()) + .about("Print the current username.") + .override_usage("whoami") + .infer_long_args(true) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + fn run_in(args: Vec<&str>) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(SharedWriter { buf: stdout_buf.clone() }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd: PathBuf::from("."), + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let argv: Vec = std::iter::once("whoami") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + #[test] + fn prints_process_user_with_trailing_newline() { + let (code, stdout, stderr) = run_in(vec![]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert!(stdout.ends_with('\n')); + let name = stdout.trim_end(); + assert!(!name.is_empty()); + // When the host exports USER it names the same effective user the + // platform lookup resolves. + if let Ok(user) = std::env::var("USER") { + assert_eq!(name, user); + } + } + + #[test] + fn rejects_operands() { + let (code, stdout, stderr) = run_in(vec!["extra"]); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert!(!stderr.is_empty(), "clap usage error must go to scope stderr"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_in(vec!["--help"]); + assert_eq!((code, stderr.as_str()), (0, "")); + assert!(stdout.contains("Print the current username.")); + } +} diff --git a/crates/vendor/uu-yes/Cargo.toml b/crates/vendor/uu-yes/Cargo.toml new file mode 100644 index 000000000..ff19aa1d0 --- /dev/null +++ b/crates/vendor/uu-yes/Cargo.toml @@ -0,0 +1,23 @@ +# Vendored from uutils/coreutils tag 0.8.0 (src/uu/yes), patched to route +# output through pi-uutils-ctx, handle a closed consumer (broken pipe) as a +# clean in-process exit, and poll the scope cancel flag so it can run +# in-process as a shell builtin. See src/yes.rs for the patch markers +# (`pi-uutils:` comments). +[package] +name = "uu_yes" +version = "0.8.0" +edition = "2024" +license = "MIT" +description = "yes ~ (uutils) repeatedly display a line with STRING (or 'y') (vendored + patched for in-process embedding)" + +[lib] +path = "src/yes.rs" + +[dependencies] +clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } +itertools = "0.14.0" +uucore = "0.8.0" +pi-uutils-ctx = { path = "../../pi-uutils-ctx" } + +[dev-dependencies] +parking_lot = "0.12" diff --git a/crates/vendor/uu-yes/LICENSE b/crates/vendor/uu-yes/LICENSE new file mode 100644 index 000000000..21bd44404 --- /dev/null +++ b/crates/vendor/uu-yes/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/vendor/uu-yes/src/yes.rs b/crates/vendor/uu-yes/src/yes.rs new file mode 100644 index 000000000..26dea7f91 --- /dev/null +++ b/crates/vendor/uu-yes/src/yes.rs @@ -0,0 +1,351 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +// cSpell:ignore strs + +// pi-uutils: vendored from uutils/coreutils 0.8.0 and patched to run in-process +// as a shell builtin. All process-global stdio is routed through +// `pi_uutils_ctx`, `translate!` strings are literalized, and the entry point no +// longer calls `std::process::exit`. Because the utility runs inside the shell +// process there is no SIGPIPE to terminate it when the consumer closes, so a +// broken-pipe write error exits cleanly with code 0 (GNU behaviour) on every +// platform, and the output loop polls the scope cancel flag so shell +// abort/timeout stops it promptly. + +use std::{ + error::Error, + ffi::OsString, + io::{self, Write}, +}; + +use clap::{Arg, ArgAction, Command, builder::ValueParser}; +use pi_uutils_ctx::format_usage; +use uucore::error::strip_errno; + +// it's possible that using a smaller or larger buffer might provide better +// performance on some systems, but honestly this is good enough +const BUF_SIZE: usize = 16 * 1024; + +/// In-process builtin entry point. Unlike upstream's `uumain`, this parses the +/// arguments directly (without the uucore clap-localization helper that would +/// terminate the process), renders clap help/usage/version to the context +/// streams, and maps the outcome to an exit code, so it is safe to run inside +/// the host shell process. +pub fn run(argv: Vec) -> i32 { + let matches = match uu_app().try_get_matches_from(argv) { + Ok(matches) => matches, + Err(err) => { + let rendered = err.to_string(); + if err.use_stderr() { + let _ = write!(pi_uutils_ctx::stderr(), "{rendered}"); + return 1; + } + let _ = write!(pi_uutils_ctx::stdout(), "{rendered}"); + return 0; + }, + }; + + let mut buffer = Vec::with_capacity(BUF_SIZE); + #[allow(clippy::unwrap_used, reason = "clap provides 'y' by default")] + let _ = args_into_buffer(&mut buffer, matches.get_many::("STRING").unwrap()); + prepare_buffer(&mut buffer); + + match exec(&buffer) { + // pi-uutils: a broken pipe means the consumer closed its end; a + // process `yes` would die from SIGPIPE (or handle EPIPE on Windows), + // so the in-process builtin exits cleanly with 0 on every platform. + ExecStop::Io(err) if err.kind() == io::ErrorKind::BrokenPipe => 0, + ExecStop::Io(err) => { + let _ = writeln!(pi_uutils_ctx::stderr(), "yes: standard output: {}", strip_errno(&err)); + 1 + }, + // pi-uutils: the shell asked the scope to cancel (abort/timeout); + // there is no signal-style exit status in-process, so return 1. + ExecStop::Cancelled => 1, + } +} + +pub fn uu_app() -> Command { + Command::new("yes") + .version(uucore::crate_version!()) + .about("Repeatedly display a line with STRING (or 'y')") + .override_usage(format_usage("yes [STRING]...")) + .arg( + Arg::new("STRING") + .default_value("y") + .value_parser(ValueParser::os_string()) + .action(ArgAction::Append), + ) + .infer_long_args(true) +} + +/// Copies words from `i` into `buf`, separated by spaces. +#[allow(clippy::unnecessary_wraps, reason = "needed on some platforms")] +fn args_into_buffer<'a>( + buf: &mut Vec, + i: impl Iterator, +) -> Result<(), Box> { + // On Unix (and wasi), OsStrs are just &[u8]'s underneath... + #[cfg(any(unix, target_os = "wasi"))] + { + #[cfg(unix)] + use std::os::unix::ffi::OsStrExt; + #[cfg(target_os = "wasi")] + use std::os::wasi::ffi::OsStrExt; + + for part in itertools::intersperse(i.map(|a| a.as_bytes()), b" ") { + buf.extend_from_slice(part); + } + } + + // But, on Windows, we must hop through a String. + #[cfg(not(any(unix, target_os = "wasi")))] + { + for part in itertools::intersperse(i.map(|a| a.to_str()), Some(" ")) { + let bytes = match part { + Some(part) => part.as_bytes(), + // pi-uutils: literalized `translate!("yes-error-invalid-utf8")`. + None => return Err("arguments contain invalid UTF-8".into()), + }; + buf.extend_from_slice(bytes); + } + } + + buf.push(b'\n'); + + Ok(()) +} + +/// Assumes buf holds a single output line forged from the command line +/// arguments, copies it repeatedly until the buffer holds as many copies as it +/// can under [`BUF_SIZE`]. +fn prepare_buffer(buf: &mut Vec) { + let line_len = buf.len(); + debug_assert!(line_len > 0, "buffer is not empty since we have newline"); + let target_size = line_len * (BUF_SIZE / line_len); // 0 if line_len is already large enough + + while buf.len() < target_size { + let to_copy = std::cmp::min(target_size - buf.len(), buf.len()); + debug_assert_eq!(to_copy % line_len, 0); + buf.extend_from_within(..to_copy); + } +} + +/// pi-uutils: why the output loop stopped. Upstream's `exec` only ever returns +/// an I/O error (the loop is infinite); in-process we also stop on scope +/// cancellation. +enum ExecStop { + Io(io::Error), + Cancelled, +} + +/// pi-uutils: replacement for upstream's `exec` — writes to the context stdout +/// instead of the process stdout and polls the scope cancel flag every +/// iteration (each iteration writes a full [`BUF_SIZE`]-ish batch, so polling +/// per iteration is cheap) so shell abort/timeout stops the loop promptly. +fn exec(bytes: &[u8]) -> ExecStop { + let mut stdout = pi_uutils_ctx::stdout(); + + loop { + if pi_uutils_ctx::is_cancelled() { + return ExecStop::Cancelled; + } + if let Err(err) = stdout.write_all(bytes) { + return ExecStop::Io(err); + } + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; + + use parking_lot::Mutex; + use pi_uutils_ctx::ScopeIo; + + use super::*; + + /// Writer that accepts up to `budget` bytes into a shared buffer, then + /// fails every further write with `fail_kind` — models a consumer that + /// closes the pipe after reading some output. + struct FailingWriter { + buf: Arc>>, + budget: usize, + fail_kind: io::ErrorKind, + } + impl Write for FailingWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.budget == 0 { + return Err(io::Error::new(self.fail_kind, "consumer gone")); + } + let n = buf.len().min(self.budget); + self.budget -= n; + self.buf.lock().extend_from_slice(&buf[..n]); + Ok(n) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + fn run_with( + args: Vec<&str>, + budget: usize, + fail_kind: io::ErrorKind, + cancelled: bool, + ) -> (i32, String, String) { + let stdout_buf = Arc::new(Mutex::new(Vec::new())); + let stderr_buf = Arc::new(Mutex::new(Vec::new())); + + #[derive(Clone)] + struct SharedWriter { + buf: Arc>>, + } + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.buf.lock().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.buf.lock().flush() + } + } + + let io = ScopeIo { + stdin: Box::new(std::io::empty()), + stdin_fd: None, + stdin_is_search_input: false, + stdout: Box::new(FailingWriter { + buf: stdout_buf.clone(), + budget, + fail_kind, + }), + stderr: Box::new(SharedWriter { buf: stderr_buf.clone() }), + cwd: PathBuf::from("."), + env: HashMap::new(), + cancel: Arc::new(std::sync::atomic::AtomicBool::new(cancelled)), + }; + + let argv: Vec = std::iter::once("yes") + .chain(args) + .map(OsString::from) + .collect(); + + let code = pi_uutils_ctx::scope(io, || run(argv)); + + let out_str = String::from_utf8(stdout_buf.lock().clone()).unwrap(); + let err_str = String::from_utf8(stderr_buf.lock().clone()).unwrap(); + + (code, out_str, err_str) + } + + #[test] + fn broken_pipe_is_clean_exit() { + // Consumer takes 100 bytes then closes: exit 0, like GNU yes dying to + // SIGPIPE without an error status visible to the shell. + let (code, stdout, stderr) = run_with(vec![], 100, io::ErrorKind::BrokenPipe, false); + assert_eq!(code, 0); + assert!(stdout.starts_with("y\ny\n"), "expected default 'y' lines, got {stdout:?}"); + assert_eq!(stdout.len(), 100); + assert_eq!(stderr, ""); + } + + #[test] + fn custom_operands_join_with_spaces_and_repeat() { + // Budget is a multiple of the line length ("hello world\n" = 12 bytes) + // so the captured output is whole lines. + let (code, stdout, stderr) = + run_with(vec!["hello", "world"], 12 * 100, io::ErrorKind::BrokenPipe, false); + assert_eq!(code, 0); + assert_eq!(stdout.lines().count(), 100); + for line in stdout.lines() { + assert_eq!(line, "hello world"); + } + assert_eq!(stderr, ""); + } + + #[test] + fn cancellation_stops_loop_promptly() { + // Pre-set cancel flag: the loop must observe it and return 1 before + // writing anything. The finite write budget is a backstop so a broken + // cancel path fails the test (as exit 0) instead of hanging forever. + let (code, stdout, stderr) = run_with(vec![], 1 << 20, io::ErrorKind::BrokenPipe, true); + assert_eq!(code, 1); + assert_eq!(stdout, ""); + assert_eq!(stderr, ""); + } + + #[test] + fn non_pipe_write_error_reports_and_fails() { + let (code, stdout, stderr) = run_with(vec![], 2, io::ErrorKind::Other, false); + assert_eq!(code, 1); + assert_eq!(stdout, "y\n"); + assert_eq!(stderr, "yes: standard output: consumer gone\n"); + } + + #[test] + fn help_renders_to_scope_stdout() { + let (code, stdout, stderr) = run_with(vec!["--help"], 1 << 20, io::ErrorKind::Other, false); + assert_eq!(code, 0); + assert!(stdout.contains("Usage:")); + assert!(stdout.contains("Repeatedly display a line")); + assert_eq!(stderr, ""); + } + + // Upstream unit tests (uutils/coreutils 0.8.0), kept verbatim apart from + // indentation. + + #[test] + fn test_prepare_buffer() { + let tests = [ + (150, 16350), + (1000, 16000), + (4093, 16372), + (4099, 12297), + (4111, 12333), + (2, 16384), + (3, 16383), + (4, 16384), + (5, 16380), + (8192, 16384), + (8191, 16382), + (8193, 8193), + (10000, 10000), + (15000, 15000), + (25000, 25000), + ]; + + for (line, final_len) in tests { + let mut v = std::iter::repeat_n(b'a', line).collect::>(); + prepare_buffer(&mut v); + assert_eq!(v.len(), final_len); + } + } + + #[test] + fn test_args_into_buf() { + { + let mut v = Vec::with_capacity(BUF_SIZE); + let default_args = ["y".into()]; + args_into_buffer(&mut v, default_args.iter()).unwrap(); + assert_eq!(String::from_utf8(v).unwrap(), "y\n"); + } + + { + let mut v = Vec::with_capacity(BUF_SIZE); + let args = ["foo".into()]; + args_into_buffer(&mut v, args.iter()).unwrap(); + assert_eq!(String::from_utf8(v).unwrap(), "foo\n"); + } + + { + let mut v = Vec::with_capacity(BUF_SIZE); + let args = ["foo".into(), "bar baz".into(), "qux".into()]; + args_into_buffer(&mut v, args.iter()).unwrap(); + assert_eq!(String::from_utf8(v).unwrap(), "foo bar baz qux\n"); + } + } +} diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index f45d4778f..2852d393e 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +### Added + +- Added an in-process `readlink` shell builtin (vendored from uutils coreutils 0.8.0), supporting `-f`/`-e`/`-m` canonicalization, `-n`/`-z` delimiters, and `-v`/`-q`/`-s` verbosity, with path operands resolved against the shell working directory. +- Added in-process shell builtins for `realpath`, `touch`, `stat`, `date`, `mktemp`, `seq`, `yes`, `printenv`, `ln`, `truncate`, `tac`, `nproc`, `uname`, `whoami`, and `hostname` (vendored from uutils coreutils 0.8.0), plus native `which` (shell PATH lookup) and `diff` (unified output, `-U`/`-q`/`-N`, binary detection, recursive directory compare) builtins. All resolve path operands against the shell working directory, read the shell's exported environment, and honor abort/timeout cancellation; `ln` is gated with the destructive set (`PI_DISABLE_UUTILS_DESTRUCTIVE`), and system-mutating modes (`date --set`, hostname setting) are disabled. + ## [16.4.5] - 2026-07-11 ### Added From 43f8999a9b18aeb82c1e2a989d49b4ac537f06a9 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 20:53:58 +0200 Subject: [PATCH 06/80] feat(ai): added cache invalidation for usage reports - Introduced a `POST /v1/usage/stale` endpoint to allow remote clients to invalidate server-side usage caches. - Updated `AuthCredentialStore` to include an optional `invalidateUsageCache` hook. - Added `invalidateUsageCache` method to `AuthStorage` for local cache invalidation and notification of remote stores. --- packages/ai/CHANGELOG.md | 5 +++ packages/ai/src/auth-broker/client.ts | 9 ++++++ packages/ai/src/auth-broker/remote-store.ts | 7 +++++ packages/ai/src/auth-broker/server.ts | 11 +++++++ packages/ai/src/auth-broker/types.ts | 5 +++ packages/ai/src/auth-broker/wire-schemas.ts | 5 +++ packages/ai/src/auth-storage.ts | 35 +++++++++++++++++++++ packages/ai/test/remote-auth-store.test.ts | 19 +++++++++++ packages/coding-agent/CHANGELOG.md | 3 +- 9 files changed, 97 insertions(+), 2 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index a3541d305..4288cc722 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +### Added + +- Added `invalidateUsageCache` method to manually clear cached usage reports +- Added support for cross-service usage cache invalidation between AuthStorage and AuthBroker + ## [16.4.5] - 2026-07-11 ### Fixed diff --git a/packages/ai/src/auth-broker/client.ts b/packages/ai/src/auth-broker/client.ts index d873acfe6..c90d83f4f 100644 --- a/packages/ai/src/auth-broker/client.ts +++ b/packages/ai/src/auth-broker/client.ts @@ -21,6 +21,7 @@ import type { SnapshotResponse, SnapshotStreamEvent, UsageResponse, + UsageStaleResponse, } from "./types"; import { credentialBlockResponseSchema, @@ -32,6 +33,7 @@ import { snapshotResponseSchema, snapshotStreamEventSchema, usageResponseSchema, + usageStaleResponseSchema, } from "./wire-schemas"; export interface AuthBrokerClientOptions { @@ -242,6 +244,13 @@ export class AuthBrokerClient { return this.#request("GET", "/v1/usage", { schema: usageResponseSchema, signal }); } + notifyUsageStale(signal?: AbortSignal): Promise { + return this.#request("POST", "/v1/usage/stale", { + schema: usageStaleResponseSchema, + signal, + }); + } + async refreshCredential(id: number, signal?: AbortSignal): Promise { return this.#request("POST", `/v1/credential/${id}/refresh`, { schema: credentialRefreshResponseSchema, diff --git a/packages/ai/src/auth-broker/remote-store.ts b/packages/ai/src/auth-broker/remote-store.ts index 4351fe616..0a13a9cfe 100644 --- a/packages/ai/src/auth-broker/remote-store.ts +++ b/packages/ai/src/auth-broker/remote-store.ts @@ -794,6 +794,13 @@ export class RemoteAuthCredentialStore implements AuthCredentialStore { } } + async invalidateUsageCache(signal?: AbortSignal): Promise { + this.#invalidateUsageCache(); + await this.#client.notifyUsageStale(signal).catch(err => { + logger.warn("auth-broker notification of stale usage failed", { error: String(err) }); + }); + } + #invalidateUsageCache(): void { this.#usageCache = undefined; this.#usageInflight = undefined; diff --git a/packages/ai/src/auth-broker/server.ts b/packages/ai/src/auth-broker/server.ts index bf5b8afb3..ce8bdf06f 100644 --- a/packages/ai/src/auth-broker/server.ts +++ b/packages/ai/src/auth-broker/server.ts @@ -608,6 +608,17 @@ export function startAuthBroker(opts: AuthBrokerServerOptions): AuthBrokerServer return json(502, { error: message }); } } + if (req.method === "POST" && pathname === "/v1/usage/stale") { + try { + opts.storage.invalidateUsageCache?.(); + logger.info("auth-broker usage cache invalidated", { peer }); + return json(200, { ok: true }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.warn("auth-broker usage cache invalidation failed", { peer, error: message }); + return json(500, { error: message }); + } + } const refreshMatch = req.method === "POST" ? pathname.match(REFRESH_ROUTE) : null; if (refreshMatch) { const id = Number.parseInt(refreshMatch[1], 10); diff --git a/packages/ai/src/auth-broker/types.ts b/packages/ai/src/auth-broker/types.ts index 6a329d2ca..d571215cf 100644 --- a/packages/ai/src/auth-broker/types.ts +++ b/packages/ai/src/auth-broker/types.ts @@ -75,6 +75,11 @@ export interface CredentialBlocksDeleteResponse { ok: boolean; } +/** POST /v1/usage/stale response body. */ +export interface UsageStaleResponse { + ok: boolean; +} + /** * POST /v1/credential request body. The OAuth `refresh` must be the *real* * refresh token (not the sentinel) — the broker is the canonical writer. diff --git a/packages/ai/src/auth-broker/wire-schemas.ts b/packages/ai/src/auth-broker/wire-schemas.ts index 35180ebf1..bece9ce55 100644 --- a/packages/ai/src/auth-broker/wire-schemas.ts +++ b/packages/ai/src/auth-broker/wire-schemas.ts @@ -258,6 +258,11 @@ export const credentialBlocksDeleteResponseSchema = type({ ok: "boolean", }); +export const usageStaleResponseSchema = type({ + "+": "reject", + ok: "boolean", +}); + // ─── Upload ──────────────────────────────────────────────────────────────── export const credentialUploadRequestSchema = type({ diff --git a/packages/ai/src/auth-storage.ts b/packages/ai/src/auth-storage.ts index 382e053e2..f4d3bd362 100644 --- a/packages/ai/src/auth-storage.ts +++ b/packages/ai/src/auth-storage.ts @@ -315,6 +315,8 @@ export interface CredentialRefreshLeaseFence { export interface AuthCredentialStore { close(): void; + /** Optional hook to notify the underlying store that usage report cache is stale. */ + invalidateUsageCache?(signal?: AbortSignal): Promise; listAuthCredentials(provider?: string): StoredAuthCredential[]; updateAuthCredential(id: number, credential: AuthCredential): void; deleteAuthCredential(id: number, disabledCause: string): void; @@ -4666,6 +4668,11 @@ export class AuthStorage { }); if (result.ok) { this.#invalidateUsageReportCache(provider, baseUrl); + if (this.#store.invalidateUsageCache) { + await this.#store.invalidateUsageCache(options.signal).catch(err => { + logger.debug("Failed to notify store of stale usage", { err }); + }); + } // The window this credential was blocked on (by markUsageLimitReached) // is now reset, so lift its temporary block — otherwise selection // keeps skipping/under-ranking the freshly-reset account. @@ -4691,6 +4698,34 @@ export class AuthStorage { } } + /** + * Force-invalidate cached usage reports so the next fetch retrieves fresh + * values from upstream providers. If `provider` is specified, only that + * provider's credentials are invalidated; otherwise, all credentials in the + * store are invalidated. + */ + invalidateUsageCache(provider?: string): void { + if (provider) { + this.#invalidateUsageReportCache(provider); + return; + } + this.#usageCacheEpoch += 1; + const expired = Date.now() - 1; + try { + const credentials = this.#store.listAuthCredentials(); + for (const entry of credentials) { + if (entry.credential.type !== "oauth") continue; + const cacheKey = this.#buildUsageReportCacheKey( + this.#buildUsageRequestForOauth(entry.provider, entry.credential), + ); + const existing = this.#usageCache.getStale(cacheKey); + this.#usageCache.set(cacheKey, { value: existing?.value ?? null, expiresAt: expired }); + } + } catch (err) { + logger.debug("Failed to list auth credentials for complete usage cache invalidation", { err }); + } + } + #invalidateUsageReportCacheForProviderKey(providerKey: string): void { const oauthSuffix = ":oauth"; if (!providerKey.endsWith(oauthSuffix)) return; diff --git a/packages/ai/test/remote-auth-store.test.ts b/packages/ai/test/remote-auth-store.test.ts index e2c81b38f..0167f7569 100644 --- a/packages/ai/test/remote-auth-store.test.ts +++ b/packages/ai/test/remote-auth-store.test.ts @@ -731,4 +731,23 @@ describe("RemoteAuthCredentialStore + AuthStorage integration", () => { expect(clientStorage.get("kagi")).toBeUndefined(); clientStorage.close(); }); + + test("client AuthStorage invalidateUsageCache notifies broker to invalidate server-side cache", async () => { + const brokerClient = new AuthBrokerClient({ url: handle!.url, token }); + const initialResult = await brokerClient.fetchSnapshot(); + if (initialResult.status !== 200) throw new Error("expected snapshot"); + const remoteStore = new RemoteAuthCredentialStore({ + client: brokerClient, + initialSnapshot: initialResult.snapshot, + }); + const clientStorage = new AuthStorage(remoteStore); + await clientStorage.reload(); + + const serverInvalidateSpy = vi.spyOn(serverStorage!, "invalidateUsageCache"); + + await remoteStore.invalidateUsageCache(); + + expect(serverInvalidateSpy).toHaveBeenCalled(); + clientStorage.close(); + }); }); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d4c69f45c..5a680b4ff 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -9,6 +9,7 @@ ### Fixed - Fixed compiled release binaries bundling `fastembed` and baking the build-machine `@anush008/tokenizers` path; native runtime dependencies now stay external for every compiled build path so Mnemopi resolves its on-demand install instead. ([#5195](https://github.com/can1357/oh-my-pi/issues/5195)) +- Fixed `/btw` side-channel turns on Codex models such as `gpt-5.6-luna` by preserving the session websocket preference instead of forcing SSE, and made Esc dismiss the active `/btw` panel before interrupting loop/maintenance work. ([#5213](https://github.com/can1357/oh-my-pi/issues/5213)) ## [16.4.5] - 2026-07-11 @@ -40,8 +41,6 @@ - Fixed compiled Linux binary extension loading failures related to bundled web-search header generation data paths. - Fixed job list and empty-poll snapshots returning empty output, ensuring running subagents without backing jobs are properly listed. - Fixed agents getting stuck waiting for messages from peers that have already stopped running. -- Fixed `/btw` side-channel turns on Codex models such as `gpt-5.6-luna` by preserving the session websocket preference instead of forcing SSE, and made Esc dismiss the active `/btw` panel before interrupting loop/maintenance work. ([#5213](https://github.com/can1357/oh-my-pi/issues/5213)) -- Fixed visible per-keystroke lag while searching in the `/resume` session picker. Literal matches now rank synchronously from a cached per-session haystack, fuzzy scoring runs in bounded background chunks that converge to the same ranking (large listings previously rebuilt a fuzzy index per token per session on every keystroke), and the prompt-history SQLite lookup — an FTS query plus a LIKE scan over every stored prompt — is debounced off the keystroke path. - Fixed compiled Linux binary extension loading when bundled web-search header generation cannot read `header-generator` data files from the build-time path. ([#5178](https://github.com/can1357/oh-my-pi/issues/5178)) - Fixed plugin custom tool loading to skip and report invalid feature entries instead of crashing startup when a plugin dependency tree leaves one feature unresolved. ([#5189](https://github.com/can1357/oh-my-pi/issues/5189)) From 6bb0878b638167f998aa1c0dedb5386890b5f361 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 20:55:19 +0200 Subject: [PATCH 07/80] feat(ai): added cache invalidation command for usage reports - Added an `invalidate` action to the usage CLI to clear cached usage reports for specific or all providers. - Updated `invalidateUsageCache` in `AuthStorage` to be asynchronous and notify the underlying store of invalidations. --- packages/ai/CHANGELOG.md | 2 +- packages/ai/src/auth-storage.ts | 38 ++++++++++++--------- packages/coding-agent/CHANGELOG.md | 4 +++ packages/coding-agent/src/cli/usage-cli.ts | 11 ++++++ packages/coding-agent/src/commands/usage.ts | 15 ++++++-- 5 files changed, 51 insertions(+), 19 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 4288cc722..9dc58b1da 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- Added `invalidateUsageCache` method to manually clear cached usage reports +- Added asynchronous `invalidateUsageCache` method to clear cached usage reports - Added support for cross-service usage cache invalidation between AuthStorage and AuthBroker ## [16.4.5] - 2026-07-11 diff --git a/packages/ai/src/auth-storage.ts b/packages/ai/src/auth-storage.ts index f4d3bd362..b88ae2b86 100644 --- a/packages/ai/src/auth-storage.ts +++ b/packages/ai/src/auth-storage.ts @@ -4704,25 +4704,31 @@ export class AuthStorage { * provider's credentials are invalidated; otherwise, all credentials in the * store are invalidated. */ - invalidateUsageCache(provider?: string): void { + async invalidateUsageCache(provider?: string, signal?: AbortSignal): Promise { if (provider) { this.#invalidateUsageReportCache(provider); - return; - } - this.#usageCacheEpoch += 1; - const expired = Date.now() - 1; - try { - const credentials = this.#store.listAuthCredentials(); - for (const entry of credentials) { - if (entry.credential.type !== "oauth") continue; - const cacheKey = this.#buildUsageReportCacheKey( - this.#buildUsageRequestForOauth(entry.provider, entry.credential), - ); - const existing = this.#usageCache.getStale(cacheKey); - this.#usageCache.set(cacheKey, { value: existing?.value ?? null, expiresAt: expired }); + } else { + this.#usageCacheEpoch += 1; + const expired = Date.now() - 1; + try { + const credentials = this.#store.listAuthCredentials(); + for (const entry of credentials) { + if (entry.credential.type !== "oauth") continue; + const cacheKey = this.#buildUsageReportCacheKey( + this.#buildUsageRequestForOauth(entry.provider, entry.credential), + ); + const existing = this.#usageCache.getStale(cacheKey); + this.#usageCache.set(cacheKey, { value: existing?.value ?? null, expiresAt: expired }); + } + } catch (err) { + logger.debug("Failed to list auth credentials for complete usage cache invalidation", { err }); } - } catch (err) { - logger.debug("Failed to list auth credentials for complete usage cache invalidation", { err }); + } + + if (this.#store.invalidateUsageCache) { + await this.#store.invalidateUsageCache(signal).catch(err => { + logger.debug("Failed to notify store of stale usage", { err }); + }); } } diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 5a680b4ff..521ff3b68 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added `invalidate` action to the usage command to clear cached usage reports + ### Changed - Retain completed and abandoned tasks in session history for improved context on resume diff --git a/packages/coding-agent/src/cli/usage-cli.ts b/packages/coding-agent/src/cli/usage-cli.ts index 8dfe9ca90..c32ab65ec 100644 --- a/packages/coding-agent/src/cli/usage-cli.ts +++ b/packages/coding-agent/src/cli/usage-cli.ts @@ -23,6 +23,7 @@ import { discoverAuthStorage } from "../sdk"; const BAR_WIDTH = 28; export interface UsageCommandArgs { + action?: string; json?: boolean; provider?: string; redact?: boolean; @@ -755,6 +756,16 @@ function redactReportForJson( export async function runUsageCommand(cmd: UsageCommandArgs): Promise { const authStorage = await discoverAuthStorage(); try { + if (cmd.action === "invalidate") { + const provider = cmd.provider?.toLowerCase(); + await authStorage.invalidateUsageCache(provider); + if (provider) { + process.stdout.write(`Invalidated cached usage reports for provider "${provider}".\n`); + } else { + process.stdout.write("Invalidated cached usage reports for all providers.\n"); + } + return; + } if (cmd.history) { const days = cmd.days !== undefined && Number.isFinite(cmd.days) && cmd.days > 0 ? cmd.days : 7; const nowMs = Date.now(); diff --git a/packages/coding-agent/src/commands/usage.ts b/packages/coding-agent/src/commands/usage.ts index 06d856a4b..bfaf3a3f3 100644 --- a/packages/coding-agent/src/commands/usage.ts +++ b/packages/coding-agent/src/commands/usage.ts @@ -1,12 +1,20 @@ /** * Show provider usage limits for every authenticated account. */ -import { Command, Flags } from "@oh-my-pi/pi-utils/cli"; +import { Args, Command, Flags } from "@oh-my-pi/pi-utils/cli"; import { runUsageCommand } from "../cli/usage-cli"; export default class Usage extends Command { static description = "Show provider usage limits for every authenticated account"; + static args = { + action: Args.string({ + description: "Optional subcommand to execute", + required: false, + options: ["invalidate"], + }), + }; + static flags = { json: Flags.boolean({ char: "j", description: "Output usage reports as JSON", default: false }), provider: Flags.string({ char: "p", description: "Only show usage for this provider id (e.g. anthropic)" }), @@ -28,11 +36,14 @@ export default class Usage extends Command { "# Redact account identifiers for screenshots\n omp usage --redact", "# Machine-readable output\n omp usage --json", "# Usage-limit trend over the last 30 days\n omp usage --history --days 30", + "# Invalidate cached usage reports for all providers\n omp usage invalidate", + "# Invalidate cached usage reports for a specific provider\n omp usage invalidate --provider anthropic", ]; async run(): Promise { - const { flags } = await this.parse(Usage); + const { args, flags } = await this.parse(Usage); await runUsageCommand({ + action: args.action, json: flags.json, provider: flags.provider, redact: flags.redact, From 7cef4a769efa767d9e71e5e0ddd000c9283773d3 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 21:01:47 +0200 Subject: [PATCH 08/80] fix(ai): improved OAuth credential resolution fallback logic - Implement a multi-pass resolution ladder to handle scenarios where plan-eligible accounts are usage-blocked. - Ensure that exhausted paid accounts are prioritized over unblocked free accounts for plan-gated models, allowing callers to receive proper usage-limit error semantics from the API provider. - Add a test case to verify credential selection when all plan-eligible accounts are currently rate-limited. --- packages/ai/CHANGELOG.md | 4 + packages/ai/src/auth-storage.ts | 75 ++++++++++--------- .../test/auth-storage-codex-selection.test.ts | 40 ++++++++++ 3 files changed, 82 insertions(+), 37 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 9dc58b1da..de037cadb 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -7,6 +7,10 @@ - Added asynchronous `invalidateUsageCache` method to clear cached usage reports - Added support for cross-service usage cache invalidation between AuthStorage and AuthBroker +### Fixed + +- Fixed OAuth credential resolution returning "No API key found" when every plan-eligible OpenAI Codex account was rate-limit blocked and the only unblocked account failed the model's plan gate: resolution now runs a last-resort ladder that first yields a plan-fitting account regardless of usage blocks (so callers get real usage-limit retry semantics), then tries every account with the plan filter dropped before reporting no credential + ## [16.4.5] - 2026-07-11 ### Fixed diff --git a/packages/ai/src/auth-storage.ts b/packages/ai/src/auth-storage.ts index b88ae2b86..245c118c6 100644 --- a/packages/ai/src/auth-storage.ts +++ b/packages/ai/src/auth-storage.ts @@ -3741,8 +3741,17 @@ export class AuthStorage { /** * Resolves an OAuth credential, trying credentials in priority order. - * Skips blocked credentials and checks usage limits for providers with usage data. - * Falls back to earliest-unblocking credential if all are blocked. + * + * Resolution ladder — a request in hand always beats "no API key": + * 1. strict: unblocked credentials only, usage limits respected, plan + * filter enforced (when any account is confirmed eligible); + * 2. plan-fitting last resort: same plan filter, but blocked/exhausted + * accounts are allowed (blocked candidates rank earliest-unblocking + * first) so the caller gets real usage-limit semantics from the wire + * instead of a missing key; + * 3. unfiltered last resort: the plan filter matched nothing usable — + * skip it and try every account once; the server is the final arbiter + * of model access. * * Returns both the API key bytes for outbound requests AND the refreshed * {@link OAuthCredential} so callers needing identity metadata (account id, @@ -3892,42 +3901,34 @@ export class AuthStorage { hasPlanRequirement && candidates.some(candidate => getOpenAICodexPlanEligibility(candidate.usage, planRequirement) === true); - const fallback = candidates[0]; + const passes: Array<{ allowBlocked: boolean; enforcePlanRequirement: boolean }> = [ + { allowBlocked: false, enforcePlanRequirement }, + { allowBlocked: true, enforcePlanRequirement }, + ]; + if (enforcePlanRequirement) passes.push({ allowBlocked: true, enforcePlanRequirement: false }); - for (const candidate of candidates) { - const resolved = await this.#tryOAuthCredential( - provider, - candidate.selection, - providerKey, - sessionId, - options, - { - checkUsage, - allowBlocked: false, - prefetchedUsage: candidate.usage, - usagePrechecked: candidate.usageChecked, - planRequirement, - enforcePlanRequirement, - strategy, - rankingContext, - blockScope, - }, - ); - if (resolved) return resolved; - } - - if (fallback && this.#isCredentialBlocked(provider, providerKey, fallback.selection.index, blockScope)) { - return this.#tryOAuthCredential(provider, fallback.selection, providerKey, sessionId, options, { - checkUsage, - allowBlocked: true, - prefetchedUsage: fallback.usage, - usagePrechecked: fallback.usageChecked, - planRequirement, - enforcePlanRequirement, - strategy, - rankingContext, - blockScope, - }); + for (const pass of passes) { + for (const candidate of candidates) { + const resolved = await this.#tryOAuthCredential( + provider, + candidate.selection, + providerKey, + sessionId, + options, + { + checkUsage, + allowBlocked: pass.allowBlocked, + prefetchedUsage: candidate.usage, + usagePrechecked: candidate.usageChecked, + planRequirement, + enforcePlanRequirement: pass.enforcePlanRequirement, + strategy, + rankingContext, + blockScope, + }, + ); + if (resolved) return resolved; + } } return undefined; diff --git a/packages/ai/test/auth-storage-codex-selection.test.ts b/packages/ai/test/auth-storage-codex-selection.test.ts index 50d859786..3a0cf25a9 100644 --- a/packages/ai/test/auth-storage-codex-selection.test.ts +++ b/packages/ai/test/auth-storage-codex-selection.test.ts @@ -1566,6 +1566,46 @@ describe("AuthStorage codex oauth ranking", () => { expect(apiKey).toBe("api-acct-go"); }); + test("yields an exhausted paid account over an idle free account for a paid-gated model", async () => { + if (!authStorage) throw new Error("test setup failed"); + + // Regression: with every plan-eligible account usage-blocked and an + // unblocked free account present, resolution used to return NO + // credential at all ("No API key found") because the old last-resort + // fallback only fired when the top-ranked candidate happened to be + // blocked — and the idle free account ranked first. The plan-fitting + // last-resort pass must yield the exhausted paid account instead, so + // the caller gets real usage-limit semantics from the wire. + await authStorage.set("openai-codex", [ + { type: "oauth", ...createCredential("acct-free", "free@example.com") }, + { type: "oauth", ...createCredential("acct-paid", "paid@example.com") }, + ]); + + usageByAccount.set( + "acct-free", + createCodexUsageReport({ + accountId: "acct-free", + primary: { usedFraction: 0.05, resetInMs: 30 * 60 * 1000 }, + secondary: { usedFraction: 0.05, resetInMs: 6 * 24 * 60 * 60 * 1000 }, + metadata: { planType: "free", email: "free@example.com" }, + }), + ); + usageByAccount.set( + "acct-paid", + createCodexUsageReport({ + accountId: "acct-paid", + primary: { usedFraction: 1, resetInMs: 2 * HOUR_MS }, + secondary: { usedFraction: 1, resetInMs: 6 * 24 * 60 * 60 * 1000 }, + metadata: { planType: "plus", email: "paid@example.com", limitReached: true }, + }), + ); + + const apiKey = await authStorage.getApiKey("openai-codex", "session-paid-gated-all-blocked", { + modelId: "gpt-5.6-sol", + }); + expect(apiKey).toBe("api-acct-paid"); + }); + test("prefers Pro accounts for codex spark models over Plus accounts", async () => { if (!authStorage) throw new Error("test setup failed"); From 2ab2da2b1d18f8360963d20cb32942b9383d4383 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 21:17:35 +0200 Subject: [PATCH 09/80] fix(coding-agent): improved role-assignment strip visibility on overflow - Prevent selected chips from being hidden when the role strip row overflows. - Implement horizontal scrolling with a leading ellipsis to maintain visibility of the selection and lookahead context. --- packages/coding-agent/CHANGELOG.md | 1 + .../src/modes/components/model-hub.ts | 32 +++++++++++++++++-- packages/coding-agent/test/model-hub.test.ts | 21 ++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 521ff3b68..1bd6d9883 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -14,6 +14,7 @@ - Fixed compiled release binaries bundling `fastembed` and baking the build-machine `@anush008/tokenizers` path; native runtime dependencies now stay external for every compiled build path so Mnemopi resolves its on-demand install instead. ([#5195](https://github.com/can1357/oh-my-pi/issues/5195)) - Fixed `/btw` side-channel turns on Codex models such as `gpt-5.6-luna` by preserving the session websocket preference instead of forcing SSE, and made Esc dismiss the active `/btw` panel before interrupting loop/maintenance work. ([#5213](https://github.com/can1357/oh-my-pi/issues/5213)) +- Fixed the Model Hub role-assignment strip hiding the selected chip once the row overflowed; the strip now scrolls horizontally, truncating passed chips behind a leading ellipsis so the selection (plus one chip of lookahead) stays visible. ## [16.4.5] - 2026-07-11 diff --git a/packages/coding-agent/src/modes/components/model-hub.ts b/packages/coding-agent/src/modes/components/model-hub.ts index 752174779..ef28c2bb7 100644 --- a/packages/coding-agent/src/modes/components/model-hub.ts +++ b/packages/coding-agent/src/modes/components/model-hub.ts @@ -1597,10 +1597,38 @@ export class ModelHubComponent implements Component { ? `${theme.fg("accent", strip.item.id)}${theme.fg("dim", " →")} ` : `${theme.fg(getRoleInfo(strip.role ?? "", this.#settings).color ?? "muted", (getRoleInfo(strip.role ?? "", this.#settings).tag ?? strip.role ?? "").toLowerCase())}${theme.fg("dim", ` · ${strip.item.id} →`)} `; + // Horizontal window: once the strip overflows, drop leading chips behind + // a dim ellipsis so the selected chip (plus one chip of lookahead when it + // fits) stays visible while cycling right. + const prefixWidth = visibleWidth(prefix); + const available = Math.max(1, width - prefixWidth); + const chipWidths = strip.chips.map( + (chip, i) => visibleWidth(` ${chip.styled} `) + (i === strip.index ? 2 : 0) + 1, + ); + // Smallest start index whose window [start..target] (with its "… " lead-in + // when start > 0) fits in the available width; `target` itself may still + // overflow when a single chip is wider than the row. + const startFor = (target: number): number => { + let start = 0; + while (start < target) { + let sum = start > 0 ? 2 : 0; + for (let i = start; i <= target; i++) sum += chipWidths[i] ?? 0; + if (sum <= available) break; + start++; + } + return start; + }; + let start = startFor(Math.min(strip.index + 1, strip.chips.length - 1)); + if (start > strip.index) start = startFor(strip.index); + let line = prefix; // Columns are relative to the frame: row() insets content by 2. - let col = 2 + visibleWidth(prefix); - for (let i = 0; i < strip.chips.length; i++) { + let col = 2 + prefixWidth; + if (start > 0) { + line += theme.fg("dim", "… "); + col += 2; + } + for (let i = start; i < strip.chips.length; i++) { const chip = strip.chips[i]; if (!chip) continue; const selected = i === strip.index; diff --git a/packages/coding-agent/test/model-hub.test.ts b/packages/coding-agent/test/model-hub.test.ts index 34991413b..a9d3bed7c 100644 --- a/packages/coding-agent/test/model-hub.test.ts +++ b/packages/coding-agent/test/model-hub.test.ts @@ -394,6 +394,27 @@ describe("ModelHub", () => { expect(call?.[4]).toBe("retryFallback"); expect(footerLine(hub.render(220))).not.toContain("inherit"); }); + + test("overflowing role strip scrolls left so the selected chip stays visible", () => { + const model = makeModel("test", "narrow-strip-model"); + const { hub } = createHub({ models: [model], scoped: true }); + installTestTheme(); + + hub.handleInput("\n"); // open the role strip + // At full width every chip fits and no left ellipsis appears. + expect(footerLine(hub.render(220))).not.toContain("…"); + + hub.handleInput(LEFT); // wrap to the trailing retry-fallback chip + const narrow = footerLine(hub.render(80)); + expect(narrow).toContain("[ retry-fallback ]"); + expect(narrow).toContain("…"); + + // Back on the first chip the window resets — no leading ellipsis. + hub.handleInput("\x1b[C"); // wrap right back to the first chip + const reset = footerLine(hub.render(80)); + expect(reset).toContain("[ default"); + expect(reset.trimStart().startsWith("…")).toBe(false); + }); }); describe("pick mode", () => { From 54bafa1cceb4326fa55c7f155b09b838076e94b5 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 21:52:28 +0200 Subject: [PATCH 10/80] feat(coding-agent): implemented interactive fallback chain configuration - Implemented model-specific keys and provider wildcards for `retry.fallbackChains` with updated resolution logic. - Added interactive fallback chain management in the model roles UI, including support for reordering and editing. - Improved fallback chain specificity rules and added comprehensive validation with startup warnings. - Fixed mouse interaction alignment and hover state coordinate mapping in the roles view. --- README.md | 2 +- docs/settings.md | 31 +- .../test/auth-storage-codex-selection.test.ts | 38 ++ packages/coding-agent/CHANGELOG.md | 5 + .../src/config/settings-schema.ts | 2 +- .../src/modes/components/model-hub.ts | 487 ++++++++++++++---- .../modes/controllers/selector-controller.ts | 33 +- .../coding-agent/src/session/agent-session.ts | 156 +++++- .../test/agent-session-retry-fallback.test.ts | 242 +++++++++ packages/coding-agent/test/model-hub.test.ts | 212 +++++++- 10 files changed, 1060 insertions(+), 148 deletions(-) diff --git a/README.md b/README.md index 49a3fb43f..a9d03c0f5 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,7 @@ Ollama `local` · Ollama Cloud · LM Studio `local` · llama.cpp `local` · vLLM ### Four knobs that make routing useful - **Custom providers** — Declare anything that speaks `openai-completions`, `openai-responses`, `openai-codex-responses`, `azure-openai-responses`, `anthropic-messages`, `google-generative-ai`, or `google-vertex` in `~/.omp/agent/models.yml`. -- **Fallback chains** — Per-role chains under `retry.fallbackChains`. When the primary throws 429s or hits a quota wall, the next entry takes the rest of the turn — restored on cooldown. +- **Fallback chains** — Per-role or per-model chains under `retry.fallbackChains`. When the primary throws 429s or hits a quota wall, the next entry takes the rest of the turn — restored on cooldown. - **Path-scoped models** — Scope `enabledModels` and `disabledProviders` entries to a `path:` prefix to pin a different model set on one repo without touching the global config. Scoped entries cover the path and everything under it. - **Round-robin credentials** — Stack API keys per provider and the runtime rotates with session affinity and per-credential backoff. Useful when one key would burn its quota by lunch. diff --git a/docs/settings.md b/docs/settings.md index 655c41ee7..e385b1c8d 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -385,6 +385,31 @@ retry: maxDelayMs: 300000 modelFallback: true fallbackRevertPolicy: cooldown-expiry + fallbackChains: + # Any role without an explicit chain inherits the "default" chain. + default: + - anthropic/claude-opus-4-5 + - openai/gpt-5.5 + - google/gemini-3-pro + # Per-role chains override the default (roles from `modelRoles`, + # including custom roles). Selectors accept an optional thinking + # suffix, e.g. openai/gpt-5.5:low. + smol: + - openai/gpt-5.5-mini + - anthropic/claude-haiku-4-5 + # Model-selector keys (any key containing "/") attach the chain to the + # model itself: it applies whenever that model is active, no matter + # which role it is assigned to, and survives role reassignment. + google/gemini-3-pro: + - google-vertex/gemini-3-pro + # A `provider/*` KEY covers every model of a provider — current or + # future. A `provider/*` ENTRY keeps the failing model's id and swaps + # the provider: google-antigravity/x -> google/x -> google-vertex/x. + # Ids missing on the target provider are skipped (near-miss ids resolve + # fuzzily); exact model keys override the wildcard for a specific model. + google-antigravity/*: + - google/* + - google-vertex/* ``` | Key | Type | Default | Notes | @@ -394,8 +419,10 @@ retry: | `retry.baseDelayMs` | number | `500` | Initial backoff. | | `retry.maxDelayMs` | number | `300000` | Backoff ceiling (5 min). | | `retry.modelFallback` | boolean | `true` | Fall back to another model when one is unavailable. | -| `retry.fallbackChains` | record | `{}` | Per-model fallback chains. | -| `retry.fallbackRevertPolicy` | enum | `cooldown-expiry` | `cooldown-expiry`, `never`. | +| `retry.fallbackChains` | record | `{}` | Maps roles, model selectors, or `provider/*` wildcards to ordered fallback selectors. Keys containing `/` are model-oriented and win over roles: `provider/model-id` matches that exact model, `provider/*` matches every model of the provider. A `provider/*` *entry* keeps the failing model's id and swaps the provider. The `default` chain covers every assigned role without its own chain. Unknown models/providers or malformed chains are reported as config warnings at startup. | +| `retry.fallbackRevertPolicy` | enum | `cooldown-expiry` | `cooldown-expiry` returns to the primary model once its suppression window ends; `never` stays on the fallback until switched manually. | + +When the active model keeps failing (429s, quota walls, provider outages) and `retry.modelFallback` is on, the session picks the chain that owns the failing model, by specificity: an exact `provider/model-id` key, then a `provider/*` wildcard, then the current role's chain, then `default`. It skips models whose selectors are still cooling down and switches for the rest of the turn. Subagents get their own per-spawn chains when their agent definition lists multiple model patterns — the first resolvable pattern is primary and the rest become its fallbacks; there is no `agent:` key in `fallbackChains`. ### Tools and approvals diff --git a/packages/ai/test/auth-storage-codex-selection.test.ts b/packages/ai/test/auth-storage-codex-selection.test.ts index 3a0cf25a9..316de7262 100644 --- a/packages/ai/test/auth-storage-codex-selection.test.ts +++ b/packages/ai/test/auth-storage-codex-selection.test.ts @@ -1606,6 +1606,44 @@ describe("AuthStorage codex oauth ranking", () => { expect(apiKey).toBe("api-acct-paid"); }); + test("attempts every exhausted account for a paid-gated model until one passes the plan gate", async () => { + if (!authStorage) throw new Error("test setup failed"); + + // Production shape of the same regression: EVERY seat is usage-blocked + // (the free seat resets soonest, so it leads the blocked ordering) and + // only a later-resetting paid seat can serve gpt-5.6-sol. The blocked + // pass must keep iterating past the plan-ineligible free seat instead + // of giving up after the first blocked candidate. + await authStorage.set("openai-codex", [ + { type: "oauth", ...createCredential("acct-free", "free@example.com") }, + { type: "oauth", ...createCredential("acct-paid", "paid@example.com") }, + ]); + + usageByAccount.set( + "acct-free", + createCodexUsageReport({ + accountId: "acct-free", + primary: { usedFraction: 1, resetInMs: 5 * 60 * 1000 }, + secondary: { usedFraction: 1, resetInMs: 5 * 60 * 1000 }, + metadata: { planType: "free", email: "free@example.com", limitReached: true }, + }), + ); + usageByAccount.set( + "acct-paid", + createCodexUsageReport({ + accountId: "acct-paid", + primary: { usedFraction: 1, resetInMs: 2 * HOUR_MS }, + secondary: { usedFraction: 1, resetInMs: 6 * 24 * 60 * 60 * 1000 }, + metadata: { planType: "plus", email: "paid@example.com", limitReached: true }, + }), + ); + + const apiKey = await authStorage.getApiKey("openai-codex", "session-sol-all-exhausted", { + modelId: "gpt-5.6-sol", + }); + expect(apiKey).toBe("api-acct-paid"); + }); + test("prefers Pro accounts for codex spark models over Plus accounts", async () => { if (!authStorage) throw new Error("test setup failed"); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 1bd6d9883..3f8816a97 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -5,16 +5,21 @@ ### Added - Added `invalidate` action to the usage command to clear cached usage reports +- Added model-oriented keys and wildcard entries to `retry.fallbackChains`: a `provider/model-id` key attaches a fallback chain to that exact model, a `provider/*` key covers every current or future model of a provider, and a `provider/*` chain entry keeps the failing model's id while swapping the provider (`google-antigravity/x` → `google/x`) — so fallbacks survive role and model reassignments without config edits. Keys resolve by specificity: exact model, then provider wildcard, then role, then `default`. +- Added fallback-chain editing to the /models Roles view: each role's `retry.fallbackChains` entries render as indented rows beneath it, `f` picks a fallback model to append, Enter on an entry replaces it, `x`/backspace removes it, and `[`/`]` (or shift+↑/↓) reorder the chain. +- Added model-keyed fallback management to the /models Roles view: model and `provider/*` chains render as a separate section below the roles (divider + "+ New fallback…" row for creating one by picking the protected model, then keying it by model or provider), with the same replace/remove/reorder editing as role chains; the model strip gains `fallbacks:` and `fallbacks:/*` chips as shortcuts. ### Changed - Retain completed and abandoned tasks in session history for improved context on resume +- Changed the Model Hub `retry-fallback` strip chip to append the model to the default fallback chain instead of prepending it, matching the chain-building order of the Roles view (already-registered models are a no-op). ### Fixed - Fixed compiled release binaries bundling `fastembed` and baking the build-machine `@anush008/tokenizers` path; native runtime dependencies now stay external for every compiled build path so Mnemopi resolves its on-demand install instead. ([#5195](https://github.com/can1357/oh-my-pi/issues/5195)) - Fixed `/btw` side-channel turns on Codex models such as `gpt-5.6-luna` by preserving the session websocket preference instead of forcing SSE, and made Esc dismiss the active `/btw` panel before interrupting loop/maintenance work. ([#5213](https://github.com/can1357/oh-my-pi/issues/5213)) - Fixed the Model Hub role-assignment strip hiding the selected chip once the row overflowed; the strip now scrolls horizontally, truncating passed chips behind a leading ellipsis so the selection (plus one chip of lookahead) stays visible. +- Fixed mouse hover and clicks in the /models Roles view landing one row above the pointer (the row mapping subtracted the status row twice). ## [16.4.5] - 2026-07-11 diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index c314ef56a..e07b14045 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -1375,7 +1375,7 @@ export const SETTINGS_SCHEMA = { group: "Retry & Fallback", label: "Retry Fallback Chains", description: - 'JSON object mapping model roles to ordered fallback model selectors, e.g. {"default":["openai/gpt-4o-mini"]}.', + 'JSON object mapping model roles, model selectors ("provider/model-id"), or provider wildcards ("provider/*") to ordered fallback selectors, e.g. {"default":["openai/gpt-4o-mini"],"google-antigravity/*":["google/*","google-vertex/*"]}. Model-oriented keys apply whenever that model/provider is active, regardless of role; a "provider/*" entry keeps the failing model\'s id and swaps the provider.', }, }, "retry.fallbackRevertPolicy": { diff --git a/packages/coding-agent/src/modes/components/model-hub.ts b/packages/coding-agent/src/modes/components/model-hub.ts index ef28c2bb7..82fa5cb6b 100644 --- a/packages/coding-agent/src/modes/components/model-hub.ts +++ b/packages/coding-agent/src/modes/components/model-hub.ts @@ -52,7 +52,29 @@ import { renderSegmentTrack } from "./segment-track"; /** `roles` is the full /models hub; `pick` is a one-shot session/embedded picker. */ export type ModelHubMode = "roles" | "pick"; -export type ModelHubAction = "modelRole" | "retryFallback"; +/** + * A row of the Roles view: a role, a model/wildcard chain-key header, one of a + * chain's fallback entries, or the trailing "+ New role…". Fallback rows under + * a chain-key header carry the key in `role` — `retry.fallbackChains` treats + * roles, `provider/model-id`, and `provider/*` keys uniformly. + */ +type RolesRow = + | { kind: "role"; role: string } + | { kind: "chainKey"; role: string } + | { kind: "fallback"; role: string; chainIndex: number; selector: string } + | { kind: "separator" } + | { kind: "newFallback" } + | { kind: "newRole" }; + +/** + * What the model browser is currently picking for: a role's model, a slot in + * a fallback chain (`role` may be a role name, model selector, or `provider/*` + * key), or the primary model a brand-new fallback chain protects. + */ +type AssignTarget = + | { kind: "role"; role: string } + | { kind: "fallback"; role: string; index: number | null } + | { kind: "fallbackKey" }; /** A `--models` scope entry (mirrors the session's scoped model list). */ export interface ScopedModelItem { @@ -61,16 +83,12 @@ export interface ScopedModelItem { } export interface ModelHubCallbacks { - /** Persist a role assignment (or a retry-fallback registration). */ - onAssign: ( - model: Model, - role: string, - thinkingLevel: ConfiguredThinkingLevel | undefined, - selector: string, - action: ModelHubAction, - ) => void; + /** Persist a role assignment. */ + onAssign: (model: Model, role: string, thinkingLevel: ConfiguredThinkingLevel | undefined, selector: string) => void; /** Clear a configured role back to auto-selection. */ onUnassign: (role: string) => void; + /** Persist a `retry.fallbackChains` entry — keyed by a role, `provider/model-id`, or `provider/*`; an empty chain clears the key. */ + onFallbackChainChange?: (role: string, chain: string[]) => void; /** Pick-mode activation: session-only switch or embedded pick. */ onPick?: (model: Model, selector: string) => void; /** Locked provider activation: forward to the /login flow. */ @@ -108,7 +126,7 @@ interface StripChip { /** Pre-styled label body (without selection decoration). */ styled: string; role?: string; - action: "assign" | "unassign" | "fallback" | "thinking"; + action: "assign" | "unassign" | "fallback" | "fallbackModel" | "fallbackProvider" | "thinking"; thinkingLevel?: ConfiguredThinkingLevel; } @@ -192,11 +210,11 @@ export class ModelHubComponent implements Component { */ #focus: "scope" | "list" = "scope"; - #roleIds: string[] = []; + #rolesRows: RolesRow[] = []; #roleIndex = 0; #roleHover: number | null = null; - #assigningRole: string | null = null; + #assigning: AssignTarget | null = null; #strip: StripState | null = null; /** Per-provider fuzzy match counts while a query is active; null when not searching. */ #searchCounts: Map | null = null; @@ -215,7 +233,7 @@ export class ModelHubComponent implements Component { #footerRow = 0; #chipRanges: ChipRange[] = []; #lockedLoginLine: number | null = null; - #rolesRowStart = 2; + #rolesRowStart = 1; constructor( tui: TUI, @@ -373,6 +391,7 @@ export class ModelHubComponent implements Component { } this.#reloadRoles(availableModels); + this.#buildRolesRows(); const mruOrder = this.#settings.getStorage()?.getModelUsageOrder() ?? []; this.#availableItems = buildBrowserItems(availableModels); @@ -545,7 +564,6 @@ export class ModelHubComponent implements Component { break; } case "roles": - this.#roleIds = this.#visibleRoleIds(); this.#roleIndex = Math.min(this.#roleIndex, Math.max(0, this.#rolesRowCount - 1)); break; default: @@ -555,6 +573,58 @@ export class ModelHubComponent implements Component { } } + /** + * The configured `retry.fallbackChains` record with malformed keys/entries + * dropped: non-array chains and non-string selectors never reach the rows + * or chain editors, so an edit through the hub replaces them wholesale. + */ + #fallbackChains(): Record { + try { + const chains = this.#settings.get("retry.fallbackChains"); + if (!chains || typeof chains !== "object" || Array.isArray(chains)) return {}; + const sanitized: Record = {}; + for (const key in chains) { + const chain = (chains as Record)[key]; + if (!Array.isArray(chain)) continue; + sanitized[key] = chain.filter((entry): entry is string => typeof entry === "string"); + } + return sanitized; + } catch { + return {}; + } + } + + /** + * Rebuild the Roles view rows: each visible role followed by its + * fallback-chain entries, then model-oriented chains (`provider/model-id` + * and `provider/*` keys) as headed groups. + */ + #buildRolesRows(): void { + const rows: RolesRow[] = []; + const chains = this.#fallbackChains(); + for (const role of this.#visibleRoleIds()) { + rows.push({ kind: "role", role }); + const chain = chains[role] ?? []; + for (let i = 0; i < chain.length; i++) { + rows.push({ kind: "fallback", role, chainIndex: i, selector: chain[i] }); + } + } + rows.push({ kind: "newRole" }); + rows.push({ kind: "separator" }); + const modelKeys = Object.keys(chains) + .filter(key => key.includes("/")) + .sort(); + for (const key of modelKeys) { + const chain = chains[key] ?? []; + rows.push({ kind: "chainKey", role: key }); + for (let i = 0; i < chain.length; i++) { + rows.push({ kind: "fallback", role: key, chainIndex: i, selector: chain[i] }); + } + } + rows.push({ kind: "newFallback" }); + this.#rolesRows = rows; + } + /** Refresh roles + dependent state after a settings mutation (assign/unassign). */ #refreshAfterMutation(): void { this.#syncFromRegistryState(); @@ -588,7 +658,7 @@ export class ModelHubComponent implements Component { this.#composeEntries(); const entry = this.#activeEntry(); if ( - this.#assigningRole === null && + this.#assigning === null && entry.kind === "provider" && (entry.locked || (counts.get(entry.providerId ?? "") ?? 0) === 0) ) { @@ -739,10 +809,16 @@ export class ModelHubComponent implements Component { this.#callbacks.onPick?.(item.model, item.selector); return; } - if (this.#assigningRole) { - const role = this.#assigningRole; - this.#assigningRole = null; - this.#assignRole(item, role, true); + if (this.#assigning) { + const target = this.#assigning; + this.#assigning = null; + if (target.kind === "role") { + this.#assignRole(item, target.role, true); + } else if (target.kind === "fallbackKey") { + this.#openFallbackKeyStrip(item); + } else { + this.#commitFallback(item, target); + } return; } this.#openRoleStrip(item); @@ -756,7 +832,7 @@ export class ModelHubComponent implements Component { const supported = this.#thinkingOptionsFor(item.model); level = supported.includes(current.thinkingLevel) ? current.thinkingLevel : ThinkingLevel.Inherit; } - this.#callbacks.onAssign(item.model, role, level, item.selector, "modelRole"); + this.#callbacks.onAssign(item.model, role, level, item.selector); this.#refreshAfterMutation(); this.#openThinkingStrip(item, role, returnToRoles); } @@ -793,6 +869,16 @@ export class ModelHubComponent implements Component { action: assignedHere ? "unassign" : "assign", }); } + chips.push({ + label: `fallbacks:${item.model.id}`, + styled: theme.fg("muted", `fallbacks:${item.model.id}`), + action: "fallbackModel", + }); + chips.push({ + label: `fallbacks:${item.model.provider}/*`, + styled: theme.fg("muted", `fallbacks:${item.model.provider}/*`), + action: "fallbackProvider", + }); chips.push({ label: "fallback", styled: theme.fg("muted", "retry-fallback"), action: "fallback" }); this.#strip = { kind: "role", item, chips, index: 0, returnToRoles: false }; } @@ -851,18 +937,20 @@ export class ModelHubComponent implements Component { this.#closeStrip(); return; case "fallback": - this.#callbacks.onAssign(strip.item.model, "default", undefined, strip.item.selector, "retryFallback"); + this.#appendFallback(strip.item, "default"); this.#closeStrip(); return; + case "fallbackModel": + this.#closeStrip(); + this.#startAssignFallback(strip.item.selector, null); + return; + case "fallbackProvider": + this.#closeStrip(); + this.#startAssignFallback(`${strip.item.model.provider}/*`, null); + return; case "thinking": if (strip.role && chip.thinkingLevel !== undefined) { - this.#callbacks.onAssign( - strip.item.model, - strip.role, - chip.thinkingLevel, - strip.item.selector, - "modelRole", - ); + this.#callbacks.onAssign(strip.item.model, strip.role, chip.thinkingLevel, strip.item.selector); this.#refreshAfterMutation(); } this.#closeStrip(); @@ -872,7 +960,7 @@ export class ModelHubComponent implements Component { /** Switch the body into assign mode for `role`: full catalog, cleared query, current model preselected. */ #startAssign(role: string): void { - this.#assigningRole = role; + this.#assigning = { kind: "role", role }; this.#focus = "scope"; this.#browser.setShowProvider(true); this.#browser.setItems([...this.#availableItems]); @@ -883,8 +971,104 @@ export class ModelHubComponent implements Component { } } + /** Browse the catalog to fill a fallback-chain slot: `index` replaces an entry, `null` appends. */ + #startAssignFallback(role: string, index: number | null): void { + this.#assigning = { kind: "fallback", role, index }; + this.#focus = "scope"; + this.#browser.setShowProvider(true); + this.#browser.setItems([...this.#availableItems]); + this.#browser.setQuery(""); + if (index !== null) { + const selector = this.#fallbackChains()[role]?.[index]; + if (selector) this.#browser.selectSelector(selector); + } + } + + /** Browse the catalog for the primary model a brand-new fallback chain protects. */ + #startAssignFallbackKey(): void { + this.#assigning = { kind: "fallbackKey" }; + this.#focus = "scope"; + this.#browser.setShowProvider(true); + this.#browser.setItems([...this.#availableItems]); + this.#browser.setQuery(""); + } + + /** Second step of "+ New fallback…": key the chain by the picked model or its whole provider. */ + #openFallbackKeyStrip(item: ModelBrowserItem): void { + const chips: StripChip[] = [ + { + label: `for ${item.selector}`, + styled: theme.fg("muted", `for ${item.selector}`), + action: "fallbackModel", + }, + { + label: `for ${item.model.provider}/*`, + styled: theme.fg("muted", `for ${item.model.provider}/*`), + action: "fallbackProvider", + }, + ]; + this.#strip = { kind: "role", item, chips, index: 0, returnToRoles: false }; + } + + /** Write the picked model into the target chain slot, dedupe, and land back on its Roles row. */ + #commitFallback(item: ModelBrowserItem, target: { role: string; index: number | null }): void { + const chain = [...(this.#fallbackChains()[target.role] ?? [])]; + const selector = item.selector; + if (target.index !== null && target.index < chain.length) { + chain[target.index] = selector; + for (let i = chain.length - 1; i >= 0; i--) { + if (i !== target.index && chain[i] === selector) chain.splice(i, 1); + } + } else if (!chain.includes(selector)) { + chain.push(selector); + } + this.#setFallbackChain(target.role, chain); + this.#browser.setQuery(""); + if (this.#mode === "roles") { + this.#setActiveEntry("roles"); + this.#focus = "list"; + const rowIndex = this.#rolesRows.findIndex( + row => row.kind === "fallback" && row.role === target.role && row.selector === selector, + ); + if (rowIndex >= 0) this.#roleIndex = rowIndex; + } + } + + /** Persist `role`'s chain through the host callback and rebuild dependent state. */ + #setFallbackChain(role: string, chain: string[]): void { + this.#callbacks.onFallbackChainChange?.(role, chain); + this.#refreshAfterMutation(); + } + + /** Append `item` to `role`'s fallback chain (no-op when already present). */ + #appendFallback(item: ModelBrowserItem, role: string): void { + const chain = [...(this.#fallbackChains()[role] ?? [])]; + if (chain.includes(item.selector)) return; + chain.push(item.selector); + this.#setFallbackChain(role, chain); + } + + /** Remove one chain entry; the cursor stays on the nearest surviving row. */ + #removeFallback(row: { role: string; chainIndex: number }): void { + const chain = [...(this.#fallbackChains()[row.role] ?? [])]; + if (row.chainIndex >= chain.length) return; + chain.splice(row.chainIndex, 1); + this.#setFallbackChain(row.role, chain); + this.#roleIndex = Math.min(this.#roleIndex, Math.max(0, this.#rolesRows.length - 1)); + } + + /** Move a chain entry one slot earlier/later; the cursor follows the moved entry. */ + #moveFallback(row: { role: string; chainIndex: number }, delta: -1 | 1): void { + const chain = [...(this.#fallbackChains()[row.role] ?? [])]; + const target = row.chainIndex + delta; + if (row.chainIndex >= chain.length || target < 0 || target >= chain.length) return; + [chain[row.chainIndex], chain[target]] = [chain[target], chain[row.chainIndex]]; + this.#setFallbackChain(row.role, chain); + this.#roleIndex += delta; + } + #cancelAssign(): void { - this.#assigningRole = null; + this.#assigning = null; this.#browser.setQuery(""); if (this.#mode === "roles") { this.#setActiveEntry("roles"); @@ -961,7 +1145,7 @@ export class ModelHubComponent implements Component { } if (matchesSelectCancel(data)) { - if (this.#assigningRole !== null) { + if (this.#assigning !== null) { this.#cancelAssign(); return; } @@ -975,8 +1159,8 @@ export class ModelHubComponent implements Component { } const entry = this.#activeEntry(); - const rolesView = entry.kind === "roles" && this.#assigningRole === null; - const lockedView = entry.kind === "provider" && entry.locked && this.#assigningRole === null; + const rolesView = entry.kind === "roles" && this.#assigning === null; + const lockedView = entry.kind === "provider" && entry.locked && this.#assigning === null; if (matchesKey(data, "tab") || matchesKey(data, "shift+tab")) { this.#focus = this.#focus === "scope" ? "list" : "scope"; @@ -1030,7 +1214,7 @@ export class ModelHubComponent implements Component { } #isBrowserView(entry: SidebarEntry): boolean { - if (this.#assigningRole !== null) return true; + if (this.#assigning !== null) return true; return entry.kind === "recent" || entry.kind === "all" || (entry.kind === "provider" && !entry.locked); } @@ -1074,16 +1258,50 @@ export class ModelHubComponent implements Component { if (entry && !this.#isHopSkipped(entry)) { // Scope changes keep an active assignment (scoping helps find the // model); landing on the Roles view cancels it. - if (entry.kind === "roles") this.#assigningRole = null; + if (entry.kind === "roles") this.#assigning = null; this.#setActiveEntry(entry.id); return; } } } - /** Row count of the roles view: every visible role plus the trailing "+ New role…" row. */ + /** Row count of the roles view (roles, their fallback entries, and the trailing "+ New role…" row). */ get #rolesRowCount(): number { - return this.#roleIds.length + 1; + return this.#rolesRows.length; + } + + /** Enter/click activation for a Roles-view row. */ + #activateRolesRow(row: RolesRow): void { + switch (row.kind) { + case "role": + this.#startAssign(row.role); + return; + case "chainKey": + this.#startAssignFallback(row.role, null); + return; + case "fallback": + this.#startAssignFallback(row.role, row.chainIndex); + return; + case "newFallback": + this.#startAssignFallbackKey(); + return; + case "newRole": + this.#openRoleNameStrip(); + return; + case "separator": + return; + } + } + + /** Step the roles cursor by one row, wrapping and skipping separator rows. */ + #stepRoleIndex(from: number, delta: -1 | 1): number { + const count = Math.max(1, this.#rolesRows.length); + let index = from; + for (let i = 0; i < count; i++) { + index = (index + delta + count) % count; + if (this.#rolesRows[index]?.kind !== "separator") break; + } + return index; } #handleRolesViewInput(data: string): void { @@ -1095,41 +1313,50 @@ export class ModelHubComponent implements Component { } return; } - const rowCount = Math.max(1, this.#rolesRowCount); if (matchesSelectUp(data)) { - this.#roleIndex = (this.#roleIndex - 1 + rowCount) % rowCount; + this.#roleIndex = this.#stepRoleIndex(this.#roleIndex, -1); return; } if (matchesSelectDown(data)) { - this.#roleIndex = (this.#roleIndex + 1) % rowCount; + this.#roleIndex = this.#stepRoleIndex(this.#roleIndex, 1); return; } - const role = this.#roleIds[this.#roleIndex]; + const row = this.#rolesRows[this.#roleIndex]; + const role = row?.kind === "role" ? row.role : undefined; if (matchesKey(data, "enter") || matchesKey(data, "return") || data === "\n") { - if (role) { - this.#startAssign(role); - } else { - // The virtual "+ New role…" row. - this.#openRoleNameStrip(); - } + if (row) this.#activateRolesRow(row); return; } if (matchesKey(data, "backspace") || matchesKey(data, "delete")) { if (role) this.#unassignRole(role); + else if (row?.kind === "fallback") this.#removeFallback(row); + else if (row?.kind === "chainKey") this.#setFallbackChain(row.role, []); return; } - // Cycle reordering: [ / shift+↑ moves the role earlier, ] / shift+↓ later. + // Reordering: [ / shift+↑ moves the row earlier, ] / shift+↓ later — + // cycle order on a role row, chain order on a fallback row. if (matchesKey(data, "shift+up")) { if (role) this.#moveCycleMembership(role, -1); + else if (row?.kind === "fallback") this.#moveFallback(row, -1); return; } if (matchesKey(data, "shift+down")) { if (role) this.#moveCycleMembership(role, 1); + else if (row?.kind === "fallback") this.#moveFallback(row, 1); return; } const printable = extractPrintableText(data); if (printable === "x") { if (role) this.#unassignRole(role); + else if (row?.kind === "fallback") this.#removeFallback(row); + else if (row?.kind === "chainKey") this.#setFallbackChain(row.role, []); + return; + } + if (printable === "f") { + if (row?.kind === "newFallback") this.#startAssignFallbackKey(); + else if (row && row.kind !== "newRole" && row.kind !== "separator") { + this.#startAssignFallback(row.role, null); + } return; } if (printable === "c") { @@ -1138,10 +1365,12 @@ export class ModelHubComponent implements Component { } if (printable === "[") { if (role) this.#moveCycleMembership(role, -1); + else if (row?.kind === "fallback") this.#moveFallback(row, -1); return; } if (printable === "]") { if (role) this.#moveCycleMembership(role, 1); + else if (row?.kind === "fallback") this.#moveFallback(row, 1); return; } if (printable === "n") { @@ -1204,9 +1433,8 @@ export class ModelHubComponent implements Component { if (overSidebar) { this.#moveSidebar(event.wheel); } else if (overBody) { - if (entry.kind === "roles" && this.#assigningRole === null) { - const count = Math.max(1, this.#rolesRowCount); - this.#roleIndex = (this.#roleIndex + event.wheel + count) % count; + if (entry.kind === "roles" && this.#assigning === null) { + this.#roleIndex = this.#stepRoleIndex(this.#roleIndex, event.wheel > 0 ? 1 : -1); } else if (this.#isBrowserView(entry)) { this.#browser.routeMouse(event, bodyLine); } @@ -1216,7 +1444,7 @@ export class ModelHubComponent implements Component { if (event.motion) { this.#sidebarHover = overSidebar ? this.#sidebarEntryIndexAt(contentLine) : null; - if (overBody && entry.kind === "roles" && this.#assigningRole === null) { + if (overBody && entry.kind === "roles" && this.#assigning === null) { const roleLine = bodyLine - this.#rolesRowStart; this.#roleHover = roleLine >= 0 && roleLine < this.#rolesRowCount ? roleLine : null; } else { @@ -1235,7 +1463,7 @@ export class ModelHubComponent implements Component { const clicked = index !== null ? this.#entries[index] : undefined; if (clicked && clicked.kind !== "separator") { const already = clicked.id === this.#activeEntryId; - if (clicked.kind === "roles") this.#assigningRole = null; + if (clicked.kind === "roles") this.#assigning = null; this.#setActiveEntry(clicked.id); // A click on Roles is a deliberate dive into the rows. if (clicked.kind === "roles") this.#focus = "list"; @@ -1247,22 +1475,20 @@ export class ModelHubComponent implements Component { } if (overBody) { - if (entry.kind === "roles" && this.#assigningRole === null) { + if (entry.kind === "roles" && this.#assigning === null) { this.#focus = "list"; const roleLine = bodyLine - this.#rolesRowStart; if (roleLine >= 0 && roleLine < this.#rolesRowCount) { - if (roleLine === this.#roleIndex) { - const role = this.#roleIds[roleLine]; - if (role) { - this.#startAssign(role); + const rowDef = this.#rolesRows[roleLine]; + if (rowDef && rowDef.kind !== "separator") { + if (roleLine === this.#roleIndex) { + this.#activateRolesRow(rowDef); } else { - this.#openRoleNameStrip(); + this.#roleIndex = roleLine; } - } else { - this.#roleIndex = roleLine; } } - } else if (entry.kind === "provider" && entry.locked && this.#assigningRole === null) { + } else if (entry.kind === "provider" && entry.locked && this.#assigning === null) { if (this.#lockedLoginLine !== null && bodyLine === this.#lockedLoginLine) { this.#requestLogin(entry); } @@ -1374,9 +1600,22 @@ export class ModelHubComponent implements Component { } #statusRow(width: number): string { - if (this.#assigningRole !== null) { - const info = getRoleInfo(this.#assigningRole, this.#settings); - const label = info.tag ?? info.name ?? this.#assigningRole; + if (this.#assigning !== null) { + if (this.#assigning.kind === "fallbackKey") { + return truncateToWidth( + theme.fg("accent", " New fallback chain — Enter picks the model it protects, Esc cancels"), + width, + ); + } + const info = getRoleInfo(this.#assigning.role, this.#settings); + const label = info.tag ?? info.name ?? this.#assigning.role; + if (this.#assigning.kind === "fallback") { + const verb = this.#assigning.index === null ? "Adding fallback for" : "Replacing fallback of"; + return truncateToWidth( + theme.fg("accent", ` ${verb} ${theme.bold(label)} — Enter picks the fallback model, Esc cancels`), + width, + ); + } return truncateToWidth( theme.fg("accent", ` Assigning ${theme.bold(label)} — Enter assigns, Esc cancels`), width, @@ -1390,7 +1629,7 @@ export class ModelHubComponent implements Component { text = this.#mode === "pick" ? this.#pickerHint : `Recently used models${scopedSuffix}`; break; case "roles": - text = "Model roles — assignments fall back to auto-selection when cleared"; + text = "Model roles — f adds a retry fallback, cleared roles fall back to auto-selection"; break; case "provider": if (entry.locked) { @@ -1415,22 +1654,71 @@ export class ModelHubComponent implements Component { #renderRolesView(width: number, rows: number): string[] { const lines: string[] = []; lines.push(""); - this.#rolesRowStart = lines.length + 1; // +1 for the status row offset handled by caller + // First row's offset in bodyLine coordinates: the mouse router's + // `bodyLine` has already dropped the status row, so this is just the + // leading blank line — no extra status-row offset here. + this.#rolesRowStart = lines.length; let tagWidth = 0; - for (const role of this.#roleIds) { - const info = getRoleInfo(role, this.#settings); - tagWidth = Math.max(tagWidth, visibleWidth(info.tag ?? info.name ?? role)); + for (const rowDef of this.#rolesRows) { + if (rowDef.kind !== "role") continue; + const info = getRoleInfo(rowDef.role, this.#settings); + tagWidth = Math.max(tagWidth, visibleWidth(info.tag ?? info.name ?? rowDef.role)); } const cycleOrder = this.#cycleOrder(); - for (let i = 0; i < this.#roleIds.length && lines.length < rows - 3; i++) { - const role = this.#roleIds[i]; - const info = getRoleInfo(role, this.#settings); - const assignment = this.#roles[role]; + for (let i = 0; i < this.#rolesRows.length && lines.length < rows - 2; i++) { + const rowDef = this.#rolesRows[i]; + if (!rowDef) continue; const selected = i === this.#roleIndex; const hovered = i === this.#roleHover; const cursor = selected ? theme.fg("accent", theme.nav.cursor) : " "; + + if (rowDef.kind === "separator") { + lines.push(` ${theme.fg("border", "─".repeat(Math.max(1, width - 6)))}`); + continue; + } + + if (rowDef.kind === "newRole" || rowDef.kind === "newFallback") { + const label = rowDef.kind === "newRole" ? "+ New role…" : "+ New fallback…"; + let line = ` ${cursor} ${theme.fg(selected ? "accent" : "dim", label)}`; + line = truncateToWidth(line, width); + if (hovered && !selected) { + line = theme.bg("selectedBg", line); + } + lines.push(line); + continue; + } + + if (rowDef.kind === "chainKey") { + const key = rowDef.role; + const slash = key.lastIndexOf("/"); + const tail = key.slice(slash + 1); + const keyStyled = theme.fg("dim", key.slice(0, slash + 1)) + (selected ? theme.fg("accent", tail) : tail); + let line = ` ${cursor} ${theme.fg("dim", theme.status.shadowed)} ${keyStyled}`; + line = truncateToWidth(line, width); + if (hovered && !selected) { + line = theme.bg("selectedBg", line); + } + lines.push(line); + continue; + } + + if (rowDef.kind === "fallback") { + const branch = theme.fg("dim", `${"".padEnd(tagWidth + 3)}↳`); + const selector = selected ? theme.fg("accent", rowDef.selector) : theme.fg("muted", rowDef.selector); + let line = ` ${cursor} ${branch} ${selector}`; + line = truncateToWidth(line, width); + if (hovered && !selected) { + line = theme.bg("selectedBg", line); + } + lines.push(line); + continue; + } + + const role = rowDef.role; + const info = getRoleInfo(role, this.#settings); + const assignment = this.#roles[role]; const tag = (info.tag ?? info.name ?? role).padEnd(tagWidth); let dot: string; @@ -1475,27 +1763,16 @@ export class ModelHubComponent implements Component { lines.push(line); } - // Trailing virtual row: create a custom role. - if (lines.length < rows - 2) { - const newRoleIndex = this.#roleIds.length; - const selected = this.#roleIndex === newRoleIndex; - const hovered = this.#roleHover === newRoleIndex; - const cursor = selected ? theme.fg("accent", theme.nav.cursor) : " "; - let line = ` ${cursor} ${theme.fg(selected ? "accent" : "dim", "+ New role…")}`; - line = truncateToWidth(line, width); - if (hovered && !selected) { - line = theme.bg("selectedBg", line); - } - lines.push(line); - } - // Live preview of the quick-switch cycle, rendered with the exact // segment track the ctrl+p status uses; the selected role's chip fills. while (lines.length < rows - 1) lines.push(""); if (rows >= 2) { const cycleKey = getKeybindings().getKeys("app.model.cycleForward")[0] ?? "ctrl+p"; if (cycleOrder.length > 0) { - const activeIndex = cycleOrder.indexOf(this.#roleIds[this.#roleIndex] ?? ""); + const selectedRow = this.#rolesRows[this.#roleIndex]; + const selectedRole = + selectedRow && (selectedRow.kind === "role" || selectedRow.kind === "fallback") ? selectedRow.role : ""; + const activeIndex = cycleOrder.indexOf(selectedRole); const track = renderSegmentTrack( cycleOrder.map(role => ({ label: role })), activeIndex, @@ -1557,14 +1834,32 @@ export class ModelHubComponent implements Component { ? "←/→ choose · Enter assign/clear · Esc cancel" : "←/→ thinking level · Enter apply · Esc keep"; } - if (this.#assigningRole !== null) { - return "Enter assign · ↑/↓ providers · type to search · Esc cancel"; + if (this.#assigning !== null) { + switch (this.#assigning.kind) { + case "fallback": + return "Enter pick fallback · ↑/↓ providers · type to search · Esc cancel"; + case "fallbackKey": + return "Enter pick the protected model · ↑/↓ providers · type to search · Esc cancel"; + default: + return "Enter assign · ↑/↓ providers · type to search · Esc cancel"; + } } const entry = this.#activeEntry(); if (entry.kind === "roles") { - return this.#focus === "list" - ? "↑/↓ roles · Enter pick · x clear · t thinking · c cycle · [/] reorder · n new · ← providers" - : "↑/↓ providers · → roles · Esc close"; + if (this.#focus !== "list") { + return "↑/↓ providers · → roles · Esc close"; + } + const row = this.#rolesRows[this.#roleIndex]; + if (row?.kind === "fallback") { + return "↑/↓ rows · Enter replace · f add another · x remove · [/] reorder · ← providers"; + } + if (row?.kind === "chainKey") { + return "↑/↓ rows · Enter/f add fallback · x clear chain · ← providers"; + } + if (row?.kind === "newFallback") { + return "↑/↓ rows · Enter new model/provider fallback chain · ← providers"; + } + return "↑/↓ rows · Enter pick · f fallback · x clear · t thinking · c cycle · [/] reorder · n new"; } if (entry.kind === "provider" && entry.locked) { return entry.oauth ? "Enter log in · ↑/↓ providers · Esc close" : "↑/↓ providers · Esc close"; @@ -1656,9 +1951,9 @@ export class ModelHubComponent implements Component { const entry = this.#activeEntry(); const bodyLines: string[] = [this.#statusRow(bodyWidth)]; - if (entry.kind === "roles" && this.#assigningRole === null) { + if (entry.kind === "roles" && this.#assigning === null) { bodyLines.push(...this.#renderRolesView(bodyWidth, contentRows - 1)); - } else if (entry.kind === "provider" && entry.locked && this.#assigningRole === null) { + } else if (entry.kind === "provider" && entry.locked && this.#assigning === null) { bodyLines.push(...this.#renderLockedView(entry, bodyWidth, contentRows - 1)); } else { this.#browser.setMaxVisible(contentRows - 1 - 5); diff --git a/packages/coding-agent/src/modes/controllers/selector-controller.ts b/packages/coding-agent/src/modes/controllers/selector-controller.ts index 82a2ce425..e101bbc5f 100644 --- a/packages/coding-agent/src/modes/controllers/selector-controller.ts +++ b/packages/coding-agent/src/modes/controllers/selector-controller.ts @@ -616,7 +616,7 @@ export class SelectorController { this.ctx.session.modelRegistry, this.ctx.session.scopedModels, { - onAssign: async (model, role, thinkingLevel, selector, action) => { + onAssign: async (model, role, thinkingLevel, selector) => { // `auto` is session-global: never baked into a per-role model value // (it can't round-trip through `model:`). Apply it to the session // separately and persist via `defaultThinkingLevel`. @@ -624,18 +624,6 @@ export class SelectorController { const concreteThinking = isAuto || thinkingLevel === undefined ? undefined : thinkingLevel; const selectorValue = selector ?? `${model.provider}/${model.id}`; try { - if (action === "retryFallback") { - const fallbackSelector = formatModelSelectorValue(selectorValue, concreteThinking); - const fallbackChains = this.ctx.settings.get("retry.fallbackChains"); - const chain = Array.isArray(fallbackChains[role]) ? fallbackChains[role] : []; - this.ctx.settings.set("retry.fallbackChains", { - ...fallbackChains, - [role]: [fallbackSelector, ...chain.filter(existing => existing !== fallbackSelector)], - }); - const roleInfo = getRoleInfo(role, settings); - this.ctx.showStatus(`${roleInfo?.name ?? role} fallback model: ${fallbackSelector}`); - return; - } if (role === "default") { const { switched } = await this.ctx.session.setModel(model, role, { selector, @@ -679,6 +667,25 @@ export class SelectorController { this.ctx.showError(error instanceof Error ? error.message : String(error)); } }, + onFallbackChainChange: (role, chain) => { + try { + const chains = { ...this.ctx.settings.get("retry.fallbackChains") }; + if (chain.length === 0) { + delete chains[role]; + } else { + chains[role] = chain; + } + this.ctx.settings.set("retry.fallbackChains", chains); + const roleInfo = getRoleInfo(role, settings); + this.ctx.showStatus( + chain.length > 0 + ? `${roleInfo?.name ?? role} fallbacks: ${chain.join(" → ")}` + : `${roleInfo?.name ?? role} fallbacks cleared`, + ); + } catch (error) { + this.ctx.showError(error instanceof Error ? error.message : String(error)); + } + }, onPick: async (model, selector) => { try { // Session-only: update agent state but don't persist the model to settings. diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index bd704627a..cb26281e3 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -1060,6 +1060,7 @@ export interface FreshSessionResult { /** Standard thinking levels */ +/** `retry.fallbackChains` config: chain key (role name or model selector) → ordered fallback selectors. */ type RetryFallbackChains = Record; type RetryFallbackRevertPolicy = "never" | "cooldown-expiry"; @@ -1072,6 +1073,7 @@ interface RetryFallbackSelector { } interface ActiveRetryFallbackState { + /** Chain key that produced this fallback: a model-role name or a model-selector key. */ role: string; originalSelector: string; originalThinkingLevel: ConfiguredThinkingLevel | undefined; @@ -1099,6 +1101,24 @@ function parseRetryFallbackSelector( }; } +/** + * `retry.fallbackChains` keys are either model-role names (`smol`, `default`) + * or model selectors (`provider/model-id[:thinking]`). Role names never + * contain a slash, so its presence marks a model-keyed chain whose primary is + * the key itself — the chain follows the model across role reassignments. + */ +function isRetryFallbackModelKey(key: string): boolean { + return key.includes("/"); +} + +/** + * A `provider/*` fallback-chain key: matches any active model of that provider, + * so one entry covers every current and future model behind the provider. + */ +function isRetryFallbackWildcardKey(key: string): boolean { + return key.endsWith("/*"); +} + function formatRetryFallbackSelector(model: Model, thinkingLevel: ThinkingLevel | undefined): string { return formatModelSelectorValue(formatModelStringWithRouting(model), thinkingLevel); } @@ -13475,36 +13495,68 @@ export class AgentSession { const configuredChains = this.settings.get("retry.fallbackChains"); if (configuredChains === undefined) return; if (!configuredChains || typeof configuredChains !== "object" || Array.isArray(configuredChains)) { - const msg = "retry.fallbackChains must be a mapping of role names to selector arrays."; + const msg = "retry.fallbackChains must be a mapping of role names or model selectors to selector arrays."; logger.warn(msg); this.configWarnings.push(msg); return; } - for (const [role, chain] of Object.entries(configuredChains)) { + for (const key in configuredChains) { + const chain = (configuredChains as RetryFallbackChains)[key]; + const keyKind = isRetryFallbackModelKey(key) ? "model" : "role"; + if (keyKind === "model") { + if (isRetryFallbackWildcardKey(key)) { + const provider = key.slice(0, -2); + if (!this.#modelRegistry.getAll().some(model => model.provider === provider)) { + const msg = `retry.fallbackChains wildcard key references unknown provider: ${key}`; + logger.warn(msg); + this.configWarnings.push(msg); + } + } else { + const parsedKey = parseRetryFallbackSelector(key, this.#modelRegistry); + if (!parsedKey) { + const msg = `Invalid model selector key in retry.fallbackChains: ${key}`; + logger.warn(msg); + this.configWarnings.push(msg); + } else if (!this.#modelRegistry.find(parsedKey.provider, parsedKey.id)) { + const msg = `retry.fallbackChains key references unknown model: ${key}`; + logger.warn(msg); + this.configWarnings.push(msg); + } + } + } if (!Array.isArray(chain)) { - const msg = `Fallback chain for role '${role}' must be an array of selector strings.`; + const msg = `Fallback chain for ${keyKind} '${key}' must be an array of selector strings.`; logger.warn(msg); this.configWarnings.push(msg); continue; } for (const selectorStr of chain) { if (typeof selectorStr !== "string") { - const msg = `Fallback chain for role '${role}' contains a non-string selector.`; + const msg = `Fallback chain for ${keyKind} '${key}' contains a non-string selector.`; logger.warn(msg); this.configWarnings.push(msg); continue; } + if (isRetryFallbackWildcardKey(selectorStr)) { + const provider = selectorStr.slice(0, -2); + if (!this.#modelRegistry.getAll().some(model => model.provider === provider)) { + const msg = `Fallback chain for ${keyKind} '${key}' references unknown provider: ${selectorStr}`; + logger.warn(msg); + this.configWarnings.push(msg); + } + continue; + } const parsed = parseRetryFallbackSelector(selectorStr, this.#modelRegistry); if (!parsed) { - const msg = `Invalid fallback selector format in role '${role}': ${selectorStr}`; + const msg = `Invalid fallback selector format in ${keyKind} '${key}': ${selectorStr}`; logger.warn(msg); this.configWarnings.push(msg); continue; } const exists = this.#modelRegistry.find(parsed.provider, parsed.id); if (!exists) { - const msg = `Fallback chain for role '${role}' references unknown model: ${selectorStr}`; + const msg = `Fallback chain for ${keyKind} '${key}' references unknown model: ${selectorStr}`; logger.warn(msg); this.configWarnings.push(msg); } @@ -13517,6 +13569,8 @@ export class AgentSession { } #getRetryFallbackPrimarySelector(role: string): RetryFallbackSelector | undefined { + if (isRetryFallbackWildcardKey(role)) return undefined; + if (isRetryFallbackModelKey(role)) return parseRetryFallbackSelector(role, this.#modelRegistry); const configuredSelector = this.settings.getModelRole(role); return configuredSelector ? parseRetryFallbackSelector(configuredSelector, this.#modelRegistry) : undefined; } @@ -13538,6 +13592,13 @@ export class AgentSession { this.#modelRegistry.suppressSelector(currentSelector, Date.now() + cooldownMs); } + /** + * Map the failing model selector to the chain key that owns it, by + * specificity: an exact model-selector key, then a `provider/*` wildcard, + * then a model role whose current assignment matches, then `default`. + * Model-oriented keys win over roles so a chain follows the model across + * role reassignments. + */ #resolveRetryFallbackRole(currentSelector: string): string | undefined { const parsedCurrent = parseRetryFallbackSelector(currentSelector, this.#modelRegistry); if (!parsedCurrent) return undefined; @@ -13551,18 +13612,33 @@ export class AgentSession { ? formatRetryFallbackBaseSelector(parseRetryFallbackSelector(currentPlainSelector) ?? parsedCurrent) : undefined; - for (const role of Object.keys(chains)) { - const primarySelector = this.#getRetryFallbackPrimarySelector(role); - if (primarySelector?.raw === currentSelector) return role; + const exactModelKeys: string[] = []; + const roleKeys: string[] = []; + for (const key in chains) { + if (!isRetryFallbackModelKey(key)) roleKeys.push(key); + else if (!isRetryFallbackWildcardKey(key)) exactModelKeys.push(key); } - for (const role of Object.keys(chains)) { - const primarySelector = this.#getRetryFallbackPrimarySelector(role); - if (!primarySelector) continue; - if (currentPlainSelector && primarySelector.raw === currentPlainSelector) return role; - const primaryBaseSelector = formatRetryFallbackBaseSelector(primarySelector); - if (primaryBaseSelector === currentBaseSelector) return role; - if (currentPlainBaseSelector && primaryBaseSelector === currentPlainBaseSelector) return role; + const matchesCurrent = (primary: RetryFallbackSelector | undefined): boolean => { + if (!primary) return false; + if (primary.raw === currentSelector || (currentPlainSelector && primary.raw === currentPlainSelector)) { + return true; + } + const base = formatRetryFallbackBaseSelector(primary); + return base === currentBaseSelector || (!!currentPlainBaseSelector && base === currentPlainBaseSelector); + }; + + // 1. Exact model-selector keys — most specific. + for (const key of exactModelKeys) { + if (matchesCurrent(this.#getRetryFallbackPrimarySelector(key))) return key; } + // 2. Provider wildcard (`provider/*`) — any active model of this provider. + const wildcardKey = `${parsedCurrent.provider}/*`; + if (Array.isArray(chains[wildcardKey])) return wildcardKey; + // 3. Role keys — matched by the role's currently-assigned model. + for (const key of roleKeys) { + if (matchesCurrent(this.#getRetryFallbackPrimarySelector(key))) return key; + } + // 4. The default chain, when default has no explicit role primary. const defaultChain = chains.default; if ( Array.isArray(defaultChain) && @@ -13574,13 +13650,45 @@ export class AgentSession { return undefined; } - #getRetryFallbackEffectiveChain(role: string): RetryFallbackSelector[] { - const primarySelector = this.#getRetryFallbackPrimarySelector(role); - if (!primarySelector) return []; - const chain = [primarySelector]; - const seen = new Set([primarySelector.raw]); + /** + * Parse one configured chain entry. A `provider/*` entry keeps the failing + * model's id and swaps the provider (google-antigravity/x → google/x); + * ids the target provider lacks are skipped by the candidate loop's + * registry lookup. + */ + #parseRetryFallbackChainEntry( + entry: string, + current: RetryFallbackSelector | undefined, + ): RetryFallbackSelector | undefined { + if (isRetryFallbackWildcardKey(entry)) { + if (!current) return undefined; + const provider = entry.slice(0, -2); + return { raw: `${provider}/${current.id}`, provider, id: current.id, thinkingLevel: undefined }; + } + return parseRetryFallbackSelector(entry, this.#modelRegistry); + } + + #getRetryFallbackEffectiveChain(role: string, currentSelector?: string): RetryFallbackSelector[] { + const parsedCurrent = currentSelector + ? parseRetryFallbackSelector(currentSelector, this.#modelRegistry) + : undefined; + const seen = new Set(); + const chain: RetryFallbackSelector[] = []; + if (isRetryFallbackWildcardKey(role)) { + // A wildcard key has no fixed primary: the active model is the + // primary, followed by the configured provider-level fallbacks. + if (parsedCurrent) { + chain.push(parsedCurrent); + seen.add(parsedCurrent.raw); + } + } else { + const primarySelector = this.#getRetryFallbackPrimarySelector(role); + if (!primarySelector) return []; + chain.push(primarySelector); + seen.add(primarySelector.raw); + } for (const selector of this.#getRetryFallbackChains()[role] ?? []) { - const parsed = parseRetryFallbackSelector(selector, this.#modelRegistry); + const parsed = this.#parseRetryFallbackChainEntry(selector, parsedCurrent); if (!parsed || seen.has(parsed.raw)) continue; seen.add(parsed.raw); chain.push(parsed); @@ -13589,7 +13697,7 @@ export class AgentSession { } #findRetryFallbackCandidates(role: string, currentSelector: string): RetryFallbackSelector[] { - let chain = this.#getRetryFallbackEffectiveChain(role); + let chain = this.#getRetryFallbackEffectiveChain(role, currentSelector); const parsedCurrent = parseRetryFallbackSelector(currentSelector, this.#modelRegistry); if (chain.length === 0 && role === "default" && parsedCurrent) { const chains = this.#getRetryFallbackChains(); @@ -13602,7 +13710,7 @@ export class AgentSession { const seen = new Set([parsedCurrent.raw]); chain = [parsedCurrent]; for (const selector of defaultChain) { - const parsed = parseRetryFallbackSelector(selector, this.#modelRegistry); + const parsed = this.#parseRetryFallbackChainEntry(selector, parsedCurrent); if (!parsed || seen.has(parsed.raw)) continue; seen.add(parsed.raw); chain.push(parsed); diff --git a/packages/coding-agent/test/agent-session-retry-fallback.test.ts b/packages/coding-agent/test/agent-session-retry-fallback.test.ts index 25c8c6791..92b5d0d27 100644 --- a/packages/coding-agent/test/agent-session-retry-fallback.test.ts +++ b/packages/coding-agent/test/agent-session-retry-fallback.test.ts @@ -219,6 +219,215 @@ describe("AgentSession retry fallback", () => { ]); }); + it("activates a model-keyed fallback chain without any role assignment", async () => { + const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5"); + const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); + if (!primaryModel || !fallbackModel) { + throw new Error("Expected bundled test models to exist"); + } + + const requestedModels: string[] = []; + const fallbackAppliedEvents: Array> = []; + const agent = createFallbackAgent(primaryModel, requestedModels); + + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.maxRetries": 1, + "retry.fallbackChains": { + [`${primaryModel.provider}/${primaryModel.id}`]: [`${fallbackModel.provider}/${fallbackModel.id}`], + }, + }); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + + session.subscribe(event => { + if (event.type === "retry_fallback_applied") { + fallbackAppliedEvents.push(event); + } + }); + + await session.prompt("Recover via model-keyed chain"); + await session.waitForIdle(); + + expect(requestedModels).toEqual([ + `${primaryModel.provider}/${primaryModel.id}`, + `${fallbackModel.provider}/${fallbackModel.id}`, + ]); + expect(session.model?.provider).toBe(fallbackModel.provider); + expect(session.model?.id).toBe(fallbackModel.id); + expect(fallbackAppliedEvents).toEqual([ + { + type: "retry_fallback_applied", + from: `${primaryModel.provider}/${primaryModel.id}`, + to: `${fallbackModel.provider}/${fallbackModel.id}`, + role: `${primaryModel.provider}/${primaryModel.id}`, + }, + ]); + }); + + it("prefers a model-keyed chain over the matching role chain", async () => { + const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5"); + const modelKeyFallback = getBundledModel("openai", "gpt-4o-mini"); + const roleChainFallback = getBundledModel("openai", "gpt-4o"); + if (!primaryModel || !modelKeyFallback || !roleChainFallback) { + throw new Error("Expected bundled test models to exist"); + } + + const requestedModels: string[] = []; + const fallbackAppliedEvents: Array> = []; + const agent = createFallbackAgent(primaryModel, requestedModels); + + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.maxRetries": 1, + "retry.fallbackChains": { + default: [`${roleChainFallback.provider}/${roleChainFallback.id}`], + [`${primaryModel.provider}/${primaryModel.id}`]: [`${modelKeyFallback.provider}/${modelKeyFallback.id}`], + }, + }); + settings.setModelRole("default", `${primaryModel.provider}/${primaryModel.id}`); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + + session.subscribe(event => { + if (event.type === "retry_fallback_applied") { + fallbackAppliedEvents.push(event); + } + }); + + await session.prompt("Model-keyed chain wins"); + await session.waitForIdle(); + + expect(requestedModels).toEqual([ + `${primaryModel.provider}/${primaryModel.id}`, + `${modelKeyFallback.provider}/${modelKeyFallback.id}`, + ]); + expect(fallbackAppliedEvents).toEqual([ + { + type: "retry_fallback_applied", + from: `${primaryModel.provider}/${primaryModel.id}`, + to: `${modelKeyFallback.provider}/${modelKeyFallback.id}`, + role: `${primaryModel.provider}/${primaryModel.id}`, + }, + ]); + }); + + it("applies a provider-wildcard chain to any model of that provider", async () => { + const primaryModel = getBundledModel("anthropic", "claude-opus-4-1"); + const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); + if (!primaryModel || !fallbackModel) { + throw new Error("Expected bundled test models to exist"); + } + + const requestedModels: string[] = []; + const fallbackAppliedEvents: Array> = []; + const agent = createFallbackAgent(primaryModel, requestedModels); + + // No exact key for this model and no role assignment: only the + // `anthropic/*` wildcard can match, proving provider-level coverage. + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.maxRetries": 1, + "retry.fallbackChains": { + "anthropic/*": [`${fallbackModel.provider}/${fallbackModel.id}`], + }, + }); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + + session.subscribe(event => { + if (event.type === "retry_fallback_applied") { + fallbackAppliedEvents.push(event); + } + }); + + await session.prompt("Recover via provider wildcard"); + await session.waitForIdle(); + + expect(requestedModels).toEqual([ + `${primaryModel.provider}/${primaryModel.id}`, + `${fallbackModel.provider}/${fallbackModel.id}`, + ]); + expect(session.model?.provider).toBe(fallbackModel.provider); + expect(session.model?.id).toBe(fallbackModel.id); + expect(fallbackAppliedEvents).toEqual([ + { + type: "retry_fallback_applied", + from: `${primaryModel.provider}/${primaryModel.id}`, + to: `${fallbackModel.provider}/${fallbackModel.id}`, + role: "anthropic/*", + }, + ]); + }); + + it("substitutes the failing model id into provider-wildcard chain entries", async () => { + const primaryModel = getBundledModel("google", "gemini-2.5-flash"); + const fallbackModel = getBundledModel("google-vertex", "gemini-2.5-flash"); + if (!primaryModel || !fallbackModel) { + throw new Error("Expected bundled test models to exist"); + } + + const requestedModels: string[] = []; + const fallbackAppliedEvents: Array> = []; + const agent = createFallbackAgent(primaryModel, requestedModels); + + // `google-vertex/*` is not a fixed target: it must adopt the failing + // model's id (google/gemini-2.5-flash -> google-vertex/gemini-2.5-flash). + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.maxRetries": 1, + "retry.fallbackChains": { + "google/*": ["google-vertex/*"], + }, + }); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + + session.subscribe(event => { + if (event.type === "retry_fallback_applied") { + fallbackAppliedEvents.push(event); + } + }); + + await session.prompt("Recover via id-preserving wildcard entry"); + await session.waitForIdle(); + + expect(requestedModels).toEqual([ + `${primaryModel.provider}/${primaryModel.id}`, + `${fallbackModel.provider}/${fallbackModel.id}`, + ]); + expect(session.model?.provider).toBe("google-vertex"); + expect(session.model?.id).toBe(primaryModel.id); + expect(fallbackAppliedEvents).toEqual([ + { + type: "retry_fallback_applied", + from: `${primaryModel.provider}/${primaryModel.id}`, + to: `google-vertex/${primaryModel.id}`, + role: "google/*", + }, + ]); + }); + it("uses the active initial model as the default fallback primary when other role fallback chains are configured", async () => { const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5"); const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); @@ -1596,6 +1805,39 @@ describe("AgentSession retry fallback", () => { ); }); + it("warns on unknown or malformed model-selector chain keys at startup", () => { + const primaryModel = getBundledModel("openai", "gpt-4o-mini"); + if (!primaryModel) { + throw new Error("Expected bundled OpenAI test model to exist"); + } + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.fallbackChains": { + "nonexistent-provider/nonexistent-model": [`${primaryModel.provider}/${primaryModel.id}`], + [`${primaryModel.provider}/${primaryModel.id}`]: ["openai/gpt-4o"], + }, + }); + const agent = new Agent({ + getApiKey: model => `${model.provider}-test-key`, + initialState: { model: primaryModel, systemPrompt: ["Test"], tools: [], messages: [] }, + streamFn: () => { + throw new Error("Not exercised"); + }, + }); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + + expect(session.configWarnings).toContain( + "retry.fallbackChains key references unknown model: nonexistent-provider/nonexistent-model", + ); + expect(session.configWarnings.filter(w => w.includes(`${primaryModel.provider}/${primaryModel.id}`))).toEqual([]); + }); + it("normalizes suppression by base selector and clears it on model refresh", async () => { const future = Date.now() + 60_000; modelRegistry.suppressSelector("openai/gpt-4o:high", future); diff --git a/packages/coding-agent/test/model-hub.test.ts b/packages/coding-agent/test/model-hub.test.ts index a9d3bed7c..95fd2d3b6 100644 --- a/packages/coding-agent/test/model-hub.test.ts +++ b/packages/coding-agent/test/model-hub.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeAll, describe, expect, test, vi } from "bun:test"; +import { afterEach, beforeAll, describe, expect, type Mock, test, vi } from "bun:test"; import { stripVTControlCharacters } from "node:util"; import { ThinkingLevel } from "@oh-my-pi/pi-agent-core"; import type { Model } from "@oh-my-pi/pi-ai"; @@ -78,6 +78,7 @@ interface HubHarness { onPick: ReturnType; onLoginRequest: ReturnType; onCancel: ReturnType; + onFallbackChainChange: Mock<(role: string, chain: string[]) => void>; } const openHubs: ModelHubComponent[] = []; @@ -100,6 +101,16 @@ function createHub(options: { const onPick = vi.fn(); const onLoginRequest = vi.fn(); const onCancel = vi.fn(); + // Mirror the controller: persist chain edits so the hub's re-read sees them. + const onFallbackChainChange = vi.fn((role: string, chain: string[]) => { + const chains = { ...settings.get("retry.fallbackChains") }; + if (chain.length === 0) { + delete chains[role]; + } else { + chains[role] = chain; + } + settings.override("retry.fallbackChains", chains); + }); const hub = new ModelHubComponent( ui, settings, @@ -111,12 +122,13 @@ function createHub(options: { onPick: options.callbacks?.onPick ?? onPick, onLoginRequest: options.callbacks?.onLoginRequest ?? onLoginRequest, onCycleOrderChange: options.callbacks?.onCycleOrderChange, + onFallbackChainChange: options.callbacks?.onFallbackChainChange ?? onFallbackChainChange, onCancel: options.callbacks?.onCancel ?? onCancel, }, options.hub, ); openHubs.push(hub); - return { hub, onAssign, onUnassign, onPick, onLoginRequest, onCancel }; + return { hub, onAssign, onUnassign, onPick, onLoginRequest, onCancel, onFallbackChainChange }; } const DOWN = "\x1b[B"; @@ -304,7 +316,8 @@ describe("ModelHub", () => { hub.handleInput(UP); // All models → Roles (since Recent is removed) hub.handleInput("\n"); // dive into rows - hub.handleInput(UP); // wraps to the trailing "+ New role…" row + hub.handleInput(UP); // wraps to the trailing "+ New fallback…" row + hub.handleInput(UP); // skips the section divider up to "+ New role…" hub.handleInput("\n"); expect(footerLine(hub.render(220))).toContain("New role name:"); @@ -317,7 +330,6 @@ describe("ModelHub", () => { const call = onAssign.mock.calls[0]; expect(call?.[1]).toBe("reviewer"); expect(call?.[3]).toBe("test/reviewer-model"); - expect(call?.[4]).toBe("modelRole"); }); }); @@ -340,7 +352,6 @@ describe("ModelHub", () => { expect(call?.[1]).toBe("default"); expect(call?.[2]).toBe(ThinkingLevel.Inherit); expect(call?.[3]).toBe("openai/gpt-5.5"); - expect(call?.[4]).toBe("modelRole"); // The thinking strip follows immediately, scoped to the model's // real ladder: gpt-5.5 tops out at xhigh — no invented max tier. @@ -379,20 +390,24 @@ describe("ModelHub", () => { expect(footerLine(hub.render(220))).not.toContain("inherit"); }); - test("retry-fallback chip fires the retryFallback action without a thinking strip", () => { + test("retry-fallback chip appends the model to the default chain without a thinking strip", () => { const model = makeModel("test", "retry-fallback-model"); - const { hub, onAssign } = createHub({ models: [model], scoped: true }); + const { hub, onAssign, onFallbackChainChange } = createHub({ models: [model], scoped: true }); installTestTheme(); hub.handleInput("\n"); hub.handleInput(LEFT); // wraps to the trailing retry-fallback chip hub.handleInput("\n"); - expect(onAssign).toHaveBeenCalledTimes(1); - const call = onAssign.mock.calls[0]; - expect(call?.[1]).toBe("default"); - expect(call?.[4]).toBe("retryFallback"); + expect(onFallbackChainChange).toHaveBeenCalledWith("default", ["test/retry-fallback-model"]); + expect(onAssign).not.toHaveBeenCalled(); expect(footerLine(hub.render(220))).not.toContain("inherit"); + + // A second registration of the same model is a no-op, not a duplicate. + hub.handleInput("\n"); + hub.handleInput(LEFT); + hub.handleInput("\n"); + expect(onFallbackChainChange).toHaveBeenCalledTimes(1); }); test("overflowing role strip scrolls left so the selected chip stays visible", () => { @@ -417,6 +432,181 @@ describe("ModelHub", () => { }); }); + describe("fallback chains in the roles view", () => { + /** Hop to the Roles sidebar entry and dive into its rows. */ + function enterRolesView(hub: ModelHubComponent): void { + hub.handleInput(UP); // All models → Roles + hub.handleInput("\n"); // dive into the rows + } + + test("renders configured chain entries as indented rows under their role", () => { + const a = makeModel("test", "model-a"); + const b = makeModel("test", "model-b"); + const settings = Settings.isolated({ + "retry.fallbackChains": { default: ["test/model-a", "test/model-b"] }, + }); + const { hub } = createHub({ models: [a, b], scoped: true, settings }); + + enterRolesView(hub); + const rendered = normalize(hub.render(220)); + expect(rendered).toContain("↳ test/model-a"); + expect(rendered).toContain("↳ test/model-b"); + }); + + test("f on a role opens fallback assignment and Enter appends the picked model", () => { + const a = makeModel("test", "model-a"); + const settings = Settings.isolated({}); + const { hub, onFallbackChainChange, onAssign } = createHub({ models: [a], scoped: true, settings }); + + enterRolesView(hub); + hub.handleInput("f"); // add a fallback for the first role (default) + expect(normalize(hub.render(220))).toContain("Adding fallback for"); + + hub.handleInput("\n"); // pick the only model + expect(onFallbackChainChange).toHaveBeenCalledWith("default", ["test/model-a"]); + expect(onAssign).not.toHaveBeenCalled(); // no role assignment, no thinking strip + expect(normalize(hub.render(220))).toContain("↳ test/model-a"); + }); + + test("x removes a chain entry and Enter on an entry replaces it", () => { + const a = makeModel("test", "model-a"); + const b = makeModel("test", "model-b"); + const settings = Settings.isolated({ + "retry.fallbackChains": { default: ["test/model-a", "test/model-b"] }, + }); + const { hub, onFallbackChainChange } = createHub({ models: [a, b], scoped: true, settings }); + + enterRolesView(hub); + hub.handleInput(DOWN); // default → its first chain entry (model-a) + hub.handleInput("\n"); // replace this entry + expect(normalize(hub.render(220))).toContain("Replacing fallback of"); + for (const ch of "model-b") hub.handleInput(ch); // search: arrows hop scopes in assign mode + hub.handleInput("\n"); + expect(onFallbackChainChange).toHaveBeenLastCalledWith("default", ["test/model-b"]); + + hub.handleInput("x"); // cursor landed on the replaced entry — remove it + expect(onFallbackChainChange).toHaveBeenLastCalledWith("default", []); + expect(normalize(hub.render(220))).not.toContain("↳"); + }); + + test("] moves a chain entry later and the cursor follows it", () => { + const a = makeModel("test", "model-a"); + const b = makeModel("test", "model-b"); + const settings = Settings.isolated({ + "retry.fallbackChains": { default: ["test/model-a", "test/model-b"] }, + }); + const { hub, onFallbackChainChange } = createHub({ models: [a, b], scoped: true, settings }); + + enterRolesView(hub); + hub.handleInput(DOWN); // first chain entry (model-a) + hub.handleInput("]"); + expect(onFallbackChainChange).toHaveBeenLastCalledWith("default", ["test/model-b", "test/model-a"]); + + // Cursor followed the moved entry: x removes model-a, not model-b. + hub.handleInput("x"); + expect(onFallbackChainChange).toHaveBeenLastCalledWith("default", ["test/model-b"]); + }); + + test("clicking a roles row hits the row under the pointer", () => { + const a = makeModel("test", "model-a"); + const { hub } = createHub({ models: [a], scoped: true }); + + hub.handleInput(UP); // All models → Roles + // Derive the pointer row from the frame itself: the fullscreen + // overlay paints from screen row 0, so frame index == screen row. + const frame = hub.render(220).map(line => stripVTControlCharacters(line)); + const screenRow = frame.findIndex(line => line.includes("DEFAULT")); + expect(screenRow).toBeGreaterThan(0); + const sgr = `\x1b[<0;61;${screenRow + 1}M`; // SGR reports are 1-based + hub.handleInput(sgr); // select (dive into rows) + hub.handleInput(sgr); // click-again activates + expect(normalize(hub.render(220))).toContain("Assigning DEFAULT"); + }); + + test("fallbacks chip keys a new chain by the selected model", () => { + const a = makeModel("test", "model-a"); + const b = makeModel("test", "model-b"); + const { hub, onFallbackChainChange } = createHub({ models: [a, b], scoped: true }); + + for (const ch of "model-a") hub.handleInput(ch); + hub.handleInput("\n"); // open the strip for model-a + hub.handleInput(LEFT); // retry-fallback + hub.handleInput(LEFT); // fallbacks:test/* + hub.handleInput(LEFT); // fallbacks:model-a + hub.handleInput("\n"); + expect(normalize(hub.render(220))).toContain("Adding fallback for test/model-a"); + + for (const ch of "model-b") hub.handleInput(ch); + hub.handleInput("\n"); + expect(onFallbackChainChange).toHaveBeenLastCalledWith("test/model-a", ["test/model-b"]); + const rendered = normalize(hub.render(220)); + expect(rendered).toContain("test/model-a"); + expect(rendered).toContain("↳ test/model-b"); + }); + + test("provider chip keys the chain by provider/*", () => { + const a = makeModel("test", "model-a"); + const b = makeModel("test", "model-b"); + const { hub, onFallbackChainChange } = createHub({ models: [a, b], scoped: true }); + + for (const ch of "model-a") hub.handleInput(ch); + hub.handleInput("\n"); + hub.handleInput(LEFT); // retry-fallback + hub.handleInput(LEFT); // fallbacks:test/* + hub.handleInput("\n"); + expect(normalize(hub.render(220))).toContain("Adding fallback for test/*"); + + for (const ch of "model-b") hub.handleInput(ch); + hub.handleInput("\n"); + expect(onFallbackChainChange).toHaveBeenLastCalledWith("test/*", ["test/model-b"]); + }); + + test("+ New fallback… picks the protected model, then keys the chain via the strip", () => { + const a = makeModel("test", "model-a"); + const b = makeModel("test", "model-b"); + const { hub, onFallbackChainChange } = createHub({ models: [a, b], scoped: true }); + + enterRolesView(hub); + hub.handleInput(UP); // wrap to the trailing "+ New fallback…" + hub.handleInput("\n"); + expect(normalize(hub.render(220))).toContain("New fallback chain"); + + for (const ch of "model-a") hub.handleInput(ch); + hub.handleInput("\n"); // pick the protected model + const strip = footerLine(hub.render(220)); + expect(strip).toContain("for test/model-a"); + expect(strip).toContain("for test/*"); + + hub.handleInput("\n"); // key by the exact model + expect(normalize(hub.render(220))).toContain("Adding fallback for test/model-a"); + for (const ch of "model-b") hub.handleInput(ch); + hub.handleInput("\n"); + expect(onFallbackChainChange).toHaveBeenLastCalledWith("test/model-a", ["test/model-b"]); + }); + + test("model-keyed chains render below the separator and x clears the whole chain", () => { + const a = makeModel("test", "model-a"); + const settings = Settings.isolated({ + "retry.fallbackChains": { "test/*": ["test/model-a"] }, + }); + const { hub, onFallbackChainChange } = createHub({ models: [a], scoped: true, settings }); + + enterRolesView(hub); + const rendered = normalize(hub.render(220)); + expect(rendered).toContain("test/*"); + expect(rendered).toContain("↳ test/model-a"); + expect(rendered).toContain("+ New fallback…"); + expect(rendered).toMatch(/─{10,}/); // the roles/fallbacks divider + + hub.handleInput(UP); // + New fallback… + hub.handleInput(UP); // ↳ test/model-a + hub.handleInput(UP); // test/* header (separator is skipped) + hub.handleInput("x"); + expect(onFallbackChainChange).toHaveBeenLastCalledWith("test/*", []); + expect(normalize(hub.render(220))).not.toContain("↳ test/model-a"); + }); + }); + describe("pick mode", () => { test("disables models below the current context size and picks the first enabled one", () => { const small = makeModel("test", "a-small", 4096); From e7955ddf3c86b845f94992c1b6e411bcc75a3988 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 11 Jul 2026 22:03:59 +0200 Subject: [PATCH 11/80] feat(coding-agent): introduced sequential message queueing and commands - Implemented `/queue` command and `->`/`=>` shorthands to support deferred, sequential message processing. - Added a robust parsing utility to handle various list-based queue inputs and automate yield management. - Integrated visual decorations and state tracking to provide real-time feedback on queueing status. - Enabled non-cursor line text decoration in the TUI to support dynamic queue header rendering and list numbering. --- packages/coding-agent/CHANGELOG.md | 1 + .../modes/components/custom-editor.test.ts | 59 +++++++- .../src/modes/components/custom-editor.ts | 53 +++++-- .../src/modes/controllers/input-controller.ts | 131 +++++++++++++++++ .../src/modes/interactive-mode.ts | 5 + .../coding-agent/src/modes/queue-input.ts | 132 ++++++++++++++++++ packages/coding-agent/src/modes/types.ts | 2 + .../src/modes/utils/ui-helpers.ts | 39 +++--- .../src/slash-commands/builtin-registry.ts | 9 ++ .../test/input-controller-skill-queue.test.ts | 29 +++- .../input-controller-slash-history.test.ts | 115 ++++++++++++++- .../selector-settings-side-effects.test.ts | 2 +- packages/tui/CHANGELOG.md | 4 + packages/tui/src/components/editor.ts | 13 +- 14 files changed, 551 insertions(+), 43 deletions(-) create mode 100644 packages/coding-agent/src/modes/queue-input.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 3f8816a97..9b021d8d6 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -8,6 +8,7 @@ - Added model-oriented keys and wildcard entries to `retry.fallbackChains`: a `provider/model-id` key attaches a fallback chain to that exact model, a `provider/*` key covers every current or future model of a provider, and a `provider/*` chain entry keeps the failing model's id while swapping the provider (`google-antigravity/x` → `google/x`) — so fallbacks survive role and model reassignments without config edits. Keys resolve by specificity: exact model, then provider wildcard, then role, then `default`. - Added fallback-chain editing to the /models Roles view: each role's `retry.fallbackChains` entries render as indented rows beneath it, `f` picks a fallback model to append, Enter on an entry replaces it, `x`/backspace removes it, and `[`/`]` (or shift+↑/↓) reorder the chain. - Added model-keyed fallback management to the /models Roles view: model and `provider/*` chains render as a separate section below the roles (divider + "+ New fallback…" row for creating one by picking the protected model, then keying it by model or provider), with the same replace/remove/reorder editing as role chains; the model strip gains `fallbacks:` and `fallbacks:/*` chips as shortcuts. +- Added `/queue ` plus `->` / `=>` composer shorthand for follow-up messages that wait until the agent yields. The shorthand opens a dim `Queueing` header and splits sequential numeric, Roman-numeral, or alphabetic lists into separately highlighted queue entries. ### Changed diff --git a/packages/coding-agent/src/modes/components/custom-editor.test.ts b/packages/coding-agent/src/modes/components/custom-editor.test.ts index 18a5fd681..24c67cefb 100644 --- a/packages/coding-agent/src/modes/components/custom-editor.test.ts +++ b/packages/coding-agent/src/modes/components/custom-editor.test.ts @@ -1,8 +1,9 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "bun:test"; +import { CURSOR_MARKER } from "@oh-my-pi/pi-tui"; import { setKittyProtocolActive } from "@oh-my-pi/pi-tui/keys"; import { $ } from "bun"; import { getDefaultPasteImageKeys } from "../../config/keybindings"; -import { getEditorTheme, initTheme } from "../theme/theme"; +import { getEditorTheme, initTheme, theme } from "../theme/theme"; import { CustomEditor, extractBracketedImagePastePaths, @@ -78,6 +79,62 @@ describe("CustomEditor placeholder decoration", () => { }); }); +describe("CustomEditor queue shorthand decoration", () => { + beforeAll(async () => { + await initTheme(); + }); + + it("reserves the first line as soon as either queue prefix is completed", () => { + for (const prefix of ["->", "=>"]) { + const editor = new CustomEditor(getEditorTheme()); + editor.handleInput(prefix[0] ?? ""); + expect(editor.getText()).toBe(prefix[0]); + + editor.handleInput(prefix[1] ?? ""); + expect(editor.getText()).toBe(`${prefix}\n`); + expect(editor.getCursor()).toEqual({ line: 1, col: 0 }); + + editor.handleInput("\x7f"); + expect(editor.getText()).toBe(`${prefix}\n`); + expect(editor.getCursor()).toEqual({ line: 1, col: 0 }); + } + }); + + it("renders the reserved line as a dim Queueing header", () => { + for (const prefix of ["->", "=>"]) { + const editor = new CustomEditor(getEditorTheme()); + editor.setText(`${prefix}\nqueue this`); + + expect(editor.decorateText(prefix)).toBe(theme.fg("dim", `Queueing ${theme.nav.selected}`)); + editor.focused = true; + const rendered = editor.render(40).map(line => Bun.stripANSI(line.replace(CURSOR_MARKER, ""))); + expect(rendered.some(line => line.includes(`Queueing ${theme.nav.selected}`))).toBe(true); + expect(rendered.every(line => Bun.stringWidth(line) === 40)).toBe(true); + expect(rendered.some(line => line.includes("queue this"))).toBe(true); + } + }); + + it("highlights dot and parenthesis markers only for detected queue lists", () => { + for (const [input, marker] of [ + ["=>\n1. first\n2. second", "1."], + ["=>\n1) first\n2) second", "1)"], + ]) { + const editor = new CustomEditor(getEditorTheme()); + editor.setText(input); + expect(editor.decorateText(`${marker} first`).startsWith(theme.fg("accent", marker))).toBe(true); + } + + const unfinished = new CustomEditor(getEditorTheme()); + unfinished.setText("=>\n1. first\n2. second\n3. third\n4."); + expect(unfinished.decorateText("1. first").startsWith(theme.fg("accent", "1."))).toBe(true); + expect(unfinished.decorateText("4.").startsWith(theme.fg("accent", "4."))).toBe(true); + + const editor = new CustomEditor(getEditorTheme()); + editor.setText("=>\n1. first\n3. third"); + expect(editor.decorateText("1. first")).toBe("1. first"); + }); +}); + describe("CustomEditor bracketed path paste", () => { it("leaves a pasted bare .png filename on the normal text path", () => { expect(extractBracketedImagePastePaths(bracketedPaste("icon-photo-default.png"))).toBeUndefined(); diff --git a/packages/coding-agent/src/modes/components/custom-editor.ts b/packages/coding-agent/src/modes/components/custom-editor.ts index bf144ff11..98394db89 100644 --- a/packages/coding-agent/src/modes/components/custom-editor.ts +++ b/packages/coding-agent/src/modes/components/custom-editor.ts @@ -6,7 +6,8 @@ import type { AppKeybinding } from "../../config/keybindings"; import { isSettingsInitialized, settings } from "../../config/settings"; import { imageReferenceHyperlink, PLACEHOLDER_REGEX, renderPlaceholders } from "../image-references"; import { hasMagicKeyword, highlightMagicKeywords } from "../magic-keywords"; -import { fgOrPlain } from "../theme/theme"; +import { isQueuedMessageList, parseQueueShorthand, QUEUE_LIST_MARKER_RE } from "../queue-input"; +import { fgOrPlain, theme } from "../theme/theme"; type ConfigurableEditorAction = Extract< AppKeybinding, @@ -326,21 +327,41 @@ export class CustomEditor extends Editor { * timer to request the next animation frame. Undefined when nobody is * listening (tests, headless callers); the timer chain still self-cleans. */ #requestShimmerRepaint: (() => void) | undefined; + #queueDecorationText: string | undefined; + #queueShorthandActive = false; + #queueListActive = false; - /** Gradient-highlight the "ultrathink" / "orchestrate" / "workflowz" keywords as the user types - * them, skipping any occurrence inside code spans, fenced blocks, or XML sections. Also make - * pasted image placeholders visually distinct and hyperlink them once their blob file exists. - * When the editor is focused, the buffer contains a magic keyword, and `magicKeywords.enabled` - * is on, the gradient shifts every frame to produce a Claude-Code-style shimmer; each render - * schedules the next frame, so losing focus, deleting the keyword, or flipping the setting - * stops the animation on its own. The static glow itself runs even when shimmering is gated - * off, matching existing behavior for the editor and sent bubbles. */ + /** Decorate magic keywords, attachments, and the queue-composer header/list markers. + * Queue shorthand reserves its first logical line as a dim `Queueing` label; sequential + * item markers use the accent color so separate follow-ups remain visible while composing. */ decorateText = (text: string): string => { - const animated = this.focused && this.#shimmerEnabled() && hasMagicKeyword(this.getText()); + const editorText = this.getText(); + const animated = this.focused && this.#shimmerEnabled() && hasMagicKeyword(editorText); const phase = animated ? (Date.now() % CustomEditor.SHIMMER_PERIOD_MS) / CustomEditor.SHIMMER_PERIOD_MS : 0; if (animated) this.#scheduleShimmerFrame(); + if (this.#queueDecorationText !== editorText) { + this.#queueDecorationText = editorText; + const queueBody = parseQueueShorthand(editorText); + this.#queueShorthandActive = queueBody !== undefined; + this.#queueListActive = queueBody !== undefined && isQueuedMessageList(queueBody); + } return renderPlaceholders(text, { - renderText: value => highlightMagicKeywords(value, undefined, phase), + renderText: value => { + const highlighted = highlightMagicKeywords(value, undefined, phase); + if (this.#queueShorthandActive && (value.startsWith("->") || value.startsWith("=>"))) { + const icon = typeof theme === "undefined" ? "➤" : theme.nav.selected; + return `${fgOrPlain("dim", `Queueing ${icon}`)}${highlighted.slice(2)}`; + } + if (this.#queueListActive) { + const markerMatch = QUEUE_LIST_MARKER_RE.exec(value); + if (markerMatch) { + const indent = markerMatch[1] ?? ""; + const markerEnd = markerMatch[0].length; + return `${indent}${fgOrPlain("accent", value.slice(indent.length, markerEnd))}${highlighted.slice(markerEnd)}`; + } + } + return highlighted; + }, renderReference: (value, kind, index) => kind === "image" ? imageReferenceHyperlink(value, index, this.imageLinks, label => @@ -628,6 +649,7 @@ export class CustomEditor extends Editor { this.#pendingInput.push(data); return; } + const hadBareQueuePrefix = this.getText() === "->" || this.getText() === "=>"; const kittyParsed = parseKittySequence(data); if (kittyParsed && (kittyParsed.modifier & 64) !== 0 && this.onCapsLock) { // Caps Lock is modifier bit 64 @@ -831,5 +853,14 @@ export class CustomEditor extends Editor { // Pass to parent for normal handling super.handleInput(data); + const cursor = this.getCursor(); + if ( + !hadBareQueuePrefix && + (this.getText() === "->" || this.getText() === "=>") && + cursor.line === 0 && + cursor.col === 2 + ) { + this.insertText("\n"); + } } } diff --git a/packages/coding-agent/src/modes/controllers/input-controller.ts b/packages/coding-agent/src/modes/controllers/input-controller.ts index 3ccd61b87..e166b7add 100644 --- a/packages/coding-agent/src/modes/controllers/input-controller.ts +++ b/packages/coding-agent/src/modes/controllers/input-controller.ts @@ -13,6 +13,7 @@ import { TinyTitleDownloadProgressComponent } from "../../modes/components/tiny- import { expandEmoticons } from "../../modes/emoji-autocomplete"; import { materializeImageReferenceLinks, shiftImageMarkers } from "../../modes/image-references"; import { createPromptActionAutocompleteProvider } from "../../modes/prompt-action-autocomplete"; +import { parseQueueShorthand, splitQueuedMessages } from "../../modes/queue-input"; import { invokeSkillCommandFromText, isKnownSkillCommand } from "../../modes/skill-command"; import type { InteractiveModeContext } from "../../modes/types"; import manualContinuePrompt from "../../prompts/system/manual-continue.md" with { type: "text" }; @@ -650,6 +651,16 @@ export class InputController { if (!text && !hasInputImages) return; + const queueBody = parseQueueShorthand(text); + if (queueBody !== undefined) { + await this.#queueForYield(queueBody, { + historyText: text, + images: inputImages, + imageLinks: inputImageLinks, + }); + return; + } + // Handle built-in slash commands if (text) { const slashResult = await executeBuiltinSlashCommand(text, { @@ -1117,6 +1128,126 @@ export class InputController { } } + /** Queue `/queue` input behind an active turn, or start it immediately when idle. */ + async handleQueueCommand(text: string): Promise { + const images = this.ctx.editor.pendingImages.length > 0 ? [...this.ctx.editor.pendingImages] : undefined; + const imageLinks = + images && this.ctx.editor.pendingImageLinks.length > 0 ? [...this.ctx.editor.pendingImageLinks] : undefined; + await this.#queueForYield(text, { images, imageLinks }); + } + + async #queueForYield( + text: string, + options: { + historyText?: string; + images?: ImageContent[]; + imageLinks?: (string | undefined)[]; + }, + ): Promise { + const splitMessages = splitQueuedMessages(text); + if (splitMessages.length === 0 && !options.images?.length) { + this.ctx.editor.clearDraft(); + this.ctx.showWarning("Usage: /queue (or start a prompt with -> / =>)"); + return; + } + + const messages = splitMessages.length > 0 ? splitMessages : [""]; + const originalDraft = this.ctx.editor.getText(); + const images = options.images?.length ? [...options.images] : undefined; + const imageLinks = options.imageLinks + ? [...options.imageLinks] + : images + ? images.map(() => undefined) + : undefined; + this.ctx.editor.clearDraft(options.historyText); + + if (this.ctx.session.isCompacting) { + for (let index = 0; index < messages.length; index++) { + this.ctx.compactionQueuedMessages.push({ + text: messages[index] ?? "", + mode: "followUp", + images: index === 0 ? images : undefined, + }); + } + this.ctx.updatePendingMessagesDisplay(); + this.ctx.showStatus( + messages.length === 1 + ? "Queued message for after compaction" + : `Queued ${messages.length} messages for after compaction`, + ); + this.ctx.ui.requestRender(); + return; + } + + const startImmediately = !this.ctx.session.isStreaming && this.ctx.session.queuedMessageCount === 0; + let queuedCount = 0; + try { + if (startImmediately && this.ctx.onInputCallback) { + const first = messages[0] ?? ""; + const submission = this.ctx.startPendingSubmission({ + text: first, + images, + imageLinks, + streamingBehavior: "followUp", + }); + this.ctx.onInputCallback(submission); + queuedCount = 1; + } + while (queuedCount < messages.length) { + const message = messages[queuedCount] ?? ""; + const queuedImages = queuedCount === 0 ? images : undefined; + await this.ctx.withLocalSubmission( + message, + async () => { + if (startImmediately && queuedCount === 0) { + await this.ctx.session.prompt(message, { + images: queuedImages, + streamingBehavior: "followUp", + }); + } else { + await this.ctx.session.followUp(message, queuedImages); + } + }, + { imageCount: queuedImages?.length ?? 0 }, + ); + queuedCount++; + } + } catch (error) { + if (queuedCount === 0) { + this.ctx.editor.setText(originalDraft); + if (images) { + this.ctx.editor.pendingImages = images; + this.ctx.editor.pendingImageLinks = imageLinks ?? images.map(() => undefined); + this.ctx.editor.imageLinks = this.ctx.editor.pendingImageLinks; + } + } else { + const remaining = messages.slice(queuedCount); + const restored = + remaining.length === 1 + ? `=> ${remaining[0]}` + : `=>\n${remaining + .map((message, index) => `${index + 1}. ${message.replaceAll("\n", "\n ")}`) + .join("\n")}`; + this.ctx.editor.setText(restored); + } + this.ctx.showError(error instanceof Error ? error.message : String(error)); + } + + this.ctx.updatePendingMessagesDisplay(); + if (queuedCount === messages.length) { + this.ctx.showStatus( + startImmediately + ? queuedCount === 1 + ? "Sent queued message" + : `Sent first message; queued ${queuedCount - 1} for later yields` + : queuedCount === 1 + ? "Queued message for when the agent yields" + : `Queued ${queuedCount} messages for when the agent yields`, + ); + } + this.ctx.ui.requestRender(); + } + /** Send editor text as a follow-up message (queued behind current stream). */ async handleFollowUp(): Promise { let text = this.ctx.editor.getExpandedText().trim(); diff --git a/packages/coding-agent/src/modes/interactive-mode.ts b/packages/coding-agent/src/modes/interactive-mode.ts index cc968c862..dc4716dec 100644 --- a/packages/coding-agent/src/modes/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive-mode.ts @@ -4287,6 +4287,11 @@ export class InteractiveMode implements InteractiveModeContext { return this.#inputController.handleImagePaste(); } + /** Queue slash-command input behind the active turn. */ + handleQueueCommand(message: string): Promise { + return this.#inputController.handleQueueCommand(message); + } + handleBtwCommand(question: string): Promise { return this.#btwController.start(question); } diff --git a/packages/coding-agent/src/modes/queue-input.ts b/packages/coding-agent/src/modes/queue-input.ts new file mode 100644 index 000000000..ee357f2fb --- /dev/null +++ b/packages/coding-agent/src/modes/queue-input.ts @@ -0,0 +1,132 @@ +const QUEUE_PREFIXES: readonly string[] = ["->", "=>"]; +/** Prefix matcher shared by queue-list parsing and editor highlighting. */ +export const QUEUE_LIST_MARKER_RE = /^([\t ]*)(\d+|[A-Za-z]+)([.)])(?=[\t ]|$)/; +const CANONICAL_ROMAN_RE = /^(?=[MDCLXVI])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/i; + +interface EnumeratedItem { + line: number; + indent: string; + marker: string; + punctuation: string; + content: string; +} + +interface EnumeratedList { + source: string; + lines: string[]; + items: EnumeratedItem[]; +} + +/** Extract the message body from the `->` / `=>` yield-queue shorthand. */ +export function parseQueueShorthand(text: string): string | undefined { + const prefix = QUEUE_PREFIXES.find(candidate => text.startsWith(candidate)); + return prefix ? text.slice(prefix.length).trim() : undefined; +} + +function parseEnumeratedItem(line: string, lineIndex: number): EnumeratedItem | undefined { + const match = QUEUE_LIST_MARKER_RE.exec(line); + if (!match) return undefined; + const [matched, indent, marker, punctuation] = match; + if (indent === undefined || marker === undefined || punctuation === undefined) return undefined; + return { line: lineIndex, indent, marker, punctuation, content: line.slice(matched.length).trimStart() }; +} + +function decodeDecimal(marker: string): number | undefined { + if (!/^\d+$/.test(marker)) return undefined; + const value = Number(marker); + return Number.isSafeInteger(value) ? value : undefined; +} + +function decodeRoman(marker: string): number | undefined { + if (!CANONICAL_ROMAN_RE.test(marker)) return undefined; + const values: Readonly> = { + I: 1, + V: 5, + X: 10, + L: 50, + C: 100, + D: 500, + M: 1000, + }; + const upper = marker.toUpperCase(); + let value = 0; + for (let index = 0; index < upper.length; index++) { + const current = values[upper[index] ?? ""]; + if (current === undefined) return undefined; + const next = values[upper[index + 1] ?? ""] ?? 0; + value += current < next ? -current : current; + } + return value; +} + +function decodeAlpha(marker: string): number | undefined { + if (!/^[A-Za-z]+$/.test(marker)) return undefined; + let value = 0; + for (const char of marker.toUpperCase()) { + value = value * 26 + char.charCodeAt(0) - 64; + if (!Number.isSafeInteger(value)) return undefined; + } + return value; +} + +function isSequential(markers: readonly string[], decode: (marker: string) => number | undefined): boolean { + let previous = decode(markers[0] ?? ""); + if (previous === undefined) return false; + for (let index = 1; index < markers.length; index++) { + const current = decode(markers[index] ?? ""); + if (current === undefined || current !== previous + 1) return false; + previous = current; + } + return true; +} + +function isEnumeratedSequence(items: readonly EnumeratedItem[]): boolean { + const markers = items.map(item => item.marker); + if (markers.every(marker => /^\d+$/.test(marker))) return isSequential(markers, decodeDecimal); + if ( + !markers.every(marker => marker === marker.toUpperCase()) && + !markers.every(marker => marker === marker.toLowerCase()) + ) { + return false; + } + return isSequential(markers, decodeRoman) || isSequential(markers, decodeAlpha); +} + +function parseEnumeratedList(text: string): EnumeratedList | undefined { + const source = text.trim(); + if (!source) return undefined; + const lines = source.split(/\r?\n/); + const first = parseEnumeratedItem(lines[0] ?? "", 0); + if (!first) return undefined; + + const items = [first]; + for (let lineIndex = 1; lineIndex < lines.length; lineIndex++) { + const item = parseEnumeratedItem(lines[lineIndex] ?? "", lineIndex); + if (item?.indent === first.indent) items.push(item); + } + if (items.length < 2 || items.some(item => item.punctuation !== first.punctuation) || !isEnumeratedSequence(items)) { + return undefined; + } + return { source, lines, items }; +} + +/** Whether text currently forms a sequential queue list, including an unfinished trailing item. */ +export function isQueuedMessageList(text: string): boolean { + return parseEnumeratedList(text) !== undefined; +} + +/** Split a sequential numeric, Roman-numeral, or alphabetic list into queue entries. */ +export function splitQueuedMessages(text: string): string[] { + const list = parseEnumeratedList(text); + if (!list) { + const source = text.trim(); + return source ? [source] : []; + } + + const messages = list.items.map((item, index) => { + const nextLine = list.items[index + 1]?.line ?? list.lines.length; + return [item.content, ...list.lines.slice(item.line + 1, nextLine)].join("\n").trim(); + }); + while (messages.at(-1) === "") messages.pop(); + return messages.length > 0 && messages.every(Boolean) ? messages : [list.source]; +} diff --git a/packages/coding-agent/src/modes/types.ts b/packages/coding-agent/src/modes/types.ts index 1f3d13c18..cf3dd39eb 100644 --- a/packages/coding-agent/src/modes/types.ts +++ b/packages/coding-agent/src/modes/types.ts @@ -377,6 +377,8 @@ export interface InteractiveModeContext { handleCtrlZ(): void; handleDequeue(): void; handleImagePaste(): Promise; + /** Queue a message for delivery only after the active agent turn would stop. */ + handleQueueCommand(message: string): Promise; handleBtwCommand(question: string): Promise; handleTanCommand(work: string): Promise; hasActiveBtw(): boolean; diff --git a/packages/coding-agent/src/modes/utils/ui-helpers.ts b/packages/coding-agent/src/modes/utils/ui-helpers.ts index 447998a74..8a5e5776f 100644 --- a/packages/coding-agent/src/modes/utils/ui-helpers.ts +++ b/packages/coding-agent/src/modes/utils/ui-helpers.ts @@ -46,6 +46,7 @@ import { type SkillPromptDetails, } from "../../session/messages"; import type { SessionContext } from "../../session/session-context"; +import { replaceTabs } from "../../tools/render-utils"; import { buildSkillCommandPrompt, invokeSkillCommandFromText, isKnownSkillCommand } from "../skill-command"; import { createAssistantMessageComponent } from "./interactive-context-helpers"; import { @@ -677,35 +678,33 @@ export class UiHelpers { this.ctx.pendingMessagesContainer.disposeChildren(); const queuedMessages = this.ctx.viewSession.getQueuedMessages() as QueuedMessages; - const steeringMessages: Array<{ message: string; label: string }> = []; - for (const message of queuedMessages.steering) { - steeringMessages.push({ message, label: "Steer" }); - } + const steeringMessages = [...queuedMessages.steering]; for (const entry of this.ctx.compactionQueuedMessages as CompactionQueuedMessage[]) { - if (entry.mode === "steer") { - steeringMessages.push({ message: entry.text, label: "Steer" }); - } + if (entry.mode === "steer") steeringMessages.push(entry.text); } - const followUpMessages: Array<{ message: string; label: string }> = []; - for (const message of queuedMessages.followUp) { - followUpMessages.push({ message, label: "Follow-up" }); - } + const followUpMessages = [...queuedMessages.followUp]; for (const entry of this.ctx.compactionQueuedMessages as CompactionQueuedMessage[]) { - if (entry.mode === "followUp") { - followUpMessages.push({ message: entry.text, label: "Follow-up" }); - } + if (entry.mode === "followUp") followUpMessages.push(entry.text); } - const allMessages = [...steeringMessages, ...followUpMessages]; - if (allMessages.length > 0) { + const groups = [ + { label: "Steering", messages: steeringMessages }, + { label: "After yield", messages: followUpMessages }, + ].filter(group => group.messages.length > 0); + if (groups.length > 0) { this.ctx.pendingMessagesContainer.addChild(new Spacer(1)); - for (const entry of allMessages) { - const queuedText = theme.fg("dim", `${entry.label}: ${entry.message}`); - this.ctx.pendingMessagesContainer.addChild(new TruncatedText(queuedText, 1, 0)); + for (const group of groups) { + const heading = theme.fg("muted", `${group.label}${theme.sep.dot}${group.messages.length}`); + this.ctx.pendingMessagesContainer.addChild(new TruncatedText(heading, 1, 0)); + for (let index = 0; index < group.messages.length; index++) { + const message = replaceTabs(group.messages[index] ?? "").replace(/\r?\n/g, " ↵ "); + const queuedText = theme.fg("dim", ` ${index + 1}. ${message}`); + this.ctx.pendingMessagesContainer.addChild(new TruncatedText(queuedText, 1, 0)); + } } const dequeueKey = this.ctx.keybindings.getDisplayString("app.message.dequeue") || "Alt+Up"; - const hintText = theme.fg("dim", `${theme.tree.hook} ${dequeueKey} to edit`); + const hintText = theme.fg("dim", ` ${theme.tree.hook} ${dequeueKey} to edit`); this.ctx.pendingMessagesContainer.addChild(new TruncatedText(hintText, 1, 0)); } } diff --git a/packages/coding-agent/src/slash-commands/builtin-registry.ts b/packages/coding-agent/src/slash-commands/builtin-registry.ts index 776fe7f13..e5a5711b9 100644 --- a/packages/coding-agent/src/slash-commands/builtin-registry.ts +++ b/packages/coding-agent/src/slash-commands/builtin-registry.ts @@ -326,6 +326,15 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray = [ if (prompt) return { prompt }; }, }, + { + name: "queue", + description: "Queue a message for after the agent yields", + inlineHint: "", + allowArgs: true, + handleTui: async (command, runtime) => { + await runtime.ctx.handleQueueCommand(command.args); + }, + }, { name: "model", aliases: ["models"], diff --git a/packages/coding-agent/test/input-controller-skill-queue.test.ts b/packages/coding-agent/test/input-controller-skill-queue.test.ts index 6e4bfc07b..9c0beb7e9 100644 --- a/packages/coding-agent/test/input-controller-skill-queue.test.ts +++ b/packages/coding-agent/test/input-controller-skill-queue.test.ts @@ -698,8 +698,33 @@ describe("UiHelpers / InputController against derived queued custom display", () const uiHelpers = new UiHelpers(ctx); uiHelpers.updatePendingMessagesDisplay(); - const rendered = pendingMessagesContainer.render(120).join("\n"); - expect(rendered).toMatch(/Steer: \/skill:test-skill arg1 arg2/); + const rendered = Bun.stripANSI(pendingMessagesContainer.render(120).join("\n")); + expect(rendered).toContain("Steering · 1"); + expect(rendered).toContain("1. /skill:test-skill arg1 arg2"); + expect(rendered).not.toContain("Steer:"); + }); + + it("groups yield follow-ups under one heading", async () => { + fixture = await createRealSession(); + const { session } = fixture; + for (const text of ["inspect types", "run tests", "summarize"]) { + session.agent.followUp({ + role: "user", + content: text, + attribution: "user", + timestamp: Date.now(), + }); + } + + const { ctx, pendingMessagesContainer } = createStubInteractiveModeContextForUiHelpers(session); + new UiHelpers(ctx).updatePendingMessagesDisplay(); + + const rendered = Bun.stripANSI(pendingMessagesContainer.render(120).join("\n")); + expect(rendered).toContain("After yield · 3"); + expect(rendered).toContain("1. inspect types"); + expect(rendered).toContain("2. run tests"); + expect(rendered).toContain("3. summarize"); + expect(rendered).not.toContain("Follow-up:"); }); it("restores the compact slash form into the editor and clears the queue", async () => { diff --git a/packages/coding-agent/test/input-controller-slash-history.test.ts b/packages/coding-agent/test/input-controller-slash-history.test.ts index 82f63edb3..9217e5aac 100644 --- a/packages/coding-agent/test/input-controller-slash-history.test.ts +++ b/packages/coding-agent/test/input-controller-slash-history.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it, vi } from "bun:test"; +import type { ImageContent } from "@oh-my-pi/pi-ai"; import { InputController } from "@oh-my-pi/pi-coding-agent/modes/controllers/input-controller"; +import { isQueuedMessageList, splitQueuedMessages } from "@oh-my-pi/pi-coding-agent/modes/queue-input"; import type { InteractiveModeContext } from "@oh-my-pi/pi-coding-agent/modes/types"; // Drives the real editor submit handler through the builtin slash dispatch @@ -8,9 +10,12 @@ import type { InteractiveModeContext } from "@oh-my-pi/pi-coding-agent/modes/typ // executeBuiltinSlashCommand and the controller returned before any // addToHistory call. The fix centralizes recording after dispatch, with a // secret filter (shouldSkipHistory) for credential-bearing commands. -function makeCtx() { +function makeCtx(isStreaming = false) { const addToHistory = vi.fn(); const handleMCPCommand = vi.fn(async () => {}); + const followUp = vi.fn(async (_text: string, _images?: ImageContent[]) => {}); + const steer = vi.fn(async (_text: string, _images?: ImageContent[]) => {}); + const onInputCallback = vi.fn(); let text = ""; const editor = { onSubmit: undefined as undefined | ((t: string) => Promise), @@ -19,34 +24,64 @@ function makeCtx() { text = t; }, addToHistory, - pendingImages: [] as unknown[], - pendingImageLinks: [] as unknown[], + pendingImages: [] as ImageContent[], + pendingImageLinks: [] as (string | undefined)[], + imageLinks: undefined as (string | undefined)[] | undefined, clearDraft(historyText?: string) { if (historyText !== undefined) addToHistory(historyText); text = ""; + this.imageLinks = undefined; + this.pendingImages = []; + this.pendingImageLinks = []; }, }; const ctx = { editor, session: { - isStreaming: false, + isStreaming, isCompacting: false, queuedMessageCount: 0, extensionRunner: undefined, + followUp, + steer, }, focusedAgentId: undefined, collabGuest: undefined, handleHotkeysCommand: vi.fn(), handleMCPCommand, showStatus: vi.fn(), + onInputCallback, + startPendingSubmission: (input: { + text: string; + images?: ImageContent[]; + imageLinks?: (string | undefined)[]; + customType?: string; + display?: boolean; + streamingBehavior?: "steer" | "followUp"; + }) => ({ ...input, cancelled: false, started: false }), ui: { requestRender: vi.fn() }, + compactionQueuedMessages: [], + withLocalSubmission: async (_text: string, fn: () => Promise) => fn(), + updatePendingMessagesDisplay: vi.fn(), + showWarning: vi.fn(), + showError: vi.fn(), } as unknown as InteractiveModeContext; - return { ctx, editor, addToHistory, handleMCPCommand }; + return { + ctx, + editor, + addToHistory, + followUp, + steer, + onInputCallback, + handleMCPCommand, + showStatus: ctx.showStatus, + }; } function controllerFor(ctx: InteractiveModeContext) { const controller = new InputController(ctx); controller.setupEditorSubmitHandler(); + ctx.handleQueueCommand = message => controller.handleQueueCommand(message); return controller; } @@ -81,4 +116,74 @@ describe("input controller — slash command history (#3148)", () => { // ...but the secret-bearing text is kept out of recallable history. expect(addToHistory).not.toHaveBeenCalled(); }); + + it("routes /queue through the yield-only follow-up queue while streaming", async () => { + const { ctx, editor, addToHistory, followUp, showStatus } = makeCtx(true); + controllerFor(ctx); + editor.setText("/queue inspect the final result"); + + await editor.onSubmit?.("/queue inspect the final result"); + + expect(followUp).toHaveBeenCalledWith("inspect the final result", undefined); + expect(addToHistory).toHaveBeenCalledWith("/queue inspect the final result"); + expect(showStatus).toHaveBeenCalledWith("Queued message for when the agent yields"); + }); + + it("starts the first queued item immediately when the session is idle", async () => { + const { ctx, editor, followUp, steer, onInputCallback, showStatus } = makeCtx(); + controllerFor(ctx); + const input = "=>\n1. inspect types\n2. run focused tests\n3. summarize failures"; + editor.setText(input); + + await editor.onSubmit?.(input); + + expect(onInputCallback).toHaveBeenCalledWith( + expect.objectContaining({ text: "inspect types", streamingBehavior: "followUp" }), + ); + expect(steer).not.toHaveBeenCalled(); + expect(followUp.mock.calls.map(call => call[0])).toEqual(["run focused tests", "summarize failures"]); + expect(showStatus).toHaveBeenCalledWith("Sent first message; queued 2 for later yields"); + }); + + it("queues an enumerated shorthand prompt as separate ordered follow-ups", async () => { + const { ctx, editor, addToHistory, followUp, showStatus } = makeCtx(true); + controllerFor(ctx); + const input = "=>\n1. inspect types\n2. run focused tests\n3. summarize failures"; + editor.setText(input); + + await editor.onSubmit?.(input); + + expect(followUp.mock.calls.map(call => call[0])).toEqual([ + "inspect types", + "run focused tests", + "summarize failures", + ]); + expect(addToHistory).toHaveBeenCalledWith(input); + expect(showStatus).toHaveBeenCalledWith("Queued 3 messages for when the agent yields"); + }); +}); + +describe("yield queue list parsing", () => { + it("recognizes numeric, Roman, and alphabetic sequences", () => { + const expected = ["first", "second", "third"]; + for (const input of [ + "1. first\n2. second\n3. third", + "I. first\nII. second\nIII. third", + "i. first\nii. second\niii. third", + "A. first\nB. second\nC. third", + "a) first\nb) second\nc) third", + ]) { + expect(splitQueuedMessages(input)).toEqual(expected); + } + }); + + it("keeps continuation lines together and rejects non-sequential markers", () => { + expect(splitQueuedMessages("1. first line\n more detail\n2. second")).toEqual([ + "first line\n more detail", + "second", + ]); + expect(splitQueuedMessages("1. first\n3. third")).toEqual(["1. first\n3. third"]); + expect(isQueuedMessageList("1. first\n2. second\n3. third\n4.")).toBe(true); + expect(splitQueuedMessages("1. first\n2. second\n3. third\n4.")).toEqual(["first", "second", "third"]); + }); }); diff --git a/packages/coding-agent/test/selector-settings-side-effects.test.ts b/packages/coding-agent/test/selector-settings-side-effects.test.ts index 547b46623..a44df7f35 100644 --- a/packages/coding-agent/test/selector-settings-side-effects.test.ts +++ b/packages/coding-agent/test/selector-settings-side-effects.test.ts @@ -128,7 +128,7 @@ describe("selector setting side effects", () => { expect(showError).not.toHaveBeenCalled(); expect(settings.get("retry.fallbackChains")).toEqual({ default: ["test/retry-fallback-model"] }); - expect(showStatus).toHaveBeenCalledWith("Default fallback model: test/retry-fallback-model"); + expect(showStatus).toHaveBeenCalledWith("Default fallbacks: test/retry-fallback-model"); } finally { hub.dispose(); } diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index d7700b4be..131733835 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added support for width-changing editor text decorators on standalone presentation lines, with decorated output safely truncated to the available content width. + ## [16.4.5] - 2026-07-11 ### Added diff --git a/packages/tui/src/components/editor.ts b/packages/tui/src/components/editor.ts index cd3e72ab0..7c53e13a3 100644 --- a/packages/tui/src/components/editor.ts +++ b/packages/tui/src/components/editor.ts @@ -387,9 +387,9 @@ export class Editor implements Component, Focusable { cursorOverride: string | undefined; /** Display width of the cursorOverride glyph (needed because override may contain ANSI escapes). */ cursorOverrideWidth: number | undefined; - /** Optional hook that styles displayed input text with zero-width ANSI escapes. - * MUST preserve visible width (may only add SGR codes, never glyphs). Applied per - * layout line to the user-text segments — never to the cursor glyph or inline hint. */ + /** Optional hook that decorates displayed user text after source-text layout. + * Width-changing output is allowed on lines without the cursor; it is truncated + * to the content width rather than reflowed. Cursor glyphs and inline hints are excluded. */ decorateText: ((text: string) => string) | undefined; #promptGutter: string | undefined; @@ -1001,6 +1001,13 @@ export class Editor implements Component, Focusable { if (!decorated) { displayText = this.#decorate(displayText); } + if (!hasCursor) { + displayWidth = visibleWidth(displayText); + if (displayWidth > lineContentWidth) { + displayText = truncateToWidth(displayText, lineContentWidth); + displayWidth = visibleWidth(displayText); + } + } const linePad = padding(Math.max(0, lineContentWidth - displayWidth)); From 666327608cfc27cb982513efdb2b23cf84f3d72c Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 00:00:35 +0200 Subject: [PATCH 12/80] feat(coding-agent): improved model search ranking by match quality - Prioritized fuzzy match quality over the most-recently-used model order when filtering the model browser. - Added bucketing to match scores to ensure stable sorting when match quality is identical. - Added unit tests to verify that exact query matches take precedence over the MRU model. --- packages/coding-agent/CHANGELOG.md | 1 + .../src/modes/components/model-browser.ts | 17 +++-- .../coding-agent/test/model-browser.test.ts | 64 +++++++++++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 packages/coding-agent/test/model-browser.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 9b021d8d6..418c5d3c4 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -21,6 +21,7 @@ - Fixed `/btw` side-channel turns on Codex models such as `gpt-5.6-luna` by preserving the session websocket preference instead of forcing SSE, and made Esc dismiss the active `/btw` panel before interrupting loop/maintenance work. ([#5213](https://github.com/can1357/oh-my-pi/issues/5213)) - Fixed the Model Hub role-assignment strip hiding the selected chip once the row overflowed; the strip now scrolls horizontally, truncating passed chips behind a leading ellipsis so the selection (plus one chip of lookahead) stays visible. - Fixed mouse hover and clicks in the /models Roles view landing one row above the pointer (the row mapping subtracted the status row twice). +- Fixed model search keeping the most-recently-used model on top of the results: match quality now ranks first (an exact `gpt-5.5` beats the active `gpt-5.6-sol`), with MRU order only breaking ties between equally good matches. ## [16.4.5] - 2026-07-11 diff --git a/packages/coding-agent/src/modes/components/model-browser.ts b/packages/coding-agent/src/modes/components/model-browser.ts index 2634a4752..8eec17093 100644 --- a/packages/coding-agent/src/modes/components/model-browser.ts +++ b/packages/coding-agent/src/modes/components/model-browser.ts @@ -13,7 +13,7 @@ import { buildModel } from "@oh-my-pi/pi-catalog/build"; import { modelsAreEqual } from "@oh-my-pi/pi-catalog/models"; import { type Component, - fuzzyFilter, + fuzzyRank, Input, matchesKey, ScrollView, @@ -432,10 +432,19 @@ export class ModelBrowser implements Component { if (query.trim()) { // Match against the displayed "provider/id" string so the user can // type what they see: bare names, provider prefixes, or scoped - // queries all flow through the same fuzzy matcher. Skip role rank - // so a weakly matching default doesn't trump a stronger match. - const matches = fuzzyFilter(this.#baseItems, query, ({ provider, id }) => `${provider}/${id}`); + // queries all flow through the same fuzzy matcher. + const ranked = fuzzyRank(this.#baseItems, query, ({ provider, id }) => `${provider}/${id}`); + const matches = ranked.map(result => result.item); + // Match quality is the primary key while searching: an exact + // "gpt-5.5" must beat the MRU (or role-assigned) "gpt-5.6", so + // role rank is skipped and MRU only breaks ties. Scores are + // bucketed so sub-point position noise (provider-name length) + // can't split equally good matches; within a bucket the stable + // sort keeps sortModelItems' MRU/version order. sortModelItems(matches, { roles: this.#roles, mruOrder: this.#mruOrder, skipRoleRank: true }); + const buckets = new Map(); + for (const result of ranked) buckets.set(result.item, Math.round(result.score / 10)); + matches.sort((a, b) => (buckets.get(a) ?? 0) - (buckets.get(b) ?? 0)); items = matches; } else { items = this.#baseItems; diff --git a/packages/coding-agent/test/model-browser.test.ts b/packages/coding-agent/test/model-browser.test.ts new file mode 100644 index 000000000..a8c34e8dd --- /dev/null +++ b/packages/coding-agent/test/model-browser.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from "bun:test"; +import type { Model } from "@oh-my-pi/pi-ai"; +import { buildModel } from "@oh-my-pi/pi-catalog/build"; +import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import { + buildBrowserItems, + ModelBrowser, + sortModelItems, +} from "@oh-my-pi/pi-coding-agent/modes/components/model-browser"; + +function makeModel(provider: string, id: string): Model { + return buildModel({ + id, + name: id, + api: "ollama-chat", + provider, + baseUrl: "https://example.com", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 1024, + }); +} + +/** Browser preloaded with `models`, MRU-sorted like the hub does on sync. */ +function makeBrowser(models: Model[], mruOrder: string[]): ModelBrowser { + const browser = new ModelBrowser(Settings.isolated({})); + const items = buildBrowserItems(models); + sortModelItems(items, { mruOrder }); + browser.setMruOrder(mruOrder); + browser.setItems(items); + return browser; +} + +describe("ModelBrowser search ranking", () => { + test("an exact query match outranks the MRU model", () => { + // Regression: with gpt-5.6-sol as the active (MRU) model, typing + // "gpt-5.5" must select gpt-5.5, not keep the MRU pinned on top. + const browser = makeBrowser( + [ + makeModel("openai-codex", "gpt-5.6-sol"), + makeModel("openai-codex", "gpt-5.6-luna"), + makeModel("openai-codex", "gpt-5.5"), + makeModel("openai-codex", "gpt-5.4"), + ], + ["openai-codex/gpt-5.6-sol", "openai-codex/gpt-5.6-luna"], + ); + + browser.setQuery("gpt-5.5"); + + expect(browser.getSelected()?.selector).toBe("openai-codex/gpt-5.5"); + }); + + test("MRU breaks ties between equally good matches", () => { + // Same model id under two providers: match quality is identical, so + // the recently used provider must win over alphabetical order. + const browser = makeBrowser([makeModel("g0i", "gpt-5.5"), makeModel("zenmux", "gpt-5.5")], ["zenmux/gpt-5.5"]); + + browser.setQuery("gpt-5.5"); + + expect(browser.getSelected()?.selector).toBe("zenmux/gpt-5.5"); + }); +}); From a9adf20af26cd444e4ae1940c75e0912b66c625c Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 00:16:09 +0200 Subject: [PATCH 13/80] fix: prevented false overlap errors by deduplicating identical edits - Enabled deduplication of byte-identical edits in `apply_edits` to prevent false overlap errors. - Added logic in `ast_edit_blocking` to filter out duplicate rewrite matches before staging. - Updated `apply_edits` to collapse multiple identical replacements into a single deterministic operation. - Added regression tests for both `apply_edits` and `ast_edit_blocking` to ensure identical matches resolve cleanly. --- crates/pi-ast/src/ops.rs | 26 +++++++++++++- crates/pi-natives/src/ast.rs | 65 ++++++++++++++++++++++++++++++++++- packages/natives/CHANGELOG.md | 4 +++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/crates/pi-ast/src/ops.rs b/crates/pi-ast/src/ops.rs index 26935fb1e..8592bb36f 100644 --- a/crates/pi-ast/src/ops.rs +++ b/crates/pi-ast/src/ops.rs @@ -187,7 +187,20 @@ pub fn rewrite_source( pub fn apply_edits(content: &str, edits: &[Edit]) -> Result { let mut sorted: Vec<&Edit> = edits.iter().collect(); - sorted.sort_by_key(|edit| edit.position); + sorted.sort_by(|a, b| { + a.position + .cmp(&b.position) + .then(a.deleted_length.cmp(&b.deleted_length)) + .then(a.inserted_text.cmp(&b.inserted_text)) + }); + // Byte-identical edits (same span, same replacement) are one deterministic + // edit: multiple patterns matching the same node collapse instead of + // tripping the overlap check. Only divergent overlaps are ambiguous. + sorted.dedup_by(|a, b| { + a.position == b.position + && a.deleted_length == b.deleted_length + && a.inserted_text == b.inserted_text + }); let mut prev_end = 0usize; for edit in &sorted { if edit.position < prev_end { @@ -299,4 +312,15 @@ mod tests { ]; assert!(apply_edits(source, &edits).is_err()); } + + #[test] + fn apply_edits_dedupes_identical_edits() { + let source = "abcdef"; + let edits = vec![ + Edit:: { position: 1, deleted_length: 3, inserted_text: b"x".to_vec() }, + Edit:: { position: 1, deleted_length: 3, inserted_text: b"x".to_vec() }, + ]; + let output = apply_edits(source, &edits).expect("identical edits should collapse to one"); + assert_eq!(output, "axef"); + } } diff --git a/crates/pi-natives/src/ast.rs b/crates/pi-natives/src/ast.rs index d7ed96af6..4eec13320 100644 --- a/crates/pi-natives/src/ast.rs +++ b/crates/pi-natives/src/ast.rs @@ -1065,12 +1065,23 @@ fn ast_edit_blocking( 'patterns: for (_pattern, rewrite, compiled) in &compiled_rules { for matched in ast.root().find_all(compiled.clone()) { ct.heartbeat()?; + let edit = matched.replace_by(rewrite.as_str()); + // Multiple rules matching the same node with the same output are one + // deterministic edit; list and count it once instead of staging a + // duplicate that trips the apply-time overlap check. + let duplicate = file_changes.iter().any(|entry: &PendingFileChange| { + entry.edit.position == edit.position + && entry.edit.deleted_length == edit.deleted_length + && entry.edit.inserted_text == edit.inserted_text + }); + if duplicate { + continue; + } if changes.len() + file_changes.len() >= max_replacements as usize { limit_reached = true; reached_max_replacements = true; break 'patterns; } - let edit = matched.replace_by(rewrite.as_str()); let range = matched.range(); let start = matched.start_pos(); let end = matched.end_pos(); @@ -1334,6 +1345,58 @@ mod tests { assert!(apply_edits(source, &edits).is_err()); } + #[test] + fn dedupes_byte_identical_edits() { + let source = "abcdef"; + let edits = vec![ + Edit:: { position: 1, deleted_length: 3, inserted_text: b"x".to_vec() }, + Edit:: { position: 1, deleted_length: 3, inserted_text: b"x".to_vec() }, + ]; + let output = apply_edits(source, &edits).expect("identical edits should collapse to one"); + assert_eq!(output, "axef"); + } + + #[test] + fn ast_edit_dedupes_identical_matches_across_rules() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be after UNIX_EPOCH") + .as_nanos(); + let root = std::env::temp_dir().join(format!("pi-ast-dedupe-{unique}")); + fs::create_dir_all(&root).expect("temp dedupe dir should be created"); + let tree = TempTree { root }; + let file_path = tree.root.join("a.ts"); + fs::write(&file_path, "const b = foo(bar);\n").expect("temp file a.ts should be written"); + + // Both rules match the same call node and produce the byte-identical + // replacement; the deterministic edit must apply once, not error as an + // ambiguous overlap. + let mut rewrites = HashMap::new(); + rewrites.insert("foo($X)".to_string(), "qux($X)".to_string()); + rewrites.insert("foo(bar)".to_string(), "qux(bar)".to_string()); + + let result = ast_edit_blocking( + task::CancelToken::default(), + Some(rewrites), + Some("ts".to_string()), + Some(tree.root.to_string_lossy().into_owned()), + None, + None, + None, + Some(false), + None, + None, + None, + ) + .expect("identical duplicate matches should apply cleanly"); + + assert_eq!(result.total_replacements, 1, "duplicate match must be counted once"); + assert_eq!( + fs::read_to_string(&file_path).expect("a.ts should be readable"), + "const b = qux(bar);\n", + ); + } + fn make_apply_failure_tree() -> TempTree { let unique = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index 2852d393e..dc6b448c6 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed `ast_edit` rejecting byte-identical duplicate replacements as "Overlapping replacements detected": multiple rewrite ops matching the same node with the same output now collapse into one deterministic edit (deduped in both the preview listing/counts and the apply pass), so only genuinely divergent overlaps error. + ### Added - Added an in-process `readlink` shell builtin (vendored from uutils coreutils 0.8.0), supporting `-f`/`-e`/`-m` canonicalization, `-n`/`-z` delimiters, and `-v`/`-q`/`-s` verbosity, with path operands resolved against the shell working directory. From a3117c2842a7de5161496a94a3adda1f7922b2d1 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 01:06:17 +0200 Subject: [PATCH 14/80] feat(agent): migrated async-drain utility to shared package for reuse - Relocated `AsyncDrain` class from `coding-agent` to `utils` package. - Updated `HistoryStorage` to import `AsyncDrain` from shared utilities. - Centralized the utility to allow reuse across the codebase. --- packages/coding-agent/CHANGELOG.md | 5 +++ .../src/session/history-storage.ts | 35 +--------------- packages/utils/CHANGELOG.md | 4 ++ packages/utils/src/async.ts | 41 +++++++++++++++++++ 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 418c5d3c4..2c1bfc7f1 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -9,14 +9,19 @@ - Added fallback-chain editing to the /models Roles view: each role's `retry.fallbackChains` entries render as indented rows beneath it, `f` picks a fallback model to append, Enter on an entry replaces it, `x`/backspace removes it, and `[`/`]` (or shift+↑/↓) reorder the chain. - Added model-keyed fallback management to the /models Roles view: model and `provider/*` chains render as a separate section below the roles (divider + "+ New fallback…" row for creating one by picking the protected model, then keying it by model or provider), with the same replace/remove/reorder editing as role chains; the model strip gains `fallbacks:` and `fallbacks:/*` chips as shortcuts. - Added `/queue ` plus `->` / `=>` composer shorthand for follow-up messages that wait until the agent yields. The shorthand opens a dim `Queueing` header and splits sequential numeric, Roman-numeral, or alphabetic lists into separately highlighted queue entries. +- Added per-model TPS/TTFT tracking: every completed assistant turn folds its timing into recency-weighted aggregates in `~/.omp/agent.db`, and the /models browser shows measured speed — a right-aligned `118t/s` column on wide terminals (plus TTFT, e.g. `0.9s 118t/s`, when wider) and `~118t/s · 0.9s ttft` facts in the selection detail line — with no dependency on the `omp stats` session scan. ### Changed - Retain completed and abandoned tasks in session history for improved context on resume - Changed the Model Hub `retry-fallback` strip chip to append the model to the default fallback chain instead of prepending it, matching the chain-building order of the Roles view (already-registered models are a no-op). +- Changed per-model perf recording (`recordModelPerf`) to be deferred like prompt history: samples are batched and written to `agent.db` in one transaction ~100ms later, keeping SQLite writes off the turn-completion hot path. ### Fixed +- Fixed uncontrollable mouse-wheel scrolling in the /models hub: the wheel moved the selection (one step per wheel event, so a single trackpad flick skipped many rows) and wrapped from the bottom back to the top. Wheel scrolling now pans the list viewport only, clamps at the ends, and leaves the selection where it is; keyboard navigation still scrolls the selection into view. Likewise, the wheel over the provider sidebar no longer switches the active scope (or triggers provider refreshes) — it just scrolls the sidebar. +- Fixed TPS being inflated several-fold when a provider hides reasoning tokens until late in the stream (e.g. `google/gemini-3.5` vs `google-vertex/gemini-3.5` reporting 648 vs 186 TPS for identical durations): `omp bench`, the per-turn usage row, and the /models perf aggregates now measure tokens/sec over the total request duration instead of the post-TTFT decode window, matching `omp stats`. Stored perf aggregates are purged and re-backfilled from stats history with the corrected math on first launch. +- Fixed the model-perf stats.db backfill freezing the TUI (~30s on multi-million-row stats databases) when /models triggered it: the import is now fire-and-forget, walks the newest rows in small chunks with event-loop yields between them, and is bounded to 90 days / 256 newest samples per model — beyond either bound the recency decay would erase the contribution anyway. - Fixed compiled release binaries bundling `fastembed` and baking the build-machine `@anush008/tokenizers` path; native runtime dependencies now stay external for every compiled build path so Mnemopi resolves its on-demand install instead. ([#5195](https://github.com/can1357/oh-my-pi/issues/5195)) - Fixed `/btw` side-channel turns on Codex models such as `gpt-5.6-luna` by preserving the session websocket preference instead of forcing SSE, and made Esc dismiss the active `/btw` panel before interrupting loop/maintenance work. ([#5213](https://github.com/can1357/oh-my-pi/issues/5213)) - Fixed the Model Hub role-assignment strip hiding the selected chip once the row overflowed; the strip now scrolls horizontally, truncating passed chips behind a leading ellipsis so the selection (plus one chip of lookahead) stays visible. diff --git a/packages/coding-agent/src/session/history-storage.ts b/packages/coding-agent/src/session/history-storage.ts index 7ef5907a5..06635de90 100644 --- a/packages/coding-agent/src/session/history-storage.ts +++ b/packages/coding-agent/src/session/history-storage.ts @@ -1,7 +1,7 @@ import { Database, type Statement } from "bun:sqlite"; import * as fs from "node:fs"; import * as path from "node:path"; -import { getHistoryDbPath, logger } from "@oh-my-pi/pi-utils"; +import { AsyncDrain, getHistoryDbPath, logger } from "@oh-my-pi/pi-utils"; export interface HistoryEntry { id: number; @@ -28,39 +28,6 @@ function escapeLikePattern(text: string): string { return text.replace(/[\\%_]/g, "\\$&"); } -class AsyncDrain { - #queue?: T[]; - #promise = Promise.resolve(); - - constructor(readonly delayMs: number = 0) {} - - push(value: T, hnd: (values: T[]) => Promise | void): Promise { - let queue = this.#queue; - if (!queue) { - this.#queue = queue = []; - const { promise, resolve, reject } = Promise.withResolvers(); - const exec = (): void => { - try { - if (this.#queue === queue) { - this.#queue = undefined; - } - resolve(hnd(queue!)); - } catch (error) { - reject(error); - } - }; - if (this.delayMs > 0) { - setTimeout(exec, this.delayMs); - } else { - queueMicrotask(exec); - } - this.#promise = promise; - } - queue.push(value); - return this.#promise; - } -} - export class HistoryStorage { #db: Database; static #instance?: HistoryStorage; diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.md index 5e14d664b..c79689cfd 100644 --- a/packages/utils/CHANGELOG.md +++ b/packages/utils/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added `AsyncDrain`, the deferred write-batching helper previously private to the coding-agent's prompt-history storage; now shared with model-perf recording. + ## [16.4.2] - 2026-07-10 ### Added diff --git a/packages/utils/src/async.ts b/packages/utils/src/async.ts index b13d8d589..ee50dde64 100644 --- a/packages/utils/src/async.ts +++ b/packages/utils/src/async.ts @@ -48,3 +48,44 @@ export function withTimeout(promise: Promise, ms: number, message: string, return wrapped; } + +/** + * Coalesces rapid-fire writes into one deferred batch. `push` queues a value + * and returns a promise for the batch flush; the first push of a batch arms a + * timer (`delayMs`, or a microtask at 0), and every push before it fires joins + * the same batch and shares the same promise. Used to keep hot paths off + * synchronous storage (prompt history, model perf). + */ +export class AsyncDrain { + #queue?: T[]; + #promise = Promise.resolve(); + + constructor(readonly delayMs: number = 0) {} + + /** Queue `value`; `hnd` receives the whole batch when the window closes. */ + push(value: T, hnd: (values: T[]) => Promise | void): Promise { + let queue = this.#queue; + if (!queue) { + this.#queue = queue = []; + const { promise, resolve, reject } = Promise.withResolvers(); + const exec = (): void => { + try { + if (this.#queue === queue) { + this.#queue = undefined; + } + resolve(hnd(queue!)); + } catch (error) { + reject(error); + } + }; + if (this.delayMs > 0) { + setTimeout(exec, this.delayMs); + } else { + queueMicrotask(exec); + } + this.#promise = promise; + } + queue.push(value); + return this.#promise; + } +} From c4fa0ebaae7c202c3ae4e173898c7a7b13e0ed77 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 01:06:19 +0200 Subject: [PATCH 15/80] feat(storage): implemented persistent model performance tracking and migration - Added `model_perf` table to store persistent, recency-weighted TPS and TTFT statistics per model. - Introduced `AgentStorage.recordModelPerf` to asynchronously capture and aggregate turn timing metrics. - Implemented a background backfill mechanism to migrate historical request data from `stats.db` into new performance aggregates. - Updated `AgentSession` to record performance metrics upon turn completion. - Bumped `SCHEMA_VERSION` to 6 and cleared existing performance data to account for improved duration-based TPS calculation. --- .../coding-agent/src/session/agent-session.ts | 10 + .../coding-agent/src/session/agent-storage.ts | 331 +++++++++++++++++- 2 files changed, 339 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index cb26281e3..8c46ad4e0 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -3840,6 +3840,16 @@ export class AgentSession { if (event.message.role === "assistant") { this.#lastAssistantMessage = event.message; const assistantMsg = event.message as AssistantMessage; + // Fold this turn's timing into per-model perf aggregates (drives the + // /models TPS/TTFT display). Errored turns measure nothing; aborted + // turns with reported usage are still valid throughput samples. + if (assistantMsg.stopReason !== "error" && assistantMsg.duration !== undefined) { + this.settings.getStorage()?.recordModelPerf(`${assistantMsg.provider}/${assistantMsg.model}`, { + outputTokens: assistantMsg.usage.output, + durationMs: assistantMsg.duration, + ttftMs: assistantMsg.ttft, + }); + } if ( assistantMsg.disabledFeatures?.includes("priority") && this.#serviceTierByFamily.anthropic === "priority" diff --git a/packages/coding-agent/src/session/agent-storage.ts b/packages/coding-agent/src/session/agent-storage.ts index abb9c7eac..d8c6dae8d 100644 --- a/packages/coding-agent/src/session/agent-storage.ts +++ b/packages/coding-agent/src/session/agent-storage.ts @@ -8,7 +8,7 @@ import { SqliteAuthCredentialStore, type StoredAuthCredential, } from "@oh-my-pi/pi-ai"; -import { getAgentDbPath, isRecord, logger } from "@oh-my-pi/pi-utils"; +import { AsyncDrain, getAgentDbPath, getStatsDbPath, isRecord, logger } from "@oh-my-pi/pi-utils"; import type { RawSettings as Settings } from "../config/settings"; /** Row shape for settings table queries */ @@ -23,8 +23,100 @@ type ModelUsageRow = { last_used_at: number; }; +/** Row shape for model_perf table queries */ +type ModelPerfRow = { + model_key: string; + samples: number; + output_tokens: number; + gen_ms: number; + ttft_samples: number; + ttft_ms: number; +}; + +/** Row shape read from an `omp stats` messages table during backfill. */ +type StatsMessageRow = { + rowid: number; + timestamp: number; + provider: string; + model: string; + output_tokens: number; + duration: number; + ttft: number | null; +}; + +/** Per-model running sums accumulated during a backfill walk. */ +type PerfAccum = { + samples: number; + outputTokens: number; + genMs: number; + ttftSamples: number; + ttftMs: number; +}; + +/** One completed request's timing, folded into the per-model aggregates. */ +export interface ModelPerfSample { + /** Output tokens the provider reported for the turn. */ + outputTokens: number; + /** Total request duration in milliseconds. */ + durationMs: number; + /** Time to first token in milliseconds; omit when the provider did not report one. */ + ttftMs?: number; +} + +/** Validated, insert-ready model_perf sample (see {@link normalizeModelPerfSample}). */ +type ModelPerfInsert = { + modelKey: string; + outputTokens: number; + durationMs: number; + ttftSamples: 0 | 1; + ttftMs: number; +}; + +/** Recency-weighted per-model performance averages. */ +export interface ModelPerfStats { + /** Decayed sample count backing the averages. */ + samples: number; + /** Average output tokens/sec over the total request duration. */ + tps: number; + /** Average time-to-first-token in milliseconds; null when no sample reported one. */ + ttftMs: number | null; +} + +/** + * Decay threshold for model_perf running sums: once a model accumulates this + * many samples, each new sample first halves every aggregate, turning the + * plain average into a recency-weighted one (provider speeds drift over time). + */ +const MODEL_PERF_DECAY_AT = 256; +/** meta-table marker set once historical stats.db rows have been imported into model_perf. */ +const MODEL_PERF_BACKFILL_KEY = "model_perf_backfill"; +/** Batch window for deferred model_perf writes; matches prompt-history's drain cadence. */ +const MODEL_PERF_FLUSH_DELAY_MS = 100; +/** Backfill ignores stats.db history older than this; decay makes stale provider speeds worthless anyway. */ +const MODEL_PERF_BACKFILL_MAX_AGE_MS = 90 * 86_400_000; +/** Rows fetched per synchronous backfill chunk — keeps per-chunk event-loop blocking under ~20ms even on cold I/O. */ +const MODEL_PERF_BACKFILL_CHUNK = 2048; +/** Hard ceiling on rows scanned per backfill run, whatever the age cutoff admits — bounds total CPU on very high-volume databases (models only seen earlier than the newest N measurable rows get no backfill). */ +const MODEL_PERF_BACKFILL_MAX_ROWS = 250_000; + +/** + * Validates one request timing and shapes it for the model_perf upsert. + * Returns null for unmeasurable samples (no tokens, no duration). Out-of-range + * TTFT (>= duration) is bogus latency data; the sample still measures throughput. + */ +function normalizeModelPerfSample(modelKey: string, sample: ModelPerfSample): ModelPerfInsert | null { + const { outputTokens, durationMs } = sample; + if (!Number.isFinite(outputTokens) || outputTokens <= 0) return null; + if (!Number.isFinite(durationMs) || durationMs <= 0) return null; + const ttftMs = + sample.ttftMs !== undefined && Number.isFinite(sample.ttftMs) && sample.ttftMs > 0 && sample.ttftMs < durationMs + ? sample.ttftMs + : undefined; + return { modelKey, outputTokens, durationMs, ttftSamples: ttftMs !== undefined ? 1 : 0, ttftMs: ttftMs ?? 0 }; +} + /** Bump when schema changes require migration */ -const SCHEMA_VERSION = 5; +const SCHEMA_VERSION = 6; const SQLITE_NOW_EPOCH = "CAST(strftime('%s','now') AS INTEGER)"; /** Singleton instances per database path */ @@ -42,9 +134,18 @@ export class AgentStorage { #listSettingsStmt: Statement; #upsertModelUsageStmt: Statement; #listModelUsageStmt: Statement; + #upsertModelPerfStmt: Statement; + #listModelPerfStmt: Statement; #modelUsageCache: string[] | null = null; + /** Only the real user db auto-imports stats.db history; custom paths (tests, embedding) opt in explicitly. */ + #autoPerfBackfill: boolean; + /** One backfill *check* per process; the persistent gate is the meta marker. */ + #perfBackfillChecked = false; + /** Coalesces per-turn perf samples into one deferred transaction off the turn's hot path. */ + #perfDrain = new AsyncDrain(MODEL_PERF_FLUSH_DELAY_MS); private constructor(dbPath: string) { + this.#autoPerfBackfill = dbPath === getAgentDbPath(); this.#ensureDir(dbPath); try { this.#db = new Database(dbPath); @@ -72,6 +173,22 @@ export class AgentStorage { this.#listModelUsageStmt = this.#db.prepare( "SELECT model_key, last_used_at FROM model_usage ORDER BY last_used_at DESC", ); + // Recency-weighted upsert: past MODEL_PERF_DECAY_AT samples, every new + // sample first halves the aggregates so old measurements fade out. + this.#upsertModelPerfStmt = this.#db.prepare( + `INSERT INTO model_perf (model_key, samples, output_tokens, gen_ms, ttft_samples, ttft_ms, updated_at) +VALUES (?1, 1, ?2, ?3, ?4, ?5, ${SQLITE_NOW_EPOCH}) +ON CONFLICT(model_key) DO UPDATE SET + samples = (CASE WHEN model_perf.samples >= ${MODEL_PERF_DECAY_AT} THEN model_perf.samples / 2 ELSE model_perf.samples END) + 1, + output_tokens = (CASE WHEN model_perf.samples >= ${MODEL_PERF_DECAY_AT} THEN model_perf.output_tokens * 0.5 ELSE model_perf.output_tokens END) + excluded.output_tokens, + gen_ms = (CASE WHEN model_perf.samples >= ${MODEL_PERF_DECAY_AT} THEN model_perf.gen_ms * 0.5 ELSE model_perf.gen_ms END) + excluded.gen_ms, + ttft_samples = (CASE WHEN model_perf.samples >= ${MODEL_PERF_DECAY_AT} THEN model_perf.ttft_samples * 0.5 ELSE model_perf.ttft_samples END) + excluded.ttft_samples, + ttft_ms = (CASE WHEN model_perf.samples >= ${MODEL_PERF_DECAY_AT} THEN model_perf.ttft_ms * 0.5 ELSE model_perf.ttft_ms END) + excluded.ttft_ms, + updated_at = ${SQLITE_NOW_EPOCH}`, + ); + this.#listModelPerfStmt = this.#db.prepare( + "SELECT model_key, samples, output_tokens, gen_ms, ttft_samples, ttft_ms FROM model_perf", + ); } /** @@ -93,6 +210,21 @@ CREATE TABLE IF NOT EXISTS model_usage ( last_used_at INTEGER NOT NULL DEFAULT (${SQLITE_NOW_EPOCH}) ); +CREATE TABLE IF NOT EXISTS model_perf ( + model_key TEXT PRIMARY KEY, + samples REAL NOT NULL DEFAULT 0, + output_tokens REAL NOT NULL DEFAULT 0, + gen_ms REAL NOT NULL DEFAULT 0, + ttft_samples REAL NOT NULL DEFAULT 0, + ttft_ms REAL NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL DEFAULT (${SQLITE_NOW_EPOCH}) +); + +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY); `); @@ -175,6 +307,14 @@ CREATE TABLE settings ( if (fromVersion < 5) { this.#migrateSchemaV4ToV5(); } + if (fromVersion < 6) { + // v5 → v6: TPS switched from the post-TTFT decode window to total + // request duration (hidden reasoning made decode-window rates bogus). + // Purge the old aggregates and re-arm the stats.db backfill so + // history is re-imported through the corrected fold. + this.#db.run("DELETE FROM model_perf"); + this.#db.prepare("DELETE FROM meta WHERE key = ?").run(MODEL_PERF_BACKFILL_KEY); + } } #migrateSchemaV4ToV5(): void { @@ -257,6 +397,8 @@ FROM model_usage_legacy this.#listSettingsStmt.finalize(); this.#upsertModelUsageStmt.finalize(); this.#listModelUsageStmt.finalize(); + this.#upsertModelPerfStmt.finalize(); + this.#listModelPerfStmt.finalize(); // SqliteAuthCredentialStore.close() finalizes its own statements and // closes the shared #db handle — must run after our statements finalize. this.#authStore.close(); @@ -317,6 +459,191 @@ FROM model_usage_legacy } } + /** + * Folds one completed request's timing into the model's perf aggregates. + * TPS is measured over the total request duration — not the post-TTFT + * decode window, which undercounts generation time (and so inflates the + * rate) when reasoning tokens are generated before the first visible + * token. Invalid samples (no tokens, no duration) are dropped. + * + * Deferred like prompt history: samples are batched and written in one + * transaction after {@link MODEL_PERF_FLUSH_DELAY_MS}, keeping SQLite off + * the turn-completion hot path. Fire-and-forget safe — flush failures are + * logged, never thrown; await the returned promise only to observe the flush. + * @param modelKey - Model key in "provider/modelId" format + */ + recordModelPerf(modelKey: string, sample: ModelPerfSample): Promise { + const row = normalizeModelPerfSample(modelKey, sample); + if (!row) return Promise.resolve(); + return this.#perfDrain.push(row, rows => this.#flushModelPerf(rows)); + } + + #flushModelPerf(rows: ModelPerfInsert[]): void { + // Kick the one-time history import too, so aggregates populate even if + // the user never opens /models. Additive merge makes ordering with live + // samples irrelevant. + this.#kickModelPerfBackfill(); + try { + this.#db.transaction((batch: ModelPerfInsert[]) => { + for (const row of batch) this.#foldModelPerf(row); + })(rows); + } catch (error) { + logger.warn("AgentStorage failed to record model perf", { error: String(error) }); + } + } + + #foldModelPerf(row: ModelPerfInsert): void { + this.#upsertModelPerfStmt.run(row.modelKey, row.outputTokens, row.durationMs, row.ttftSamples, row.ttftMs); + } + + /** + * Returns recency-weighted TPS/TTFT averages for every model with recorded + * requests, keyed by "provider/modelId". Read by the /models browser. + * Also kicks the one-time background stats.db import; until it completes, + * models without live samples are simply absent. + */ + getModelPerf(): Map { + this.#kickModelPerfBackfill(); + const stats = new Map(); + try { + for (const row of this.#listModelPerfStmt.all() as ModelPerfRow[]) { + if (row.gen_ms <= 0 || row.output_tokens <= 0) continue; + stats.set(row.model_key, { + samples: row.samples, + tps: (row.output_tokens * 1000) / row.gen_ms, + ttftMs: row.ttft_samples > 0 ? row.ttft_ms / row.ttft_samples : null, + }); + } + } catch (error) { + logger.warn("AgentStorage failed to read model perf", { error: String(error) }); + } + return stats; + } + + /** + * One-time, non-blocking import of historical request timings from the + * `omp stats` database (`~/.omp/stats.db`) into model_perf. Fire-and-forget: + * the walk runs in bounded chunks with event-loop yields between them + * (bun:sqlite is synchronous — an unbounded scan here froze the TUI for + * ~30s on multi-million-row stats databases), and the persistent meta + * marker is only set on success so a crash or error retries next process. + * A missing stats.db leaves the marker unset so a later `omp stats` run + * still gets imported. No-op for non-default db paths. + */ + #kickModelPerfBackfill(): void { + if (!this.#autoPerfBackfill || this.#perfBackfillChecked) return; + this.#perfBackfillChecked = true; + try { + const marker = this.#db.prepare("SELECT value FROM meta WHERE key = ?").get(MODEL_PERF_BACKFILL_KEY); + if (marker) return; + const statsDbPath = getStatsDbPath(); + if (!fs.existsSync(statsDbPath)) return; + void this.backfillModelPerfFromStats(statsDbPath) + .then(imported => { + this.#db + .prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)") + .run(MODEL_PERF_BACKFILL_KEY, "complete"); + logger.info("AgentStorage imported model perf history from stats.db", { imported }); + }) + .catch(error => { + logger.warn("AgentStorage model perf backfill failed", { error: String(error) }); + }); + } catch (error) { + logger.warn("AgentStorage model perf backfill failed", { error: String(error) }); + } + } + + /** + * Imports recent measurable request rows from an `omp stats` database + * (`messages` table) into the model_perf aggregates. Walks newest-first + * over the timestamp index in {@link MODEL_PERF_BACKFILL_CHUNK}-row chunks, + * yielding to the event loop between chunks, and keeps at most + * {@link MODEL_PERF_DECAY_AT} rows per model within the + * {@link MODEL_PERF_BACKFILL_MAX_AGE_MS} window — beyond either bound the + * live decay would erase the contribution anyway. Errored turns are + * excluded; aborted turns with reported usage count, matching live capture. + * Sums land in one additive transaction at the end, so concurrent live + * samples merge correctly regardless of order. + * @param statsDbPath - Path to a stats.db file; opened read-only + * @returns Number of rows folded in + * @throws When the stats db cannot be opened or queried + */ + async backfillModelPerfFromStats(statsDbPath: string): Promise { + const statsDb = new Database(statsDbPath, { readonly: true }); + try { + statsDb.run("PRAGMA busy_timeout = 5000"); + const select = statsDb.prepare( + `SELECT rowid, timestamp, provider, model, output_tokens, duration, ttft +FROM messages +WHERE (timestamp < ?1 OR (timestamp = ?1 AND rowid < ?2)) + AND timestamp >= ?3 + AND duration > 0 AND output_tokens > 0 AND stop_reason != 'error' +ORDER BY timestamp DESC, rowid DESC +LIMIT ?4`, + ); + const cutoff = Date.now() - MODEL_PERF_BACKFILL_MAX_AGE_MS; + const sums = new Map(); + let cursorTimestamp = Number.MAX_SAFE_INTEGER; + let cursorRowid = Number.MAX_SAFE_INTEGER; + let scanned = 0; + let imported = 0; + while (scanned < MODEL_PERF_BACKFILL_MAX_ROWS) { + const chunk = Math.min(MODEL_PERF_BACKFILL_CHUNK, MODEL_PERF_BACKFILL_MAX_ROWS - scanned); + const rows = select.all(cursorTimestamp, cursorRowid, cutoff, chunk) as StatsMessageRow[]; + if (rows.length === 0) break; + scanned += rows.length; + const last = rows[rows.length - 1]; + cursorTimestamp = last.timestamp; + cursorRowid = last.rowid; + for (const row of rows) { + const key = `${row.provider}/${row.model}`; + let accum = sums.get(key); + if (accum && accum.samples >= MODEL_PERF_DECAY_AT) continue; + const normalized = normalizeModelPerfSample(key, { + outputTokens: row.output_tokens, + durationMs: row.duration, + ttftMs: row.ttft ?? undefined, + }); + if (!normalized) continue; + if (!accum) { + accum = { samples: 0, outputTokens: 0, genMs: 0, ttftSamples: 0, ttftMs: 0 }; + sums.set(key, accum); + } + accum.samples += 1; + accum.outputTokens += normalized.outputTokens; + accum.genMs += normalized.durationMs; + accum.ttftSamples += normalized.ttftSamples; + accum.ttftMs += normalized.ttftMs; + imported++; + } + if (rows.length < chunk) break; + // Yield so a chunked walk never freezes the TUI (bun:sqlite is sync). + await Bun.sleep(0); + } + if (sums.size > 0) { + const upsert = this.#db.prepare( + `INSERT INTO model_perf (model_key, samples, output_tokens, gen_ms, ttft_samples, ttft_ms, updated_at) +VALUES (?1, ?2, ?3, ?4, ?5, ?6, ${SQLITE_NOW_EPOCH}) +ON CONFLICT(model_key) DO UPDATE SET + samples = model_perf.samples + excluded.samples, + output_tokens = model_perf.output_tokens + excluded.output_tokens, + gen_ms = model_perf.gen_ms + excluded.gen_ms, + ttft_samples = model_perf.ttft_samples + excluded.ttft_samples, + ttft_ms = model_perf.ttft_ms + excluded.ttft_ms, + updated_at = ${SQLITE_NOW_EPOCH}`, + ); + this.#db.transaction(() => { + for (const [key, accum] of sums) { + upsert.run(key, accum.samples, accum.outputTokens, accum.genMs, accum.ttftSamples, accum.ttftMs); + } + })(); + } + return imported; + } finally { + statsDb.close(); + } + } + /** * Checks if any auth credentials exist in storage. * @returns True if at least one credential is stored From a0dcb8ae20ec5258d6b32d24209d2b8ab41d90a9 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 01:06:22 +0200 Subject: [PATCH 16/80] feat(ui): enabled performance monitoring and viewport navigation - Integrated performance statistics (TPS/TTFT) into ModelBrowser and ModelHub components with adaptive visibility. - Decoupled mouse wheel behavior from selection logic to enable smoother viewport panning. - Improved UI navigation by preventing unwanted wrapping in role indexes and adding viewport snapping logic. - Added comprehensive test suites to verify performance display rendering and corrected input interaction behavior. --- .../src/modes/components/advisor-config.ts | 4 +- .../src/modes/components/model-browser.ts | 130 ++++++++++++++---- .../src/modes/components/model-hub.ts | 58 +++++--- .../coding-agent/test/model-browser.test.ts | 44 +++++- packages/coding-agent/test/model-hub.test.ts | 79 +++++++++++ 5 files changed, 267 insertions(+), 48 deletions(-) diff --git a/packages/coding-agent/src/modes/components/advisor-config.ts b/packages/coding-agent/src/modes/components/advisor-config.ts index 7f7358654..6543a7aae 100644 --- a/packages/coding-agent/src/modes/components/advisor-config.ts +++ b/packages/coding-agent/src/modes/components/advisor-config.ts @@ -461,7 +461,8 @@ export class AdvisorConfigOverlayComponent implements Component { } #showModelPicker(index: number): void { - const mruOrder = this.#settings.getStorage()?.getModelUsageOrder() ?? []; + const storage = this.#settings.getStorage(); + const mruOrder = storage?.getModelUsageOrder() ?? []; let models: ReadonlyArray; if (this.#scopedModels.length > 0) { models = this.#scopedModels.map(scoped => scoped.model); @@ -477,6 +478,7 @@ export class AdvisorConfigOverlayComponent implements Component { const picker = new ModelBrowser(this.#settings, {}); picker.setMruOrder(mruOrder); + picker.setPerfStats(storage?.getModelPerf() ?? new Map()); picker.setItems(items); picker.onActivate = item => { const efforts = getSupportedEfforts(item.model); diff --git a/packages/coding-agent/src/modes/components/model-browser.ts b/packages/coding-agent/src/modes/components/model-browser.ts index 8eec17093..4f37b196f 100644 --- a/packages/coding-agent/src/modes/components/model-browser.ts +++ b/packages/coding-agent/src/modes/components/model-browser.ts @@ -24,6 +24,7 @@ import { import { formatNumber } from "@oh-my-pi/pi-utils"; import { getRoleInfo, MODEL_ROLE_IDS } from "../../config/model-roles"; import type { Settings } from "../../config/settings"; +import type { ModelPerfStats } from "../../session/agent-storage"; import { AUTO_THINKING, type ConfiguredThinkingLevel } from "../../thinking"; import { theme } from "../theme/theme"; import { @@ -227,6 +228,18 @@ function formatContext(model: Model): string { return `${formatNumber(ctx).toLowerCase()} ${theme.icon.context.replace(/:$/, "")}`; } +/** `118t/s` average output speed; one decimal below 10 t/s. */ +function formatTps(tps: number): string { + const value = tps >= 10 ? String(Math.round(tps)) : tps.toFixed(1); + return `${value}t/s`; +} + +/** `0.9s` average time-to-first-token; whole seconds from 10s up. */ +function formatTtft(ms: number): string { + const seconds = ms / 1000; + return seconds >= 10 ? `${Math.round(seconds)}s` : `${seconds.toFixed(1)}s`; +} + /** Pad `text` on the left to `width` terminal columns (ANSI/emoji aware). */ function padLeftVisible(text: string, width: number): string { const missing = width - visibleWidth(text); @@ -250,6 +263,12 @@ export interface ModelBrowserOptions { const LIST_ROW_START = 2; /** Rendered rows after the list window: blank + two detail rows. */ const DETAIL_ROWS = 3; +/** Row width from which the measured-perf column appears (TPS only). */ +const PERF_TPS_MIN_WIDTH = 76; +/** Row width from which the perf column also includes TTFT. */ +const PERF_FULL_MIN_WIDTH = 96; +/** What the per-row perf column shows at the current width. */ +type PerfMode = "off" | "tps" | "full"; /** * The reusable browser component. Renders a fixed-height block @@ -263,6 +282,7 @@ export class ModelBrowser implements Component { #visibleItems: ModelBrowserItem[] = []; #roles: RoleAssignments = {}; #mruOrder: ReadonlyArray = []; + #perf: ReadonlyMap = new Map(); #selectedIndex = 0; #hoveredIndex: number | null = null; #maxVisible = 10; @@ -270,6 +290,7 @@ export class ModelBrowser implements Component { #currentContextTokens: number; #disableOverContext: boolean; #emptyText?: () => string | undefined; + /** First visible list row; panned by the wheel, snapped to the selection on keyboard navigation. */ #windowStart = 0; #windowCount = 0; @@ -310,7 +331,14 @@ export class ModelBrowser implements Component { this.#mruOrder = order; } + /** Measured TPS/TTFT averages keyed by `provider/id` selector (see AgentStorage.getModelPerf). */ + setPerfStats(perf: ReadonlyMap): void { + this.#perf = perf; + } + setMaxVisible(rows: number): void { + // No selection snap here: hosts call this on every render, and it must + // not undo wheel panning. render() re-clamps the window. this.#maxVisible = Math.max(1, rows); } @@ -345,6 +373,7 @@ export class ModelBrowser implements Component { const index = this.#visibleItems.findIndex(item => item.selector === selector); if (index < 0) return false; this.#selectedIndex = this.#coerceSelectedIndex(index); + this.#ensureSelectedVisible(); return true; } @@ -372,6 +401,21 @@ export class ModelBrowser implements Component { return clamped; } + /** Clamp a window start into `[0, total - maxVisible]`. */ + #clampWindowStart(start: number): number { + return Math.max(0, Math.min(start, this.#visibleItems.length - this.#maxVisible)); + } + + /** Scroll just enough to keep the selected row inside the window. */ + #ensureSelectedVisible(): void { + if (this.#selectedIndex < this.#windowStart) { + this.#windowStart = this.#selectedIndex; + } else if (this.#selectedIndex >= this.#windowStart + this.#maxVisible) { + this.#windowStart = this.#selectedIndex - this.#maxVisible + 1; + } + this.#windowStart = this.#clampWindowStart(this.#windowStart); + } + moveSelection(delta: number): void { const count = this.#visibleItems.length; if (count === 0) return; @@ -389,6 +433,7 @@ export class ModelBrowser implements Component { #setSelectedIndex(index: number): void { if (index === this.#selectedIndex) return; this.#selectedIndex = index; + this.#ensureSelectedVisible(); this.onSelectionChange?.(this.getSelected()); } @@ -451,6 +496,7 @@ export class ModelBrowser implements Component { } this.#visibleItems = this.#insertSeparator(items); this.#selectedIndex = this.#coerceSelectedIndex(Math.min(this.#selectedIndex, this.#visibleItems.length - 1)); + this.#ensureSelectedVisible(); this.onSelectionChange?.(this.getSelected()); } @@ -508,36 +554,37 @@ export class ModelBrowser implements Component { */ routeMouse(event: SgrMouseEvent, line: number): void { if (event.wheel !== null) { - this.moveSelection(event.wheel); - return; - } - const listLine = line - LIST_ROW_START; - if (listLine < 0 || listLine >= this.#windowCount) { - if (event.motion && this.#hoveredIndex !== null) { - this.#hoveredIndex = null; - } - return; - } - const index = this.#windowStart + listLine; - const item = this.#visibleItems[index]; - if (!item || this.#isDisabled(item)) { - this.#hoveredIndex = null; + // Wheel pans the window; it never moves the selection and never wraps. + this.#windowStart = this.#clampWindowStart(this.#windowStart + event.wheel); + this.#hoveredIndex = this.#hoverIndexAt(line); return; } if (event.motion) { - this.#hoveredIndex = index; + this.#hoveredIndex = this.#hoverIndexAt(line); return; } - if (event.leftClick) { - // Settings idiom: click selects, click-again activates. - if (index === this.#selectedIndex) { - this.onActivate?.(item); - } else { - this.#setSelectedIndex(index); - } + if (!event.leftClick) return; + const index = this.#hoverIndexAt(line); + const item = index !== null ? this.#visibleItems[index] : undefined; + if (index === null || !item) return; + // Settings idiom: click selects, click-again activates. + if (index === this.#selectedIndex) { + this.onActivate?.(item); + } else { + this.#setSelectedIndex(index); } } + /** List index under a frame-local row, or null when off-list or on a disabled row. */ + #hoverIndexAt(line: number): number | null { + const listLine = line - LIST_ROW_START; + if (listLine < 0 || listLine >= this.#windowCount) return null; + const index = this.#windowStart + listLine; + const item = this.#visibleItems[index]; + if (!item || this.#isDisabled(item)) return null; + return index; + } + #chipsFor(model: Model): string { const parts: string[] = []; const seen = new Set(); @@ -554,6 +601,16 @@ export class ModelBrowser implements Component { return parts.length > 0 ? ` ${parts.join(" ")}` : ""; } + /** `0.9s 118t/s` measured-perf cell for the row's meta block; empty when unmeasured or the column is off. */ + #perfCell(item: ModelBrowserItem, mode: PerfMode): string { + if (mode === "off") return ""; + const perf = this.#perf.get(item.selector); + if (!perf) return ""; + const tps = formatTps(perf.tps); + if (mode === "full" && perf.ttftMs !== null) return `${formatTtft(perf.ttftMs)} ${tps}`; + return tps; + } + #renderRow( item: ModelBrowserItem, width: number, @@ -561,6 +618,8 @@ export class ModelBrowser implements Component { hovered: boolean, ctxWidth: number, costWidth: number, + perfWidth: number, + perfMode: PerfMode, ): string { if (item.id === "separator") { const dashCount = Math.max(0, width - 4); @@ -576,8 +635,11 @@ export class ModelBrowser implements Component { : ""; let left = `${prefix}${providerPrefix}${name}${this.#chipsFor(item.model)}${overLimit}`; - const meta = `${theme.fg("dim", padLeftVisible(formatContext(item.model), ctxWidth))} ${theme.fg("dim", padLeftVisible(formatCostPair(item.model), costWidth))}`; - const metaWidth = ctxWidth + costWidth + 2; + // Perf column collapses entirely when no visible row has measurements. + const perfCol = + perfWidth > 0 ? `${theme.fg("dim", padLeftVisible(this.#perfCell(item, perfMode), perfWidth))} ` : ""; + const meta = `${perfCol}${theme.fg("dim", padLeftVisible(formatContext(item.model), ctxWidth))} ${theme.fg("dim", padLeftVisible(formatCostPair(item.model), costWidth))}`; + const metaWidth = ctxWidth + costWidth + 2 + (perfWidth > 0 ? perfWidth + 2 : 0); const available = Math.max(1, width - metaWidth - 1); left = truncateToWidth(left, available); const gap = Math.max(0, available - visibleWidth(left)); @@ -603,6 +665,11 @@ export class ModelBrowser implements Component { facts.push(`${formatCostPair(model)} per M`); if (model.reasoning) facts.push("reasoning"); if (model.input.includes("image")) facts.push("vision"); + const perf = this.#perf.get(selected.selector); + if (perf) { + facts.push(`~${formatTps(perf.tps)}`); + if (perf.ttftMs !== null) facts.push(`${formatTtft(perf.ttftMs)} ttft`); + } const line1 = truncateToWidth(theme.fg("muted", ` ${facts.join(" · ")}`), width); if (this.#isDisabled(selected)) { @@ -635,12 +702,12 @@ export class ModelBrowser implements Component { lines.push(""); const total = this.#visibleItems.length; - const startIndex = Math.max( - 0, - Math.min(this.#selectedIndex - Math.floor(this.#maxVisible / 2), total - this.#maxVisible), - ); + // The window is persistent state: wheel scrolling panned it, keyboard + // navigation snapped it to the selection. Re-clamp here because items + // or maxVisible may have changed since. + this.#windowStart = this.#clampWindowStart(this.#windowStart); + const startIndex = this.#windowStart; const endIndex = Math.min(startIndex + this.#maxVisible, total); - this.#windowStart = startIndex; this.#windowCount = Math.max(0, endIndex - startIndex); if (total === 0) { @@ -653,11 +720,14 @@ export class ModelBrowser implements Component { // scanning the entire catalog on every render. let ctxWidth = 0; let costWidth = 0; + const perfMode: PerfMode = width >= PERF_FULL_MIN_WIDTH ? "full" : width >= PERF_TPS_MIN_WIDTH ? "tps" : "off"; + let perfWidth = 0; for (let i = startIndex; i < endIndex; i++) { const item = this.#visibleItems[i]; if (!item) continue; ctxWidth = Math.max(ctxWidth, visibleWidth(formatContext(item.model))); costWidth = Math.max(costWidth, visibleWidth(formatCostPair(item.model))); + perfWidth = Math.max(perfWidth, visibleWidth(this.#perfCell(item, perfMode))); } const rows: string[] = []; @@ -672,6 +742,8 @@ export class ModelBrowser implements Component { i === this.#hoveredIndex, ctxWidth, costWidth, + perfWidth, + perfMode, ), ); } diff --git a/packages/coding-agent/src/modes/components/model-hub.ts b/packages/coding-agent/src/modes/components/model-hub.ts index 82fa5cb6b..7771baf5f 100644 --- a/packages/coding-agent/src/modes/components/model-hub.ts +++ b/packages/coding-agent/src/modes/components/model-hub.ts @@ -202,6 +202,8 @@ export class ModelHubComponent implements Component { #searchTotal = 0; #activeEntryId = "all"; #sidebarScroll = 0; + /** Snap the sidebar viewport to the active entry on the next render; wheel panning leaves it free. */ + #sidebarFollowActive = true; #sidebarHover: number | null = null; /** * Arrow-key ownership: `scope` (default) hops the sidebar even while the @@ -393,11 +395,13 @@ export class ModelHubComponent implements Component { this.#reloadRoles(availableModels); this.#buildRolesRows(); - const mruOrder = this.#settings.getStorage()?.getModelUsageOrder() ?? []; + const storage = this.#settings.getStorage(); + const mruOrder = storage?.getModelUsageOrder() ?? []; this.#availableItems = buildBrowserItems(availableModels); sortModelItems(this.#availableItems, { roles: this.#roles, mruOrder }); this.#browser.setRoles(this.#roles); this.#browser.setMruOrder(mruOrder); + this.#browser.setPerfStats(storage?.getModelPerf() ?? new Map()); const bySelector = new Map(this.#availableItems.map(item => [item.selector, item])); this.#recentItems = []; @@ -521,6 +525,7 @@ export class ModelHubComponent implements Component { this.#entries = entries; if (!entries.some(entry => entry.id === this.#activeEntryId)) { this.#activeEntryId = "all"; + this.#sidebarFollowActive = true; } } @@ -531,6 +536,7 @@ export class ModelHubComponent implements Component { #setActiveEntry(id: string): void { if (!this.#entries.some(entry => entry.id === id)) return; this.#activeEntryId = id; + this.#sidebarFollowActive = true; this.#applyScope(); const entry = this.#activeEntry(); // Hops must never steal arrow focus: landing on a scope keeps provider @@ -1293,15 +1299,23 @@ export class ModelHubComponent implements Component { } } - /** Step the roles cursor by one row, wrapping and skipping separator rows. */ - #stepRoleIndex(from: number, delta: -1 | 1): number { - const count = Math.max(1, this.#rolesRows.length); + /** Step the roles cursor by one row, skipping separator rows. Wraps at the ends unless `wrap: false` (then the cursor stays put). */ + #stepRoleIndex(from: number, delta: -1 | 1, options: { wrap?: boolean } = {}): number { + const wrap = options.wrap ?? true; + const count = this.#rolesRows.length; + if (count === 0) return 0; let index = from; for (let i = 0; i < count; i++) { - index = (index + delta + count) % count; - if (this.#rolesRows[index]?.kind !== "separator") break; + const next = index + delta; + if (next < 0 || next >= count) { + if (!wrap) return from; + index = (next + count) % count; + } else { + index = next; + } + if (this.#rolesRows[index]?.kind !== "separator") return index; } - return index; + return from; } #handleRolesViewInput(data: string): void { @@ -1431,10 +1445,13 @@ export class ModelHubComponent implements Component { if (event.wheel !== null) { if (overSidebar) { - this.#moveSidebar(event.wheel); + // Wheel pans the sidebar viewport; picking a scope is click/keys only. + const maxScroll = Math.max(0, this.#entries.length - this.#contentRowCount); + this.#sidebarScroll = Math.max(0, Math.min(this.#sidebarScroll + event.wheel, maxScroll)); + this.#sidebarHover = this.#sidebarEntryIndexAt(contentLine); } else if (overBody) { if (entry.kind === "roles" && this.#assigning === null) { - this.#roleIndex = this.#stepRoleIndex(this.#roleIndex, event.wheel > 0 ? 1 : -1); + this.#roleIndex = this.#stepRoleIndex(this.#roleIndex, event.wheel > 0 ? 1 : -1, { wrap: false }); } else if (this.#isBrowserView(entry)) { this.#browser.routeMouse(event, bodyLine); } @@ -1520,15 +1537,22 @@ export class ModelHubComponent implements Component { } #renderSidebar(width: number, rows: number): string[] { - const activeIndex = Math.max( - 0, - this.#entries.findIndex(entry => entry.id === this.#activeEntryId), - ); - if (this.#entries.length > rows) { - this.#sidebarScroll = Math.max(0, Math.min(activeIndex - Math.floor(rows / 2), this.#entries.length - rows)); - } else { - this.#sidebarScroll = 0; + // The scroll offset is persistent: the wheel pans it freely. Only an + // activation (keys, click, programmatic) snaps the viewport to the + // active entry, and only far enough to reveal it. + if (this.#sidebarFollowActive) { + const activeIndex = Math.max( + 0, + this.#entries.findIndex(entry => entry.id === this.#activeEntryId), + ); + if (activeIndex < this.#sidebarScroll) { + this.#sidebarScroll = activeIndex; + } else if (activeIndex >= this.#sidebarScroll + rows) { + this.#sidebarScroll = activeIndex - rows + 1; + } + this.#sidebarFollowActive = false; } + this.#sidebarScroll = Math.max(0, Math.min(this.#sidebarScroll, Math.max(0, this.#entries.length - rows))); const lines: string[] = []; for (let i = this.#sidebarScroll; i < Math.min(this.#entries.length, this.#sidebarScroll + rows); i++) { diff --git a/packages/coding-agent/test/model-browser.test.ts b/packages/coding-agent/test/model-browser.test.ts index a8c34e8dd..2dc3a9968 100644 --- a/packages/coding-agent/test/model-browser.test.ts +++ b/packages/coding-agent/test/model-browser.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { beforeAll, describe, expect, test } from "bun:test"; import type { Model } from "@oh-my-pi/pi-ai"; import { buildModel } from "@oh-my-pi/pi-catalog/build"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; @@ -7,6 +7,7 @@ import { ModelBrowser, sortModelItems, } from "@oh-my-pi/pi-coding-agent/modes/components/model-browser"; +import { initTheme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; function makeModel(provider: string, id: string): Model { return buildModel({ @@ -62,3 +63,44 @@ describe("ModelBrowser search ranking", () => { expect(browser.getSelected()?.selector).toBe("zenmux/gpt-5.5"); }); }); + +describe("ModelBrowser perf display", () => { + beforeAll(async () => { + // render() reads the global theme singleton. + await initTheme(false); + }); + + function makePerfBrowser(): ModelBrowser { + const browser = new ModelBrowser(Settings.isolated({})); + browser.setItems(buildBrowserItems([makeModel("openai", "gpt-5")])); + browser.setPerfStats(new Map([["openai/gpt-5", { samples: 12, tps: 118.4, ttftMs: 930 }]])); + return browser; + } + + function renderPlain(browser: ModelBrowser, width: number): string[] { + return browser.render(width).map(line => Bun.stripANSI(line)); + } + + test("row perf column scales with width: off, TPS-only, TTFT+TPS", () => { + const browser = makePerfBrowser(); + + expect(renderPlain(browser, 70)[2]).not.toContain("t/s"); + expect(renderPlain(browser, 80)[2]).toContain("118t/s"); + const wideRow = renderPlain(browser, 120)[2]; + expect(wideRow).toContain("0.9s 118t/s"); + }); + + test("detail line shows measured perf regardless of width", () => { + const browser = makePerfBrowser(); + + const lines = renderPlain(browser, 70); + expect(lines[lines.length - 2]).toContain("~118t/s · 0.9s ttft"); + }); + + test("models without measurements render no perf cell", () => { + const browser = new ModelBrowser(Settings.isolated({})); + browser.setItems(buildBrowserItems([makeModel("openai", "gpt-5")])); + + expect(renderPlain(browser, 120)[2]).not.toContain("t/s"); + }); +}); diff --git a/packages/coding-agent/test/model-hub.test.ts b/packages/coding-agent/test/model-hub.test.ts index 95fd2d3b6..93d31ad6c 100644 --- a/packages/coding-agent/test/model-hub.test.ts +++ b/packages/coding-agent/test/model-hub.test.ts @@ -689,6 +689,85 @@ describe("ModelHub", () => { }); }); + describe("mouse wheel", () => { + // SGR wheel reports: button 64 = up, 65 = down. Column 100 lands in the + // body pane, column 3 in the sidebar; row 10 is inside the content rows. + const WHEEL_UP_BODY = "\x1b[<64;100;10M"; + const WHEEL_DOWN_BODY = "\x1b[<65;100;10M"; + const WHEEL_UP_SIDEBAR = "\x1b[<64;3;10M"; + const WHEEL_DOWN_SIDEBAR = "\x1b[<65;3;10M"; + + test("wheel pans the model list without moving the selection and clamps at the ends", () => { + const models = Array.from({ length: 40 }, (_, i) => makeModel("test", `model-${String(i).padStart(2, "0")}`)); + const { hub, onPick } = createHub({ models, scoped: true, hub: { mode: "pick" } }); + + const before = normalize(hub.render(220)); // establishes mouse geometry + hub.handleInput("\n"); + expect(onPick).toHaveBeenCalledTimes(1); + const initiallySelected = onPick.mock.calls[0]?.[0]; + + // Panning reveals rows that were below the fold... + for (let i = 0; i < 8; i++) hub.handleInput(WHEEL_DOWN_BODY); + const panned = normalize(hub.render(220)); + const modelIdsIn = (frame: string) => new Set(Array.from(frame.matchAll(/model-\d\d/g), match => match[0])); + const beforeIds = modelIdsIn(before); + const revealed = [...modelIdsIn(panned)].filter(id => !beforeIds.has(id)); + expect(revealed.length).toBeGreaterThan(0); + + // ...but never moves the selection: Enter still picks the same model. + hub.handleInput("\n"); + expect(onPick).toHaveBeenCalledTimes(2); + expect(onPick.mock.calls[1]?.[0]).toBe(initiallySelected); + + // The window clamps at the bottom instead of wrapping back to the top... + for (let i = 0; i < 500; i++) hub.handleInput(WHEEL_DOWN_BODY); + const saturated = normalize(hub.render(220)); + hub.handleInput(WHEEL_DOWN_BODY); + expect(normalize(hub.render(220))).toBe(saturated); + + // ...and scrolling back up restores the original window exactly. + for (let i = 0; i < 500; i++) hub.handleInput(WHEEL_UP_BODY); + expect(normalize(hub.render(220))).toBe(before); + }); + + test("wheel over the sidebar never changes the active scope or schedules refreshes", () => { + vi.useFakeTimers(); + try { + const refreshProvider = vi.fn(async () => {}); + const { hub } = createHub({ + models: [makeModel("prov-a", "model-a"), makeModel("prov-b", "model-b")], + registry: { refreshProvider }, + }); + + expect(normalize(hub.render(220))).toContain("All available models"); + + // Two hops under the old wheel-selects behavior would land on a + // provider scope; the viewport pan must leave the scope alone. + for (let i = 0; i < 2; i++) hub.handleInput(WHEEL_DOWN_SIDEBAR); + expect(normalize(hub.render(220))).toContain("All available models"); + for (let i = 0; i < 2; i++) hub.handleInput(WHEEL_UP_SIDEBAR); + expect(normalize(hub.render(220))).toContain("All available models"); + + // No scope change means no provider auto-refresh either. + vi.advanceTimersByTime(200); // past the 120ms provider-refresh debounce + expect(refreshProvider).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + test("wheel in the roles view clamps at the top instead of wrapping to the bottom rows", () => { + const { hub } = createHub({ models: [makeModel("test", "model-a")], scoped: true }); + + hub.handleInput(UP); // All models → Roles + hub.render(220); // establish mouse geometry + for (let i = 0; i < 4; i++) hub.handleInput(WHEEL_UP_BODY); // cursor stays on the first role + hub.handleInput("\n"); // dive into the rows + hub.handleInput("\n"); // activate the cursor row + expect(normalize(hub.render(220))).toContain("Assigning DEFAULT"); + }); + }); + describe("provider scopes and search", () => { test("search inside a provider scope keeps that provider's model (#4522)", () => { const openrouterGlm = makeModel("openrouter", "z-ai/glm-5.2"); From b6559861d0e7ebb39f62116e277c5b5fe3c07f3f Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 01:06:24 +0200 Subject: [PATCH 17/80] feat(cli): standardized throughput calculation to total duration - Removed the `computeTokensPerSecond` helper function in favor of a direct calculation. - Standardized throughput to use total request duration instead of post-TTFT decode time. - Updated UI usage display to calculate tokens per second against total duration. - Prevented over-inflation of throughput metrics caused by hidden reasoning tokens. --- packages/coding-agent/src/cli/bench-cli.ts | 33 ++++--------------- .../src/modes/components/usage-row.ts | 11 +++---- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/packages/coding-agent/src/cli/bench-cli.ts b/packages/coding-agent/src/cli/bench-cli.ts index fd635e43f..f324aad76 100644 --- a/packages/coding-agent/src/cli/bench-cli.ts +++ b/packages/coding-agent/src/cli/bench-cli.ts @@ -74,7 +74,7 @@ export interface BenchRunSuccess { ttftMs: number; durationMs: number; outputTokens: number; - /** Generation throughput measured over the post-first-token window. */ + /** Output tokens/sec over the total request duration. */ tokensPerSecond: number; } @@ -181,23 +181,6 @@ function hasVisibleFinalContent(message: AssistantMessage): boolean { }); } -/** - * Tokens/s over the generation window (duration minus TTFT) so queue/prefill - * latency does not dilute throughput. Falls back to total duration when the - * response arrived as a single chunk (TTFT ~ duration). - */ -export function computeTokensPerSecond( - outputTokens: number, - durationMs: number, - ttftMs: number, - deltaChunkCount: number, -): number { - const decodeMs = durationMs - ttftMs; - // Fall back to total duration when the response arrived as a single chunk/non-streaming. - const windowMs = decodeMs > 0 && deltaChunkCount >= 2 ? decodeMs : durationMs; - return windowMs > 0 ? (outputTokens * 1000) / windowMs : 0; -} - interface BenchRequestOptions { apiKey: ApiKeyResolver; sessionId: string; @@ -247,17 +230,10 @@ async function runBenchRequest( headers: model.provider === "openrouter" ? { "X-OpenRouter-Cache": "false" } : undefined, }); let message: AssistantMessage | undefined; - let deltaChunkCount = 0; for await (const event of stream) { if (firstTokenAt === undefined && isFirstTokenEvent(event)) { firstTokenAt = now(); } - if ( - (event.type === "text_delta" || event.type === "thinking_delta" || event.type === "toolcall_delta") && - event.delta.length > 0 - ) { - deltaChunkCount++; - } if (event.type === "error") { return { ok: false, error: event.error.errorMessage ?? "request failed" }; } @@ -291,7 +267,12 @@ async function runBenchRequest( ttftMs, durationMs, outputTokens, - tokensPerSecond: computeTokensPerSecond(outputTokens, durationMs, ttftMs, deltaChunkCount), + // TPS over the TOTAL request duration, deliberately not the post-TTFT + // decode window: reasoning models can spend seconds generating hidden + // thinking tokens (counted in usage.output) before the first visible + // byte, so "duration - TTFT" inflates TPS several-fold on providers + // that buffer or hide reasoning (e.g. google vs google-vertex). + tokensPerSecond: durationMs > 0 ? (outputTokens * 1000) / durationMs : 0, }; } catch (error) { return { ok: false, error: getErrorMessage(error) }; diff --git a/packages/coding-agent/src/modes/components/usage-row.ts b/packages/coding-agent/src/modes/components/usage-row.ts index 78b012623..8ce18ec1f 100644 --- a/packages/coding-agent/src/modes/components/usage-row.ts +++ b/packages/coding-agent/src/modes/components/usage-row.ts @@ -18,12 +18,11 @@ export function createUsageRowBlock(usage: Usage, durationMs?: number, ttftMs?: parts.push(`${theme.icon.time} ${(ttftMs / 1000).toFixed(1)}s`); } if (durationMs && durationMs > MIN_DURATION_MS && usage.output > 0) { - // Throughput excludes TTFT — generation time is duration minus time-to-first-token. - const genMs = durationMs - (ttftMs ?? 0); - if (genMs > MIN_DURATION_MS) { - const tokPerSec = (usage.output / genMs) * 1000; - parts.push(`${theme.icon.throughput} ${tokPerSec.toFixed(1)}/s`); - } + // TPS over the total request duration — the post-TTFT window undercounts + // generation time when reasoning tokens are hidden before the first + // visible byte, inflating the rate. + const tokPerSec = (usage.output / durationMs) * 1000; + parts.push(`${theme.icon.throughput} ${tokPerSec.toFixed(1)}/s`); } const block = new Container(); block.addChild(new Spacer(1)); From 41317cc231212c14bdb9972a8581997f8f2552e6 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 01:06:27 +0200 Subject: [PATCH 18/80] test(perf): validated performance sample aggregation and backfill logic - Verified throughput (TPS) and latency (TTFT) aggregation logic for multiple samples. - Confirmed data sanitization by dropping invalid samples like NaN, zero duration, or missing values. - Ensured backfilling correctly filters stale, errored, and empty turns from the stats database. - Validated that backfill operations respect the maximum sample limit per model. - Validated the async deferral behavior in recording perf samples. --- .../test/agent-storage-model-perf.test.ts | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 packages/coding-agent/test/agent-storage-model-perf.test.ts diff --git a/packages/coding-agent/test/agent-storage-model-perf.test.ts b/packages/coding-agent/test/agent-storage-model-perf.test.ts new file mode 100644 index 000000000..e0c450513 --- /dev/null +++ b/packages/coding-agent/test/agent-storage-model-perf.test.ts @@ -0,0 +1,161 @@ +import { Database } from "bun:sqlite"; +import { afterEach, describe, expect, it } from "bun:test"; +import * as path from "node:path"; +import { AgentStorage } from "@oh-my-pi/pi-coding-agent/session/agent-storage"; +import { TempDir } from "@oh-my-pi/pi-utils"; + +describe("AgentStorage model perf aggregates", () => { + let tempDir: TempDir; + + afterEach(async () => { + AgentStorage.resetInstance(); + if (tempDir) { + try { + await tempDir.remove(); + } catch {} + tempDir = undefined as unknown as TempDir; + } + }); + + async function openStorage(): Promise { + tempDir = TempDir.createSync("@omp-agent-storage-perf-"); + return AgentStorage.open(path.join(tempDir.path(), "agent.db")); + } + + it("averages TPS over total request duration and TTFT over reporting samples", async () => { + const storage = await openStorage(); + + // 1000 tokens over 6000ms + 500 tokens over 3000ms → 1500 tokens / 9s → 166.67 t/s + // Back-to-back samples join one deferred batch; awaiting the shared flush + // promise makes both visible. + storage.recordModelPerf("openai/gpt-5", { outputTokens: 1000, durationMs: 6000, ttftMs: 1000 }); + await storage.recordModelPerf("openai/gpt-5", { outputTokens: 500, durationMs: 3000, ttftMs: 500 }); + + const stats = storage.getModelPerf().get("openai/gpt-5"); + expect(stats).toBeDefined(); + expect(stats?.samples).toBe(2); + expect(stats?.tps).toBeCloseTo(1500000 / 9000, 5); + expect(stats?.ttftMs).toBeCloseTo(750, 5); + }); + + it("keeps TTFT null when no sample reported one and uses full duration for TPS", async () => { + const storage = await openStorage(); + + // No ttft → 1000 tokens / 4s → 250 t/s + await storage.recordModelPerf("zai/glm-5", { outputTokens: 1000, durationMs: 4000 }); + + const stats = storage.getModelPerf().get("zai/glm-5"); + expect(stats?.tps).toBeCloseTo(250, 5); + expect(stats?.ttftMs).toBeNull(); + }); + + it("reports identical TPS regardless of TTFT (hidden-reasoning regression)", async () => { + const storage = await openStorage(); + + // Same duration and token count, wildly different TTFT: a provider that + // hides reasoning until late (ttft ~ duration) must not report inflated + // throughput vs one that streams from the start. + storage.recordModelPerf("google/gemini", { outputTokens: 1020, durationMs: 7000, ttftMs: 5700 }); + await storage.recordModelPerf("google-vertex/gemini", { outputTokens: 1020, durationMs: 7000, ttftMs: 1700 }); + + const hidden = storage.getModelPerf().get("google/gemini"); + const streamed = storage.getModelPerf().get("google-vertex/gemini"); + expect(hidden?.tps).toBeCloseTo(1020000 / 7000, 5); + expect(streamed?.tps).toBeCloseTo(1020000 / 7000, 5); + }); + + it("drops unmeasurable samples instead of polluting the aggregates", async () => { + const storage = await openStorage(); + + await storage.recordModelPerf("openai/gpt-5", { outputTokens: 0, durationMs: 4000 }); + await storage.recordModelPerf("openai/gpt-5", { outputTokens: 100, durationMs: 0 }); + await storage.recordModelPerf("openai/gpt-5", { outputTokens: Number.NaN, durationMs: 4000 }); + + expect(storage.getModelPerf().has("openai/gpt-5")).toBe(false); + }); + + it("ignores out-of-range TTFT but keeps the throughput sample", async () => { + const storage = await openStorage(); + + // ttft >= duration is bogus latency data; the sample still measures TPS. + await storage.recordModelPerf("openai/gpt-5", { outputTokens: 1000, durationMs: 4000, ttftMs: 5000 }); + + const stats = storage.getModelPerf().get("openai/gpt-5"); + expect(stats?.tps).toBeCloseTo(250, 5); + expect(stats?.ttftMs).toBeNull(); + }); + + it("defers the write off the record path and lands it once the flush promise resolves", async () => { + const storage = await openStorage(); + + const flushed = storage.recordModelPerf("openai/gpt-5", { outputTokens: 1000, durationMs: 4000 }); + // Recording is deferred: nothing is visible before the batch flushes. + expect(storage.getModelPerf().has("openai/gpt-5")).toBe(false); + + await flushed; + expect(storage.getModelPerf().get("openai/gpt-5")?.tps).toBeCloseTo(250, 5); + }); + + it("backfills perf aggregates from an omp stats database, excluding errored and stale turns", async () => { + const storage = await openStorage(); + + // Minimal stats.db fixture: only the columns the backfill query reads. + const statsDbPath = path.join(tempDir.path(), "stats.db"); + const statsDb = new Database(statsDbPath); + statsDb.run(`CREATE TABLE messages ( + provider TEXT, model TEXT, output_tokens INTEGER, duration INTEGER, + ttft INTEGER, stop_reason TEXT, timestamp INTEGER + )`); + const insert = statsDb.prepare("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?, ?)"); + const now = Date.now(); + // Two valid turns totaling 1500 tokens over 8.5s, one with ttft missing. + insert.run("openai", "gpt-5", 1000, 6000, 1000, "stop", now - 5000); + insert.run("openai", "gpt-5", 500, 2500, null, "stop", now - 4000); + // Errored and empty turns must not pollute the averages. + insert.run("openai", "gpt-5", 9999, 1, null, "error", now - 3000); + insert.run("openai", "gpt-5", 0, 4000, null, "stop", now - 2000); + // Rows older than the recency window are stale provider speeds; skip them. + insert.run("openai", "gpt-5", 100_000, 1000, null, "stop", now - 120 * 86_400_000); + insert.run("zai", "glm-5", 300, 3000, 1000, "aborted", now - 1000); + statsDb.close(); + + const imported = await storage.backfillModelPerfFromStats(statsDbPath); + + expect(imported).toBe(3); + const gpt = storage.getModelPerf().get("openai/gpt-5"); + // 1500 tokens over 6000ms + 2500ms total durations → 176.47 t/s. + expect(gpt?.samples).toBe(2); + expect(gpt?.tps).toBeCloseTo(1500000 / 8500, 5); + expect(gpt?.ttftMs).toBeCloseTo(1000, 5); + // Aborted turns with reported usage are valid samples, like live capture. + const glm = storage.getModelPerf().get("zai/glm-5"); + expect(glm?.tps).toBeCloseTo(100, 5); + }); + + it("caps the backfill at the newest samples per model", async () => { + const storage = await openStorage(); + + const statsDbPath = path.join(tempDir.path(), "stats.db"); + const statsDb = new Database(statsDbPath); + statsDb.run(`CREATE TABLE messages ( + provider TEXT, model TEXT, output_tokens INTEGER, duration INTEGER, + ttft INTEGER, stop_reason TEXT, timestamp INTEGER + )`); + const insert = statsDb.prepare("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?, ?)"); + const now = Date.now(); + // 300 rows: the newest 256 run at 100 t/s, the older 44 at a wild + // 10000 t/s. Only the newest 256 may count. + for (let i = 0; i < 300; i++) { + const fast = i < 44; // smallest timestamps = oldest rows + insert.run("openai", "gpt-5", fast ? 10_000 : 100, 1000, null, "stop", now - (300 - i) * 1000); + } + statsDb.close(); + + const imported = await storage.backfillModelPerfFromStats(statsDbPath); + + expect(imported).toBe(256); + const stats = storage.getModelPerf().get("openai/gpt-5"); + expect(stats?.samples).toBe(256); + expect(stats?.tps).toBeCloseTo(100, 5); + }); +}); From d54dcc22240a5aabb43b7d7f016472af9fae2a25 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 01:35:50 +0200 Subject: [PATCH 19/80] feat(coding-agent): allowed model fallback after retry budget exhaustion - Permit model fallback even if the retry budget is exhausted when the current provider is locked by credential rotation or usage limits. - Reset the retry budget when successfully switching to a fallback model to ensure the new model has a full allowance of retries. - Added a regression test to verify that credential rotation failure triggers model fallback. --- packages/coding-agent/CHANGELOG.md | 1 + .../coding-agent/src/session/agent-session.ts | 52 +++++++++----- .../test/agent-session-retry-fallback.test.ts | 69 +++++++++++++++++++ 3 files changed, 106 insertions(+), 16 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 2c1bfc7f1..3d3300f69 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -19,6 +19,7 @@ ### Fixed +- Fixed failure to trigger model fallback when the retry budget is exhausted by credential rotation - Fixed uncontrollable mouse-wheel scrolling in the /models hub: the wheel moved the selection (one step per wheel event, so a single trackpad flick skipped many rows) and wrapped from the bottom back to the top. Wheel scrolling now pans the list viewport only, clamps at the ends, and leaves the selection where it is; keyboard navigation still scrolls the selection into view. Likewise, the wheel over the provider sidebar no longer switches the active scope (or triggers provider refreshes) — it just scrolls the sidebar. - Fixed TPS being inflated several-fold when a provider hides reasoning tokens until late in the stream (e.g. `google/gemini-3.5` vs `google-vertex/gemini-3.5` reporting 648 vs 186 TPS for identical durations): `omp bench`, the per-turn usage row, and the /models perf aggregates now measure tokens/sec over the total request duration instead of the post-TTFT decode window, matching `omp stats`. Stored perf aggregates are purged and re-backfilled from stats history with the corrected math on first launch. - Fixed the model-perf stats.db backfill freezing the TUI (~30s on multi-million-row stats databases) when /models triggered it: the import is now fire-and-forget, walks the newest rows in small chunks with event-loop yields between them, and is bounded to 90 days / 256 newest samples per model — beyond either bound the recency decay would erase the contribution anyway. diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 8c46ad4e0..31d6b1383 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -13993,20 +13993,13 @@ export class AgentSession { this.#retryResolve = resolve; } - if (this.#retryAttempt > retrySettings.maxRetries) { - await this.#persistRetryLifecycleErrorMessage(message); - // Max retries exceeded, emit final failure and reset - await this.#emitSessionEvent({ - type: "auto_retry_end", - success: false, - attempt: this.#retryAttempt - 1, - finalError: message.errorMessage, - }); - this.#clearPendingRecoveredRetryErrors(); - this.#retryAttempt = 0; - this.#resolveRetry(); // Resolve so waitForRetry() completes - return false; - } + // All attempts on the current model are spent. Don't fail yet: the + // fallback chain below gets one last consult. Credential rotation can + // consume the entire budget without the fallback branch ever running + // (every rotation sets switchedCredential and skips it), so without + // this last resort a provider-wide usage cap never fails over to the + // configured chain. + const retryBudgetExhausted = this.#retryAttempt > retrySettings.maxRetries; const errorMessage = message.errorMessage || "Unknown error"; const id = this.#classifyRetryMessage(message); @@ -14025,7 +14018,12 @@ export class AgentSession { this.#resetCurrentResponsesProviderSession("stale replay error"); } - if (this.model && !staleOpenAIResponsesReplayError && AIError.is(id, AIError.Flag.UsageLimit)) { + if ( + !retryBudgetExhausted && + this.model && + !staleOpenAIResponsesReplayError && + AIError.is(id, AIError.Flag.UsageLimit) + ) { const retryAfterMs = parsedRetryAfterMs ?? calculateRateLimitBackoffMs(parseRateLimitReason(errorMessage)); const outcome = await this.#modelRegistry.authStorage.markUsageLimitReached( this.model.provider, @@ -14071,7 +14069,9 @@ export class AgentSession { const allowModelFallback = options?.allowModelFallback !== false; const currentSelector = this.model ? formatRetryFallbackSelector(this.model, this.thinkingLevel) : undefined; if (!staleOpenAIResponsesReplayError && !switchedCredential && currentSelector) { - if (allowModelFallback && retrySettings.modelFallback) { + // A refusal chain stops at the retry budget: the exhausted-attempt + // last resort is for provider failures, not classifier decisions. + if (allowModelFallback && retrySettings.modelFallback && !(retryBudgetExhausted && classifierRefusal)) { if (!classifierRefusal) { this.#noteRetryFallbackCooldown(currentSelector, parsedRetryAfterMs, errorMessage); } @@ -14090,6 +14090,26 @@ export class AgentSession { delayMs = parsedRetryAfterMs; } } + if (retryBudgetExhausted) { + if (!switchedModel) { + await this.#persistRetryLifecycleErrorMessage(message); + // Max retries exceeded and no fallback model to switch to: emit + // final failure and reset. + await this.#emitSessionEvent({ + type: "auto_retry_end", + success: false, + attempt: this.#retryAttempt - 1, + finalError: message.errorMessage, + }); + this.#clearPendingRecoveredRetryErrors(); + this.#retryAttempt = 0; + this.#resolveRetry(); // Resolve so waitForRetry() completes + return false; + } + // The fallback model gets a fresh retry budget — leaving the spent + // counter in place would exhaust it again on its first error. + this.#retryAttempt = 1; + } if (classifierRefusal && !switchedModel) { this.#retryAttempt = 0; this.#resolveRetry(); diff --git a/packages/coding-agent/test/agent-session-retry-fallback.test.ts b/packages/coding-agent/test/agent-session-retry-fallback.test.ts index 92b5d0d27..0c11ed000 100644 --- a/packages/coding-agent/test/agent-session-retry-fallback.test.ts +++ b/packages/coding-agent/test/agent-session-retry-fallback.test.ts @@ -322,6 +322,75 @@ describe("AgentSession retry fallback", () => { ]); }); + it("falls back to the chain when credential rotation exhausts the retry budget", async () => { + const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5"); + const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); + if (!primaryModel || !fallbackModel) { + throw new Error("Expected bundled test models to exist"); + } + + const requestedModels: string[] = []; + const mock = createMockModel(); + const agent = new Agent({ + getApiKey: model => `${model.provider}-test-key`, + initialState: { + model: primaryModel, + systemPrompt: ["Test"], + tools: [], + messages: [], + }, + streamFn: (model, context, options) => { + requestedModels.push(`${model.provider}/${model.id}`); + if (model.provider === primaryModel.provider && model.id === primaryModel.id) { + mock.push({ throw: "429 usage_limit_reached" }); + } else { + mock.push({ content: [`ok:${model.provider}/${model.id}`] }); + } + return mock.stream(model, context, options); + }, + }); + + // Rotation always claims a sibling credential is available — the shape + // of a multi-account pool where the sibling check passes but every + // subsequent request keeps failing on the same capped account. + vi.spyOn(modelRegistry.authStorage, "markUsageLimitReached").mockResolvedValue({ switched: true }); + + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.baseDelayMs": 5, + "retry.maxRetries": 2, + "retry.fallbackChains": { + [`${primaryModel.provider}/${primaryModel.id}`]: [`${fallbackModel.provider}/${fallbackModel.id}`], + }, + }); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + const { retryStartEvents, retryEndEvents } = trackRetryEvents(session); + + await session.prompt("Exhaust rotation, then fail over"); + await session.waitForIdle(); + + // Two rotation retries burn the budget on the primary; the exhausted + // attempt consults the chain instead of giving up. + expect(requestedModels).toEqual([ + `${primaryModel.provider}/${primaryModel.id}`, + `${primaryModel.provider}/${primaryModel.id}`, + `${primaryModel.provider}/${primaryModel.id}`, + `${fallbackModel.provider}/${fallbackModel.id}`, + ]); + expect(session.model?.provider).toBe(fallbackModel.provider); + expect(session.model?.id).toBe(fallbackModel.id); + // The fallback model gets a fresh retry budget (attempt resets to 1). + expect(retryStartEvents.map(event => event.attempt)).toEqual([1, 2, 1]); + expect(retryEndEvents).toHaveLength(1); + expect(retryEndEvents[0]).toMatchObject({ success: true }); + }); + it("applies a provider-wildcard chain to any model of that provider", async () => { const primaryModel = getBundledModel("anthropic", "claude-opus-4-1"); const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); From 12466ecf40111342466db3d8caaf6f85ead790d2 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 02:50:43 +0200 Subject: [PATCH 20/80] chore: bump version to 16.4.6 --- Cargo.lock | 10 ++--- Cargo.toml | 2 +- bun.lock | 62 +++++++++++++-------------- crates/pi-natives/src/lib.rs | 2 +- package.json | 24 +++++------ packages/agent/package.json | 2 +- packages/ai/CHANGELOG.md | 2 + packages/ai/package.json | 2 +- packages/catalog/package.json | 2 +- packages/coding-agent/CHANGELOG.md | 2 + packages/coding-agent/package.json | 2 +- packages/hashline/package.json | 2 +- packages/mnemopi/package.json | 2 +- packages/natives/CHANGELOG.md | 8 ++-- packages/natives/native/index.d.ts | 2 +- packages/natives/native/index.js | 2 +- packages/natives/package.json | 2 +- packages/snapcompact/package.json | 2 +- packages/stats/package.json | 2 +- packages/swarm-extension/package.json | 2 +- packages/tui/CHANGELOG.md | 2 + packages/tui/package.json | 2 +- packages/utils/CHANGELOG.md | 2 + packages/utils/package.json | 2 +- packages/wire/package.json | 2 +- 25 files changed, 77 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc40d1526..c40f46dcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3264,7 +3264,7 @@ dependencies = [ [[package]] name = "pi-ast" -version = "16.4.5" +version = "16.4.6" dependencies = [ "anyhow", "ast-grep-core", @@ -3333,7 +3333,7 @@ dependencies = [ [[package]] name = "pi-iso" -version = "16.4.5" +version = "16.4.6" dependencies = [ "async-trait", "libc", @@ -3345,7 +3345,7 @@ dependencies = [ [[package]] name = "pi-natives" -version = "16.4.5" +version = "16.4.6" dependencies = [ "anyhow", "arboard", @@ -3398,7 +3398,7 @@ dependencies = [ [[package]] name = "pi-shell" -version = "16.4.5" +version = "16.4.6" dependencies = [ "anyhow", "brush-builtins", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "pi-walker" -version = "16.4.5" +version = "16.4.6" dependencies = [ "dashmap", "globset", diff --git a/Cargo.toml b/Cargo.toml index e70a113de..53c08f355 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["crates/vendor/brush-core", "crates/vendor/brush-builtins"] resolver = "3" [workspace.package] -version = "16.4.5" +version = "16.4.6" edition = "2024" license = "MIT" authors = ["Can Boluk"] diff --git a/bun.lock b/bun.lock index 626703d4d..867b96322 100644 --- a/bun.lock +++ b/bun.lock @@ -21,7 +21,7 @@ }, "packages/agent": { "name": "@oh-my-pi/pi-agent-core", - "version": "16.4.5", + "version": "16.4.6", "dependencies": { "@oh-my-pi/pi-ai": "catalog:", "@oh-my-pi/pi-catalog": "catalog:", @@ -39,7 +39,7 @@ }, "packages/ai": { "name": "@oh-my-pi/pi-ai", - "version": "16.4.5", + "version": "16.4.6", "dependencies": { "@bufbuild/protobuf": "catalog:", "@oh-my-pi/pi-catalog": "catalog:", @@ -55,7 +55,7 @@ }, "packages/catalog": { "name": "@oh-my-pi/pi-catalog", - "version": "16.4.5", + "version": "16.4.6", "dependencies": { "@bufbuild/protobuf": "catalog:", "@oh-my-pi/pi-utils": "catalog:", @@ -69,7 +69,7 @@ }, "packages/coding-agent": { "name": "@oh-my-pi/pi-coding-agent", - "version": "16.4.5", + "version": "16.4.6", "bin": { "omp": "src/cli.ts", }, @@ -138,7 +138,7 @@ }, "packages/hashline": { "name": "@oh-my-pi/hashline", - "version": "16.4.5", + "version": "16.4.6", "dependencies": { "diff": "catalog:", "lru-cache": "catalog:", @@ -149,7 +149,7 @@ }, "packages/mnemopi": { "name": "@oh-my-pi/pi-mnemopi", - "version": "16.4.5", + "version": "16.4.6", "bin": { "mnemopi": "src/cli.ts", }, @@ -175,7 +175,7 @@ }, "packages/natives": { "name": "@oh-my-pi/pi-natives", - "version": "16.4.5", + "version": "16.4.6", "devDependencies": { "@napi-rs/cli": "catalog:", "@types/bun": "catalog:", @@ -183,7 +183,7 @@ }, "packages/snapcompact": { "name": "@oh-my-pi/snapcompact", - "version": "16.4.5", + "version": "16.4.6", "dependencies": { "@oh-my-pi/pi-ai": "catalog:", "@oh-my-pi/pi-natives": "catalog:", @@ -196,7 +196,7 @@ }, "packages/stats": { "name": "@oh-my-pi/omp-stats", - "version": "16.4.5", + "version": "16.4.6", "bin": { "omp-stats": "./src/index.ts", }, @@ -222,7 +222,7 @@ }, "packages/swarm-extension": { "name": "@oh-my-pi/swarm-extension", - "version": "16.4.5", + "version": "16.4.6", "bin": { "omp-swarm": "src/cli.ts", }, @@ -248,7 +248,7 @@ }, "packages/tui": { "name": "@oh-my-pi/pi-tui", - "version": "16.4.5", + "version": "16.4.6", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "@oh-my-pi/pi-utils": "catalog:", @@ -289,7 +289,7 @@ }, "packages/utils": { "name": "@oh-my-pi/pi-utils", - "version": "16.4.5", + "version": "16.4.6", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "handlebars": "catalog:", @@ -302,7 +302,7 @@ }, "packages/wire": { "name": "@oh-my-pi/pi-wire", - "version": "16.4.5", + "version": "16.4.6", "devDependencies": { "@types/bun": "catalog:", }, @@ -342,18 +342,18 @@ "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.7.0", - "@oh-my-pi/hashline": "16.4.5", - "@oh-my-pi/omp-stats": "16.4.5", - "@oh-my-pi/pi-agent-core": "16.4.5", - "@oh-my-pi/pi-ai": "16.4.5", - "@oh-my-pi/pi-catalog": "16.4.5", - "@oh-my-pi/pi-coding-agent": "16.4.5", - "@oh-my-pi/pi-mnemopi": "16.4.5", - "@oh-my-pi/pi-natives": "16.4.5", - "@oh-my-pi/pi-tui": "16.4.5", - "@oh-my-pi/pi-utils": "16.4.5", - "@oh-my-pi/pi-wire": "16.4.5", - "@oh-my-pi/snapcompact": "16.4.5", + "@oh-my-pi/hashline": "16.4.6", + "@oh-my-pi/omp-stats": "16.4.6", + "@oh-my-pi/pi-agent-core": "16.4.6", + "@oh-my-pi/pi-ai": "16.4.6", + "@oh-my-pi/pi-catalog": "16.4.6", + "@oh-my-pi/pi-coding-agent": "16.4.6", + "@oh-my-pi/pi-mnemopi": "16.4.6", + "@oh-my-pi/pi-natives": "16.4.6", + "@oh-my-pi/pi-tui": "16.4.6", + "@oh-my-pi/pi-utils": "16.4.6", + "@oh-my-pi/pi-wire": "16.4.6", + "@oh-my-pi/snapcompact": "16.4.6", "@opentelemetry/api": "^1.9.1", "@opentelemetry/context-async-hooks": "^2.7.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.218.0", @@ -490,7 +490,7 @@ "@emnapi/core": ["@emnapi/core@1.11.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ=="], - "@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], + "@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA=="], @@ -1516,11 +1516,9 @@ "@opentelemetry/sdk-metrics/@opentelemetry/resources": ["@opentelemetry/resources@2.7.1", "", { "dependencies": { "@opentelemetry/core": "2.7.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ=="], - "@rolldown/binding-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], + "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ=="], - "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA=="], - - "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], + "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.2", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA=="], @@ -1536,7 +1534,7 @@ "cli-progress/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - "cli-truncate/string-width": ["string-width@8.2.1", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA=="], + "cli-truncate/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], "cliui/wrap-ansi": ["wrap-ansi@9.0.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww=="], @@ -1574,7 +1572,7 @@ "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], - "wrap-ansi/string-width": ["string-width@8.2.1", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA=="], + "wrap-ansi/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], diff --git a/crates/pi-natives/src/lib.rs b/crates/pi-natives/src/lib.rs index 881d589e0..b3d6a640a 100644 --- a/crates/pi-natives/src/lib.rs +++ b/crates/pi-natives/src/lib.rs @@ -248,7 +248,7 @@ fn create_windows_napi_tokio_runtime() -> Option { /// MUST stay in sync with `VERSION_SENTINEL_EXPORT` in /// `packages/natives/native/index.js` (which derives the name from /// `package.json#version`). -#[napi(js_name = "__piNativesV16_4_5")] +#[napi(js_name = "__piNativesV16_4_6")] pub const fn pi_natives_version_sentinel() {} /// Native module entry point: install crash diagnostics before any tool can diff --git a/package.json b/package.json index 3256bf5bc..e9c758e51 100644 --- a/package.json +++ b/package.json @@ -25,18 +25,18 @@ "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.7.0", - "@oh-my-pi/hashline": "16.4.5", - "@oh-my-pi/omp-stats": "16.4.5", - "@oh-my-pi/pi-agent-core": "16.4.5", - "@oh-my-pi/pi-ai": "16.4.5", - "@oh-my-pi/pi-catalog": "16.4.5", - "@oh-my-pi/pi-coding-agent": "16.4.5", - "@oh-my-pi/pi-mnemopi": "16.4.5", - "@oh-my-pi/pi-natives": "16.4.5", - "@oh-my-pi/pi-tui": "16.4.5", - "@oh-my-pi/pi-utils": "16.4.5", - "@oh-my-pi/pi-wire": "16.4.5", - "@oh-my-pi/snapcompact": "16.4.5", + "@oh-my-pi/hashline": "16.4.6", + "@oh-my-pi/omp-stats": "16.4.6", + "@oh-my-pi/pi-agent-core": "16.4.6", + "@oh-my-pi/pi-ai": "16.4.6", + "@oh-my-pi/pi-catalog": "16.4.6", + "@oh-my-pi/pi-coding-agent": "16.4.6", + "@oh-my-pi/pi-mnemopi": "16.4.6", + "@oh-my-pi/pi-natives": "16.4.6", + "@oh-my-pi/pi-tui": "16.4.6", + "@oh-my-pi/pi-utils": "16.4.6", + "@oh-my-pi/pi-wire": "16.4.6", + "@oh-my-pi/snapcompact": "16.4.6", "@opentelemetry/api": "^1.9.1", "@opentelemetry/context-async-hooks": "^2.7.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.218.0", diff --git a/packages/agent/package.json b/packages/agent/package.json index 24a0b7215..d2233f3be 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-agent-core", - "version": "16.4.5", + "version": "16.4.6", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index de037cadb..f72bb7642 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.6] - 2026-07-12 + ### Added - Added asynchronous `invalidateUsageCache` method to clear cached usage reports diff --git a/packages/ai/package.json b/packages/ai/package.json index 18ead39b3..3d25f92ff 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-ai", - "version": "16.4.5", + "version": "16.4.6", "description": "Unified LLM API with automatic model discovery and provider configuration", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/catalog/package.json b/packages/catalog/package.json index f2da06966..e5c62572d 100644 --- a/packages/catalog/package.json +++ b/packages/catalog/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-catalog", - "version": "16.4.5", + "version": "16.4.6", "description": "Model catalog for omp: bundled model database, provider discovery descriptors, model identity, classification, and equivalence", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 3d3300f69..87cd5f7e2 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.6] - 2026-07-12 + ### Added - Added `invalidate` action to the usage command to clear cached usage reports diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 247d4a6ca..5a808ccde 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-coding-agent", - "version": "16.4.5", + "version": "16.4.6", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/hashline/package.json b/packages/hashline/package.json index e7422f323..f218a2ad9 100644 --- a/packages/hashline/package.json +++ b/packages/hashline/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/hashline", - "version": "16.4.5", + "version": "16.4.6", "description": "Hashline: a compact, line-anchored patch language and applier. Pluggable FS/IO so it works over disk, in-memory, or any custom backend.", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/mnemopi/package.json b/packages/mnemopi/package.json index 2cd318cfe..d0a7ef18b 100644 --- a/packages/mnemopi/package.json +++ b/packages/mnemopi/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-mnemopi", - "version": "16.4.5", + "version": "16.4.6", "description": "Local SQLite memory engine for Oh My Pi agents", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index dc6b448c6..7f715115d 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,15 +2,17 @@ ## [Unreleased] -### Fixed - -- Fixed `ast_edit` rejecting byte-identical duplicate replacements as "Overlapping replacements detected": multiple rewrite ops matching the same node with the same output now collapse into one deterministic edit (deduped in both the preview listing/counts and the apply pass), so only genuinely divergent overlaps error. +## [16.4.6] - 2026-07-12 ### Added - Added an in-process `readlink` shell builtin (vendored from uutils coreutils 0.8.0), supporting `-f`/`-e`/`-m` canonicalization, `-n`/`-z` delimiters, and `-v`/`-q`/`-s` verbosity, with path operands resolved against the shell working directory. - Added in-process shell builtins for `realpath`, `touch`, `stat`, `date`, `mktemp`, `seq`, `yes`, `printenv`, `ln`, `truncate`, `tac`, `nproc`, `uname`, `whoami`, and `hostname` (vendored from uutils coreutils 0.8.0), plus native `which` (shell PATH lookup) and `diff` (unified output, `-U`/`-q`/`-N`, binary detection, recursive directory compare) builtins. All resolve path operands against the shell working directory, read the shell's exported environment, and honor abort/timeout cancellation; `ln` is gated with the destructive set (`PI_DISABLE_UUTILS_DESTRUCTIVE`), and system-mutating modes (`date --set`, hostname setting) are disabled. +### Fixed + +- Fixed `ast_edit` rejecting byte-identical duplicate replacements as "Overlapping replacements detected": multiple rewrite ops matching the same node with the same output now collapse into one deterministic edit (deduped in both the preview listing/counts and the apply pass), so only genuinely divergent overlaps error. + ## [16.4.5] - 2026-07-11 ### Added diff --git a/packages/natives/native/index.d.ts b/packages/natives/native/index.d.ts index b68526ad6..4e0394861 100644 --- a/packages/natives/native/index.d.ts +++ b/packages/natives/native/index.d.ts @@ -170,7 +170,7 @@ export declare function __ompInstallTokioRuntime(): void * `packages/natives/native/index.js` (which derives the name from * `package.json#version`). */ -export declare function __piNativesV16_4_5(): void +export declare function __piNativesV16_4_6(): void /** * Apply ast-grep rewrite rules to matching files; honors `dryRun` and returns diff --git a/packages/natives/native/index.js b/packages/natives/native/index.js index 8f21f078c..58f09faab 100644 --- a/packages/natives/native/index.js +++ b/packages/natives/native/index.js @@ -24,7 +24,7 @@ export const Shell = nativeBindings.Shell; // functions export const __ompInstallTokioRuntime = nativeBindings.__ompInstallTokioRuntime; -export const __piNativesV16_4_5 = nativeBindings.__piNativesV16_4_5; +export const __piNativesV16_4_6 = nativeBindings.__piNativesV16_4_6; export const astEdit = nativeBindings.astEdit; export const astGrep = nativeBindings.astGrep; export const astMatch = nativeBindings.astMatch; diff --git a/packages/natives/package.json b/packages/natives/package.json index ca2d8ae60..8555cc095 100644 --- a/packages/natives/package.json +++ b/packages/natives/package.json @@ -1,6 +1,6 @@ { "name": "@oh-my-pi/pi-natives", - "version": "16.4.5", + "version": "16.4.6", "description": "Native Rust bindings for grep, clipboard, image processing, syntax highlighting, PTY, and shell operations via N-API", "type": "module", "homepage": "https://omp.sh", diff --git a/packages/snapcompact/package.json b/packages/snapcompact/package.json index b79f6ee66..e22d389da 100644 --- a/packages/snapcompact/package.json +++ b/packages/snapcompact/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/snapcompact", - "version": "16.4.5", + "version": "16.4.6", "description": "Bitmap-frame context compression for vision-capable LLMs", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/stats/package.json b/packages/stats/package.json index f9c991c7b..e104e9f00 100644 --- a/packages/stats/package.json +++ b/packages/stats/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/omp-stats", - "version": "16.4.5", + "version": "16.4.6", "description": "Local observability dashboard for pi AI usage statistics", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/swarm-extension/package.json b/packages/swarm-extension/package.json index 2a616ced6..3e74a0598 100644 --- a/packages/swarm-extension/package.json +++ b/packages/swarm-extension/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/swarm-extension", - "version": "16.4.5", + "version": "16.4.6", "description": "Swarm orchestration extension for omp", "homepage": "https://omp.sh", "author": "Derek Rynd", diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 131733835..7ece1d17c 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.6] - 2026-07-12 + ### Added - Added support for width-changing editor text decorators on standalone presentation lines, with decorated output safely truncated to the available content width. diff --git a/packages/tui/package.json b/packages/tui/package.json index 546e14817..d1ed89a22 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-tui", - "version": "16.4.5", + "version": "16.4.6", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.md index c79689cfd..992cfb967 100644 --- a/packages/utils/CHANGELOG.md +++ b/packages/utils/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.6] - 2026-07-12 + ### Added - Added `AsyncDrain`, the deferred write-batching helper previously private to the coding-agent's prompt-history storage; now shared with model-perf recording. diff --git a/packages/utils/package.json b/packages/utils/package.json index 9852ab949..0093a8dde 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-utils", - "version": "16.4.5", + "version": "16.4.6", "description": "Shared utilities for pi packages", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/wire/package.json b/packages/wire/package.json index 7868b98e1..eb392d0c3 100644 --- a/packages/wire/package.json +++ b/packages/wire/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-wire", - "version": "16.4.5", + "version": "16.4.6", "description": "Shared wire protocol types for Oh My Pi packages", "homepage": "https://omp.sh", "author": "Can Boluk", From 20c0a2e4101d8507e7cbbaf547baa4f9f2340b73 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 03:13:15 +0200 Subject: [PATCH 21/80] test(coding-agent): fixed storage tests for schema v6 and slow CI disks - Compat tests assert the exported SCHEMA_VERSION instead of a stale hardcoded 5; the perf-sample change bumped agent.db to v6. - Backfill cap fixture inserts its 300 rows in one transaction; per-row implicit transactions fsynced 300 times and timed out on CI runners. --- packages/coding-agent/src/session/agent-storage.ts | 4 ++-- .../test/agent-storage-model-perf.test.ts | 13 ++++++++----- .../test/agent-storage-sqlite-compat.test.ts | 6 +++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/coding-agent/src/session/agent-storage.ts b/packages/coding-agent/src/session/agent-storage.ts index d8c6dae8d..345edafa7 100644 --- a/packages/coding-agent/src/session/agent-storage.ts +++ b/packages/coding-agent/src/session/agent-storage.ts @@ -115,8 +115,8 @@ function normalizeModelPerfSample(modelKey: string, sample: ModelPerfSample): Mo return { modelKey, outputTokens, durationMs, ttftSamples: ttftMs !== undefined ? 1 : 0, ttftMs: ttftMs ?? 0 }; } -/** Bump when schema changes require migration */ -const SCHEMA_VERSION = 6; +/** Current agent.db schema version; bump when schema changes require migration. */ +export const SCHEMA_VERSION = 6; const SQLITE_NOW_EPOCH = "CAST(strftime('%s','now') AS INTEGER)"; /** Singleton instances per database path */ diff --git a/packages/coding-agent/test/agent-storage-model-perf.test.ts b/packages/coding-agent/test/agent-storage-model-perf.test.ts index e0c450513..c1ab335e6 100644 --- a/packages/coding-agent/test/agent-storage-model-perf.test.ts +++ b/packages/coding-agent/test/agent-storage-model-perf.test.ts @@ -144,11 +144,14 @@ describe("AgentStorage model perf aggregates", () => { const insert = statsDb.prepare("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?, ?)"); const now = Date.now(); // 300 rows: the newest 256 run at 100 t/s, the older 44 at a wild - // 10000 t/s. Only the newest 256 may count. - for (let i = 0; i < 300; i++) { - const fast = i < 44; // smallest timestamps = oldest rows - insert.run("openai", "gpt-5", fast ? 10_000 : 100, 1000, null, "stop", now - (300 - i) * 1000); - } + // 10000 t/s. Only the newest 256 may count. One transaction: per-row + // implicit transactions fsync 300 times and time out on slow CI disks. + statsDb.transaction(() => { + for (let i = 0; i < 300; i++) { + const fast = i < 44; // smallest timestamps = oldest rows + insert.run("openai", "gpt-5", fast ? 10_000 : 100, 1000, null, "stop", now - (300 - i) * 1000); + } + })(); statsDb.close(); const imported = await storage.backfillModelPerfFromStats(statsDbPath); diff --git a/packages/coding-agent/test/agent-storage-sqlite-compat.test.ts b/packages/coding-agent/test/agent-storage-sqlite-compat.test.ts index 522aac4ab..290da4024 100644 --- a/packages/coding-agent/test/agent-storage-sqlite-compat.test.ts +++ b/packages/coding-agent/test/agent-storage-sqlite-compat.test.ts @@ -1,7 +1,7 @@ import { Database } from "bun:sqlite"; import { afterEach, describe, expect, it } from "bun:test"; import * as path from "node:path"; -import { AgentStorage } from "@oh-my-pi/pi-coding-agent/session/agent-storage"; +import { AgentStorage, SCHEMA_VERSION } from "@oh-my-pi/pi-coding-agent/session/agent-storage"; import { TempDir } from "@oh-my-pi/pi-utils"; import { readTableSql } from "./helpers/sqlite-inspect"; @@ -53,7 +53,7 @@ describe("AgentStorage SQLite compatibility", () => { storage.recordModelUsage("openai/gpt-5"); expect(storage.getModelUsageOrder()).toEqual(["openai/gpt-5"]); - expect(readSchemaVersion(dbPath)).toBe(5); + expect(readSchemaVersion(dbPath)).toBe(SCHEMA_VERSION); expect(readTableSql(dbPath, "settings")).not.toContain("unixepoch("); expect(readTableSql(dbPath, "settings")).toContain("strftime('%s','now')"); expect(readTableSql(dbPath, "model_usage")).not.toContain("unixepoch("); @@ -87,7 +87,7 @@ describe("AgentStorage SQLite compatibility", () => { const storage = await AgentStorage.open(dbPath); - expect(readSchemaVersion(dbPath)).toBe(5); + expect(readSchemaVersion(dbPath)).toBe(SCHEMA_VERSION); expect(readTableSql(dbPath, "settings")).not.toContain("unixepoch("); expect(readTableSql(dbPath, "settings")).toContain("strftime('%s','now')"); expect(readTableSql(dbPath, "model_usage")).not.toContain("unixepoch("); From b1c882e89c2a089a1b91ce4894deb4ba3a5170ba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:42:14 +0000 Subject: [PATCH 22/80] Update VOUCHED list https://github.com/can1357/oh-my-pi/discussions/4944#discussioncomment-DC_kwDOQxs0bc4BDLtw --- .github/VOUCHED.td | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td index b767e973a..e8c1d6be5 100644 --- a/.github/VOUCHED.td +++ b/.github/VOUCHED.td @@ -133,6 +133,7 @@ tdiant tjboudreaux tsagi2045 turbomolli +unravl usr-bin-roygbiv vmcall voidchecksum From 1b0b18c7a6846e618a4b7dcdbaa2ec34ce06a335 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 12:42:32 +0200 Subject: [PATCH 23/80] fix(stats): handled malformed session entries to prevent sync crashes - Coerce missing `stopReason`, token counts, and timestamps at the parser boundary to satisfy database NOT NULL constraints. - Skip assistant entries missing essential model attribution or usage data instead of aborting the sync process. - Filter invalid tool call blocks that lack required identifiers to ensure successful database insertion. --- packages/stats/CHANGELOG.md | 4 + packages/stats/src/parser.ts | 58 ++++++++- .../test/parser-malformed-entries.test.ts | 119 ++++++++++++++++++ 3 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 packages/stats/test/parser-malformed-entries.test.ts diff --git a/packages/stats/CHANGELOG.md b/packages/stats/CHANGELOG.md index 5897b909c..76f3de847 100644 --- a/packages/stats/CHANGELOG.md +++ b/packages/stats/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed a `SQLITE_CONSTRAINT_NOTNULL` crash (`messages.stop_reason`) aborting the entire session sync when a persisted assistant message lacks a `stopReason`. Malformed entries — missing stop reason, token counts, or message timestamp — are now coerced at the parser boundary, and entries with no usage or model attribution are skipped instead of failing the batch insert. + ## [16.4.2] - 2026-07-10 ### Fixed diff --git a/packages/stats/src/parser.ts b/packages/stats/src/parser.ts index ce590119f..897de549f 100644 --- a/packages/stats/src/parser.ts +++ b/packages/stats/src/parser.ts @@ -6,7 +6,9 @@ import { getPriorityPremiumRequests, resolveModelServiceTier, type ServiceTierByFamily, + type ToolCall, type ToolResultMessage, + type Usage, } from "@oh-my-pi/pi-ai"; import { getSessionsDir, isEnoent, readLines } from "@oh-my-pi/pi-utils"; import type { @@ -142,6 +144,14 @@ function extractUserStats(sessionFile: string, folder: string, entry: SessionMes /** * Extract stats from an assistant message entry. + * + * Session JSONL on disk is not guaranteed to match the current + * `AssistantMessage` shape: crash-truncated turns, sessions written by older + * versions, and foreign producers all flow through this parser. Every field + * returned here feeds a NOT NULL column in stats.db, so malformed entries are + * coerced (missing `stopReason`, token counts, `timestamp`) or skipped + * (missing `model`/`provider`/`api`/`usage`) instead of crashing the whole + * sync with a constraint violation. */ function extractStats( sessionFile: string, @@ -152,6 +162,9 @@ function extractStats( ): MessageStats | null { const msg = entry.message as AssistantMessage; if (msg?.role !== "assistant") return null; + if (typeof msg.model !== "string" || typeof msg.provider !== "string" || typeof msg.api !== "string") return null; + const rawUsage = msg.usage as Partial | undefined; + if (!rawUsage || typeof rawUsage !== "object") return null; // Backfill: when the session recorded `priority` as the active service tier // at this point but the AI usage payload was captured before priority @@ -159,11 +172,29 @@ function extractStats( // "Premium Reqs" stat aggregates priority traffic on re-sync. Trust any // non-zero value already in `usage.premiumRequests` (Copilot multipliers or // the new AI code path) and only synthesise when the field is missing/zero. - const recorded = msg.usage.premiumRequests ?? 0; + const recorded = rawUsage.premiumRequests ?? 0; const model = { provider: msg.provider, api: msg.api, id: msg.model }; const tier = resolveModelServiceTier(currentServiceTier, model); const derived = recorded > 0 ? recorded : getPriorityPremiumRequests(tier, model); - const usage = derived === recorded ? msg.usage : { ...msg.usage, premiumRequests: derived }; + const wellFormed = + typeof rawUsage.input === "number" && + typeof rawUsage.output === "number" && + typeof rawUsage.cacheRead === "number" && + typeof rawUsage.cacheWrite === "number" && + typeof rawUsage.totalTokens === "number"; + const usage: Usage = + wellFormed && derived === recorded + ? (rawUsage as Usage) + : { + ...rawUsage, + input: rawUsage.input ?? 0, + output: rawUsage.output ?? 0, + cacheRead: rawUsage.cacheRead ?? 0, + cacheWrite: rawUsage.cacheWrite ?? 0, + totalTokens: rawUsage.totalTokens ?? 0, + cost: rawUsage.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + premiumRequests: derived, + }; return { sessionFile, @@ -172,16 +203,25 @@ function extractStats( model: msg.model, provider: msg.provider, api: msg.api, - timestamp: msg.timestamp, + timestamp: coerceEntryTimestamp(msg.timestamp, entry), duration: msg.duration ?? null, ttft: msg.ttft ?? null, - stopReason: msg.stopReason, + // A message persisted without a terminal stop reason never completed + // normally: classify by whether it carried an error. + stopReason: msg.stopReason ?? (msg.errorMessage ? "error" : "aborted"), errorMessage: msg.errorMessage ?? null, usage, agentType, }; } +/** Message timestamp, falling back to the entry's ISO timestamp, then 0. */ +function coerceEntryTimestamp(timestamp: number | undefined, entry: SessionMessageEntry): number { + if (typeof timestamp === "number" && Number.isFinite(timestamp)) return timestamp; + const ts = Date.parse(entry.timestamp); + return Number.isFinite(ts) ? ts : 0; +} + /** * Extract one {@link ToolCallStats} per `toolCall` content block of an * assistant message. Returns an empty array for turns without tool calls. @@ -194,8 +234,14 @@ function extractToolCalls( ): ToolCallStats[] { const msg = entry.message as AssistantMessage; if (msg?.role !== "assistant" || !Array.isArray(msg.content)) return []; + // `tool_calls` columns are NOT NULL: skip turns that can't be attributed + // (malformed persisted entries — see extractStats) and blocks missing ids. + if (typeof msg.model !== "string" || typeof msg.provider !== "string") return []; - const blocks = msg.content.filter(block => block.type === "toolCall"); + const blocks = msg.content.filter( + (block): block is ToolCall => + block.type === "toolCall" && typeof block.id === "string" && typeof block.name === "string", + ); if (blocks.length === 0) return []; return blocks.map(block => { @@ -213,7 +259,7 @@ function extractToolCalls( toolName: block.name, model: msg.model, provider: msg.provider, - timestamp: msg.timestamp, + timestamp: coerceEntryTimestamp(msg.timestamp, entry), agentType, callsInTurn: blocks.length, argsChars, diff --git a/packages/stats/test/parser-malformed-entries.test.ts b/packages/stats/test/parser-malformed-entries.test.ts new file mode 100644 index 000000000..ee18a19ef --- /dev/null +++ b/packages/stats/test/parser-malformed-entries.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { initDb, insertMessageStats, insertToolCalls } from "@oh-my-pi/omp-stats/db"; +import { parseSessionFile } from "@oh-my-pi/omp-stats/parser"; +import { getSessionsDir } from "@oh-my-pi/pi-utils"; +import { installStatsTestIsolation } from "./helpers/temp-agent"; + +installStatsTestIsolation("@pi-stats-malformed-"); + +const USAGE = { + input: 10, + output: 20, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 30, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +function assistantEntry(id: string, message: Record): string { + return JSON.stringify({ + type: "message", + id, + timestamp: "2026-07-12T00:00:00.000Z", + message: { + role: "assistant", + api: "anthropic-messages", + provider: "anthropic", + model: "claude-fable-5", + ...message, + }, + }); +} + +async function writeSession(lines: string[]): Promise { + const dir = path.join(getSessionsDir(), "--tmp--malformed"); + await fs.mkdir(dir, { recursive: true }); + const file = path.join(dir, "session.jsonl"); + await Bun.write(file, `${lines.join("\n")}\n`); + return file; +} + +// Regression: a single persisted assistant message missing `stopReason` (or +// usage/token fields) used to bind NULL into stats.db's NOT NULL columns and +// crash the entire sync with SQLITE_CONSTRAINT_NOTNULL. The parser must +// coerce or skip malformed entries so the batch always inserts. +describe("malformed session entries", () => { + it("coerces a missing stopReason instead of failing the NOT NULL insert", async () => { + const file = await writeSession([ + assistantEntry("a1", { content: [{ type: "text", text: "hi" }], usage: USAGE, timestamp: 1752000000000 }), + assistantEntry("a2", { + content: [], + usage: USAGE, + timestamp: 1752000001000, + errorMessage: "boom", + }), + ]); + + const result = await parseSessionFile(file); + expect(result.stats.map(s => s.stopReason)).toEqual(["aborted", "error"]); + + await initDb(); + expect(insertMessageStats(result.stats)).toBe(2); + }); + + it("zero-fills missing token counts and falls back to the entry timestamp", async () => { + const file = await writeSession([ + assistantEntry("a1", { + content: [], + stopReason: "stop", + usage: { cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + }), + ]); + + const result = await parseSessionFile(file); + expect(result.stats).toHaveLength(1); + const stats = result.stats[0]; + expect(stats.usage.totalTokens).toBe(0); + expect(stats.timestamp).toBe(Date.parse("2026-07-12T00:00:00.000Z")); + + await initDb(); + expect(insertMessageStats(result.stats)).toBe(1); + }); + + it("skips assistant entries with no usage or model attribution", async () => { + const file = await writeSession([ + assistantEntry("a1", { content: [], stopReason: "stop" }), + JSON.stringify({ + type: "message", + id: "a2", + timestamp: "2026-07-12T00:00:00.000Z", + message: { role: "assistant", content: [], stopReason: "stop", usage: USAGE }, + }), + assistantEntry("ok", { content: [], stopReason: "stop", usage: USAGE, timestamp: 1752000002000 }), + ]); + + const result = await parseSessionFile(file); + expect(result.stats.map(s => s.entryId)).toEqual(["ok"]); + }); + + it("keeps tool_calls insertable when the turn lacks a message timestamp", async () => { + const file = await writeSession([ + assistantEntry("a1", { + content: [ + { type: "toolCall", id: "call-1", name: "bash", arguments: { command: "ls" } }, + { type: "toolCall", name: "broken" }, // no id: unattributable, must be skipped + ], + usage: USAGE, + }), + ]); + + const result = await parseSessionFile(file); + expect(result.toolCalls.map(c => c.toolCallId)).toEqual(["call-1"]); + expect(result.toolCalls[0].timestamp).toBe(Date.parse("2026-07-12T00:00:00.000Z")); + + await initDb(); + expect(insertToolCalls(result.toolCalls)).toBe(1); + }); +}); From 4fa5b61b05e6f053ab0b4a412a211d1affbee1d3 Mon Sep 17 00:00:00 2001 From: Hayden Evan Date: Thu, 9 Jul 2026 21:29:55 +0700 Subject: [PATCH 24/80] Add plan review copy hotkey --- packages/coding-agent/CHANGELOG.md | 4 +++ .../modes/components/plan-review-overlay.ts | 7 ++++ .../src/modes/interactive-mode.ts | 13 +++++++ .../test/interactive-mode-plan-review.test.ts | 34 ++++++++++++++++++ .../components/plan-review-overlay.test.ts | 35 +++++++++++++++++++ 5 files changed, 93 insertions(+) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 87cd5f7e2..9ba855d41 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added a `c` hotkey in the plan-review overlay that copies the current reviewed plan markdown to the system clipboard, including in-overlay edits. + ## [16.4.6] - 2026-07-12 ### Added diff --git a/packages/coding-agent/src/modes/components/plan-review-overlay.ts b/packages/coding-agent/src/modes/components/plan-review-overlay.ts index 11f7a96d2..4d286cab6 100644 --- a/packages/coding-agent/src/modes/components/plan-review-overlay.ts +++ b/packages/coding-agent/src/modes/components/plan-review-overlay.ts @@ -82,6 +82,8 @@ export interface PlanReviewOverlayCallbacks { onPick: (label: string) => void; /** Invoked on Esc / cancel. */ onCancel: () => void; + /** Invoked with the current full plan text when the copy hotkey is pressed. */ + onCopyPlan?: (content: string) => void | Promise; /** Invoked when the external-editor key is pressed (overlay stays open). */ onExternalEditor?: () => void; /** Invoked when the external-editor key edits the active annotation draft. */ @@ -302,6 +304,10 @@ export class PlanReviewOverlay implements Component { this.callbacks.onExternalEditor(); return; } + if (this.callbacks.onCopyPlan && keyData === "c") { + void this.callbacks.onCopyPlan(joinPlanSections(this.#sections)); + return; + } if (matchesKey(keyData, "tab") || keyData === "\t") { this.#cycleRegion(1); return; @@ -677,6 +683,7 @@ export class PlanReviewOverlay implements Component { parts.push("↑↓ scroll", "⇧ faster", "pgup/pgdn", "g/G ends"); break; } + if (this.callbacks.onCopyPlan) parts.push("c copy"); parts.push("tab regions"); if (this.#externalEditorLabel && this.#focus !== "toc") parts.push(`${this.#externalEditorLabel} editor`); parts.push(this.#helpSuffix); diff --git a/packages/coding-agent/src/modes/interactive-mode.ts b/packages/coding-agent/src/modes/interactive-mode.ts index dc4716dec..c53bea59b 100644 --- a/packages/coding-agent/src/modes/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive-mode.ts @@ -120,6 +120,7 @@ import { formatPhaseDisplayName, todoMatchesAnyDescription } from "../tools/todo import { ToolError } from "../tools/tool-errors"; import { vocalizer } from "../tts/vocalizer"; import { renderTreeList } from "../tui/tree-list"; +import { copyToClipboard } from "../utils/clipboard"; import type { EventBus } from "../utils/event-bus"; import { getEditorCommand, openInEditor } from "../utils/external-editor"; import { getSessionAccentAnsi, getSessionAccentHex } from "../utils/session-color"; @@ -2517,6 +2518,7 @@ export class InteractiveMode implements InteractiveModeContext { { onPick: choice => finish(choice), onCancel: () => finish(undefined), + onCopyPlan: content => void this.#copyPlanToClipboard(content), onExternalEditor: dialogOptions?.onExternalEditor, onAnnotationExternalEditor: (draft, commit) => void this.#openPlanAnnotationInExternalEditor(draft, commit), onPlanEdited: dialogOptions?.onPlanEdited, @@ -2583,6 +2585,17 @@ export class InteractiveMode implements InteractiveModeContext { return contextUsage !== undefined && contextUsage.percent > PLAN_KEEP_CONTEXT_DISABLE_THRESHOLD_PERCENT; } + async #copyPlanToClipboard(content: string): Promise { + try { + await copyToClipboard(content); + this.showStatus("Copied plan to clipboard"); + } catch (error) { + this.showWarning( + `Failed to copy plan to clipboard: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + async #openPlanInExternalEditor(planFilePath: string): Promise { const editorCmd = getEditorCommand(); if (!editorCmd) { diff --git a/packages/coding-agent/test/interactive-mode-plan-review.test.ts b/packages/coding-agent/test/interactive-mode-plan-review.test.ts index 6ebf4a59a..7590f3a58 100644 --- a/packages/coding-agent/test/interactive-mode-plan-review.test.ts +++ b/packages/coding-agent/test/interactive-mode-plan-review.test.ts @@ -18,6 +18,7 @@ import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage"; import { SILENT_ABORT_MARKER, USER_INTERRUPT_LABEL } from "@oh-my-pi/pi-coding-agent/session/messages"; import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager"; import { AUTO_THINKING } from "@oh-my-pi/pi-coding-agent/thinking"; +import * as clipboard from "@oh-my-pi/pi-coding-agent/utils/clipboard"; import { setKeybindings, Text } from "@oh-my-pi/pi-tui"; import { formatNumber, TempDir } from "@oh-my-pi/pi-utils"; @@ -332,6 +333,39 @@ describe("InteractiveMode plan review rendering", () => { } }); + it("copies the overlay's current edited plan markdown from the real plan review overlay", async () => { + let capturedOverlay: PlanReviewOverlay | undefined; + const overlayHandle = { hide: vi.fn() }; + vi.spyOn(mode.ui, "showOverlay").mockImplementation(component => { + capturedOverlay = component as PlanReviewOverlay; + return overlayHandle as never; + }); + const copySpy = vi.spyOn(clipboard, "copyToClipboard").mockResolvedValue(undefined); + const statusSpy = vi.spyOn(mode, "showStatus"); + const constructorPlan = "# Plan\n\nOriginal constructor body.\n"; + const editedPlan = "# Plan\n\nEdited in overlay.\n\n## Verify\n\n- run focused test\n"; + + const choice = mode.showPlanReview(constructorPlan, "Plan mode - next step", [ + "Approve and execute", + "Refine plan", + ]); + + expect(capturedOverlay).toBeDefined(); + const overlay = capturedOverlay!; + overlay.setPlanContent(editedPlan); + overlay.handleInput("c"); + await Promise.resolve(); + + expect(copySpy).toHaveBeenCalledTimes(1); + expect(copySpy).toHaveBeenCalledWith(editedPlan); + expect(copySpy).not.toHaveBeenCalledWith(constructorPlan); + expect(statusSpy).toHaveBeenCalledWith("Copied plan to clipboard"); + + overlay.handleInput("\x1b"); + await expect(choice).resolves.toBeUndefined(); + expect(overlayHandle.hide).toHaveBeenCalled(); + }); + it("Refine with no annotations silently aborts approval and returns to the editor", async () => { const planFilePath = "local://PLAN.md"; const resolvedPlanPath = resolveLocalUrlToPath(planFilePath, { diff --git a/packages/coding-agent/test/modes/components/plan-review-overlay.test.ts b/packages/coding-agent/test/modes/components/plan-review-overlay.test.ts index ef9841fd4..37733fee2 100644 --- a/packages/coding-agent/test/modes/components/plan-review-overlay.test.ts +++ b/packages/coding-agent/test/modes/components/plan-review-overlay.test.ts @@ -195,6 +195,22 @@ describe("PlanReviewOverlay", () => { expect(out).not.toContain("para 199"); }); + it("copies the current plan content on c and advertises the hotkey when available", () => { + const onCopyPlan = vi.fn(); + const overlay = new PlanReviewOverlay( + "# Original plan\n\nold body", + { promptTitle: "next", options: APPROVAL_OPTIONS }, + { onPick: vi.fn(), onCancel: vi.fn(), onCopyPlan }, + ); + + overlay.setPlanContent("# Edited plan\n\nnew body"); + expect(render(overlay)).toContain("c copy"); + overlay.handleInput("c"); + + expect(onCopyPlan).toHaveBeenCalledTimes(1); + expect(onCopyPlan).toHaveBeenCalledWith("# Edited plan\n\nnew body\n"); + }); + // Plan with ≥2 headings + nesting, wide enough for the sidebar at width 80. const SECTION_PLAN = "# Overview\n\nintro body\n\n## Goal\n\ngoal body\n\n## Steps\n\nstep body\n\n# Risks\n\nrisk body\n"; @@ -322,6 +338,25 @@ describe("PlanReviewOverlay", () => { expect(restored).toContain("goal body"); }); + it("copies the edited plan after deleting a section in the overlay", () => { + const onCopyPlan = vi.fn(); + const overlay = new PlanReviewOverlay( + SECTION_PLAN, + { promptTitle: "next", options: APPROVAL_OPTIONS }, + { onPick: vi.fn(), onCancel: vi.fn(), onCopyPlan }, + ); + render(overlay); + overlay.handleInput(TAB); // -> toc (Overview) + overlay.handleInput(DOWN); // -> Goal + overlay.handleInput("d"); + overlay.handleInput("c"); + + expect(onCopyPlan).toHaveBeenCalledTimes(1); + expect(onCopyPlan).toHaveBeenCalledWith( + "# Overview\n\nintro body\n\n## Steps\n\nstep body\n\n# Risks\n\nrisk body\n", + ); + }); + it("annotates a section and emits feedback for the Refine loop", () => { const onFeedbackChange = vi.fn(); const overlay = new PlanReviewOverlay( From 1822603b2d46810e50270cd21862a8ce935bc7ba Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 13:05:01 +0200 Subject: [PATCH 25/80] feat(coding-agent): improved model browser keyboard navigation and focus visuals - Added support for home, end, and page navigation in the model browser. - Restricted the hover background band to mouse interactions, using cursor glyphs and text accents for keyboard selection instead. - Synchronized focus states between the model hub sidebar and the browser pane. - Removed redundant "login" labels from locked provider entries. - Prevented navigation keys from triggering the input-priority grace period, ensuring responsive movement after idle states. - Limited the input queue-drain delay to Ctrl+C and Escape double-press gestures. --- packages/coding-agent/CHANGELOG.md | 7 ++ .../src/modes/components/model-browser.ts | 71 +++++++++++-------- .../src/modes/components/model-hub.ts | 58 ++++++++------- packages/coding-agent/test/model-hub.test.ts | 13 ++-- packages/tui/CHANGELOG.md | 4 ++ packages/tui/src/tui.ts | 12 ++-- packages/tui/test/input-priority.test.ts | 36 ++++++++++ 7 files changed, 134 insertions(+), 67 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 9ba855d41..74b448c2a 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,8 +4,15 @@ ### Added +- Enabled Home and End keyboard navigation in the model browser - Added a `c` hotkey in the plan-review overlay that copies the current reviewed plan markdown to the system clipboard, including in-overlay edits. +### Changed + +- Streamlined list view styling by removing inline model role chips from row entries +- Clarified active selection state by only displaying cursor and highlight bands in focused panes +- Removed the redundant "login" label from inactive (locked) provider entries in the Model Hub sidebar + ## [16.4.6] - 2026-07-12 ### Added diff --git a/packages/coding-agent/src/modes/components/model-browser.ts b/packages/coding-agent/src/modes/components/model-browser.ts index 4f37b196f..1ae63ee24 100644 --- a/packages/coding-agent/src/modes/components/model-browser.ts +++ b/packages/coding-agent/src/modes/components/model-browser.ts @@ -293,6 +293,8 @@ export class ModelBrowser implements Component { /** First visible list row; panned by the wheel, snapped to the selection on keyboard navigation. */ #windowStart = 0; #windowCount = 0; + /** Whether the host pane owns arrow keys; drives cursor strength and the selected-row band. */ + #focused = true; /** Enter or click-on-selected. */ onActivate?: (item: ModelBrowserItem) => void; @@ -345,6 +347,10 @@ export class ModelBrowser implements Component { setShowProvider(show: boolean): void { this.#showProvider = show; } + /** Focused: accent cursor + selected-row background band. Unfocused: dim cursor, no band. */ + setFocused(focused: boolean): void { + this.#focused = focused; + } /** Total rendered height for the current `maxVisible` (host layout budgeting). */ get renderedRows(): number { @@ -416,18 +422,27 @@ export class ModelBrowser implements Component { this.#windowStart = this.#clampWindowStart(this.#windowStart); } - moveSelection(delta: number): void { + /** + * Move the selection by `delta` rows, skipping disabled rows. Single steps + * wrap at the ends; `wrap: false` (page/home/end jumps) clamps instead. + */ + moveSelection(delta: number, options: { wrap?: boolean } = {}): void { const count = this.#visibleItems.length; if (count === 0) return; - let index = this.#selectedIndex; - for (let step = 0; step < count; step++) { - index = (index + delta + count) % count; - const item = this.#visibleItems[index]; - if (item && !this.#isDisabled(item)) { - this.#setSelectedIndex(index); - return; + if (options.wrap ?? true) { + let index = this.#selectedIndex; + for (let step = 0; step < count; step++) { + index = (index + delta + count) % count; + const item = this.#visibleItems[index]; + if (item && !this.#isDisabled(item)) { + this.#setSelectedIndex(index); + return; + } } + return; } + const target = Math.max(0, Math.min(this.#selectedIndex + delta, count - 1)); + this.#setSelectedIndex(this.#coerceSelectedIndex(target)); } #setSelectedIndex(index: number): void { @@ -514,11 +529,19 @@ export class ModelBrowser implements Component { return; } if (matchesSelectPageUp(data)) { - this.moveSelection(-this.#maxVisible); + this.moveSelection(-this.#maxVisible, { wrap: false }); return; } if (matchesSelectPageDown(data)) { - this.moveSelection(this.#maxVisible); + this.moveSelection(this.#maxVisible, { wrap: false }); + return; + } + if (matchesKey(data, "home")) { + this.moveSelection(-this.#visibleItems.length, { wrap: false }); + return; + } + if (matchesKey(data, "end")) { + this.moveSelection(this.#visibleItems.length, { wrap: false }); return; } if (matchesKey(data, "enter") || matchesKey(data, "return") || data === "\n") { @@ -574,6 +597,10 @@ export class ModelBrowser implements Component { this.#setSelectedIndex(index); } } + /** Drop the hover band. Hosts call this when the pointer leaves the browser pane. */ + clearHover(): void { + this.#hoveredIndex = null; + } /** List index under a frame-local row, or null when off-list or on a disabled row. */ #hoverIndexAt(line: number): number | null { @@ -585,22 +612,6 @@ export class ModelBrowser implements Component { return index; } - #chipsFor(model: Model): string { - const parts: string[] = []; - const seen = new Set(); - const pushChip = (role: string) => { - if (seen.has(role)) return; - seen.add(role); - const assignment = this.#roles[role]; - if (!assignment || !modelsAreEqual(assignment.model, model)) return; - if (getRoleInfo(role, this.#settings).hidden) return; - parts.push(formatRoleChip(role, assignment, this.#settings)); - }; - for (const role of MODEL_ROLE_IDS) pushChip(role); - for (const role in this.#roles) pushChip(role); - return parts.length > 0 ? ` ${parts.join(" ")}` : ""; - } - /** `0.9s 118t/s` measured-perf cell for the row's meta block; empty when unmeasured or the column is off. */ #perfCell(item: ModelBrowserItem, mode: PerfMode): string { if (mode === "off") return ""; @@ -627,13 +638,13 @@ export class ModelBrowser implements Component { return ` ${line} `; } const disabled = this.#isDisabled(item); - const prefix = selected ? `${theme.fg("accent", theme.nav.cursor)} ` : " "; + const prefix = selected && this.#focused ? `${theme.fg("accent", theme.nav.cursor)} ` : " "; const providerPrefix = this.#showProvider ? theme.fg("dim", `${item.provider}/`) : ""; const name = selected ? theme.fg("accent", item.id) : item.id; const overLimit = disabled ? ` ${theme.status.disabled} context>${formatNumber(item.model.contextWindow ?? 0).toLowerCase()}` : ""; - let left = `${prefix}${providerPrefix}${name}${this.#chipsFor(item.model)}${overLimit}`; + let left = `${prefix}${providerPrefix}${name}${overLimit}`; // Perf column collapses entirely when no visible row has measurements. const perfCol = @@ -648,7 +659,9 @@ export class ModelBrowser implements Component { if (disabled) { line = theme.fg("dim", Bun.stripANSI(line)); } - if (hovered && !selected && !disabled) { + // The bg band is reserved for the mouse: it marks hover, nothing else. + // Keyboard selection is the cursor glyph + accent name. + if (hovered && !disabled) { line = theme.bg("selectedBg", line); } return line; diff --git a/packages/coding-agent/src/modes/components/model-hub.ts b/packages/coding-agent/src/modes/components/model-hub.ts index 7771baf5f..8ddf6ec08 100644 --- a/packages/coding-agent/src/modes/components/model-hub.ts +++ b/packages/coding-agent/src/modes/components/model-hub.ts @@ -461,7 +461,7 @@ export class ModelHubComponent implements Component { label: providerId, providerId, locked: isLocked, - annotation: isLocked ? "login" : String(availableCounts.get(providerId) ?? 0), + annotation: isLocked ? undefined : String(availableCounts.get(providerId) ?? 0), oauth: oauthIds.has(providerId), catalogCount: catalogCounts.get(providerId) ?? 0, }); @@ -1468,6 +1468,10 @@ export class ModelHubComponent implements Component { this.#roleHover = null; if (overBody && this.#isBrowserView(entry)) { this.#browser.routeMouse(event, bodyLine); + } else { + // Pointer left the browser pane: without this, the last + // hovered row keeps its band while the sidebar hovers too. + this.#browser.clearHover(); } } return true; @@ -1578,11 +1582,10 @@ export class ModelHubComponent implements Component { // While searching, entries the hop skips gray out: locked and // zero-match providers, an empty Recent, and the Roles view. const muted = entry.locked || matchCount === 0 || (searching && entry.kind === "roles"); - const cursor = active - ? this.#focus === "scope" - ? theme.fg("accent", theme.nav.cursor) - : theme.fg("dim", theme.nav.cursor) - : " "; + // The sidebar's active entry is state, not a cursor: accent label + // plus a cursor glyph while the sidebar owns the arrows. The band + // stays in the body pane so the two never look alike. + const cursor = active && this.#focus === "scope" ? theme.fg("accent", theme.nav.cursor) : " "; let icon: string; if (entry.kind === "recent") { @@ -1597,7 +1600,7 @@ export class ModelHubComponent implements Component { const labelStyled = muted ? theme.fg("dim", entry.label) : active - ? theme.fg("accent", entry.label) + ? theme.bold(theme.fg("accent", entry.label)) : entry.label; const refreshing = entry.providerId ? this.#refreshingProviders.has(entry.providerId) : false; @@ -1614,8 +1617,10 @@ export class ModelHubComponent implements Component { line = `${left}${" ".repeat(width - leftWidth - annWidth)}${annotationStyled}`; } else { line = truncateToWidth(left, width); + const lineWidth = visibleWidth(line); + if (lineWidth < width) line += " ".repeat(width - lineWidth); } - if (hovered && !active) { + if (hovered) { line = theme.bg("selectedBg", line); } lines.push(line); @@ -1675,6 +1680,17 @@ export class ModelHubComponent implements Component { return truncateToWidth(theme.fg("muted", ` ${text}`), width); } + /** Clamp a roles row to `width`; the bg band is reserved for mouse hover. */ + #finishRolesRow(line: string, width: number, hovered: boolean): string { + let out = truncateToWidth(line, width); + if (hovered) { + const w = visibleWidth(out); + if (w < width) out += " ".repeat(width - w); + return theme.bg("selectedBg", out); + } + return out; + } + #renderRolesView(width: number, rows: number): string[] { const lines: string[] = []; lines.push(""); @@ -1691,12 +1707,14 @@ export class ModelHubComponent implements Component { } const cycleOrder = this.#cycleOrder(); + const listFocused = this.#focus === "list"; for (let i = 0; i < this.#rolesRows.length && lines.length < rows - 2; i++) { const rowDef = this.#rolesRows[i]; if (!rowDef) continue; const selected = i === this.#roleIndex; const hovered = i === this.#roleHover; - const cursor = selected ? theme.fg("accent", theme.nav.cursor) : " "; + // The unfocused pane draws no cursor; accent text still marks the row. + const cursor = selected && listFocused ? theme.fg("accent", theme.nav.cursor) : " "; if (rowDef.kind === "separator") { lines.push(` ${theme.fg("border", "─".repeat(Math.max(1, width - 6)))}`); @@ -1706,10 +1724,7 @@ export class ModelHubComponent implements Component { if (rowDef.kind === "newRole" || rowDef.kind === "newFallback") { const label = rowDef.kind === "newRole" ? "+ New role…" : "+ New fallback…"; let line = ` ${cursor} ${theme.fg(selected ? "accent" : "dim", label)}`; - line = truncateToWidth(line, width); - if (hovered && !selected) { - line = theme.bg("selectedBg", line); - } + line = this.#finishRolesRow(line, width, hovered); lines.push(line); continue; } @@ -1720,10 +1735,7 @@ export class ModelHubComponent implements Component { const tail = key.slice(slash + 1); const keyStyled = theme.fg("dim", key.slice(0, slash + 1)) + (selected ? theme.fg("accent", tail) : tail); let line = ` ${cursor} ${theme.fg("dim", theme.status.shadowed)} ${keyStyled}`; - line = truncateToWidth(line, width); - if (hovered && !selected) { - line = theme.bg("selectedBg", line); - } + line = this.#finishRolesRow(line, width, hovered); lines.push(line); continue; } @@ -1732,10 +1744,7 @@ export class ModelHubComponent implements Component { const branch = theme.fg("dim", `${"".padEnd(tagWidth + 3)}↳`); const selector = selected ? theme.fg("accent", rowDef.selector) : theme.fg("muted", rowDef.selector); let line = ` ${cursor} ${branch} ${selector}`; - line = truncateToWidth(line, width); - if (hovered && !selected) { - line = theme.bg("selectedBg", line); - } + line = this.#finishRolesRow(line, width, hovered); lines.push(line); continue; } @@ -1778,12 +1787,8 @@ export class ModelHubComponent implements Component { const lineWidth = visibleWidth(line); if (rightWidth > 0 && lineWidth + rightWidth + 2 <= width) { line = `${line}${" ".repeat(width - lineWidth - rightWidth - 1)}${right}`; - } else { - line = truncateToWidth(line, width); - } - if (hovered && !selected) { - line = theme.bg("selectedBg", line); } + line = this.#finishRolesRow(line, width, hovered); lines.push(line); } @@ -1981,6 +1986,7 @@ export class ModelHubComponent implements Component { bodyLines.push(...this.#renderLockedView(entry, bodyWidth, contentRows - 1)); } else { this.#browser.setMaxVisible(contentRows - 1 - 5); + this.#browser.setFocused(this.#focus === "list"); bodyLines.push(...this.#browser.render(bodyWidth)); } diff --git a/packages/coding-agent/test/model-hub.test.ts b/packages/coding-agent/test/model-hub.test.ts index 93d31ad6c..674d87435 100644 --- a/packages/coding-agent/test/model-hub.test.ts +++ b/packages/coding-agent/test/model-hub.test.ts @@ -151,7 +151,7 @@ describe("ModelHub", () => { }); describe("role chips and roles view", () => { - test("shows configured role chips with thinking glyphs, including custom roles", () => { + test("tags the selected model's roles in the detail line, including custom roles", () => { const model = getBundledModel("anthropic", "claude-sonnet-4-5"); if (!model) throw new Error("Expected bundled model anthropic/claude-sonnet-4-5"); const settings = Settings.isolated({ @@ -173,7 +173,7 @@ describe("ModelHub", () => { expect(rendered).toContain("●smol"); }); - test("renders hollow chips for auto-selected role fallbacks", () => { + test("list rows carry no role chips; only the selected model's detail line is tagged", () => { const settings = Settings.isolated({}); const haiku = makeModel("test", "claude-haiku-4.5"); const codex = makeModel("test", "gpt-5.1-codex"); @@ -181,10 +181,11 @@ describe("ModelHub", () => { installTestTheme(); const rendered = normalize(hub.render(220)); - // No roles configured: auto-selection still tags the small/reasoning - // candidates (smol → haiku, slow → codex), rendered hollow. - expect(rendered).toContain("○smol"); - expect(rendered).toContain("○slow"); + // Auto-selection tags smol → haiku and slow → codex, but only the + // selected model's chips render (in the detail line). With row + // chips both would appear at once. + const hollow = ["○smol", "○slow"].filter(chip => rendered.includes(chip)); + expect(hollow).toHaveLength(1); expect(rendered).not.toContain("●smol"); }); diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 7ece1d17c..bc6b81c73 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed keyboard navigation paying an extra frame of input latency after idle; the queue-drain grace now applies only to Ctrl+C and Escape double-press gestures. + ## [16.4.6] - 2026-07-12 ### Added diff --git a/packages/tui/src/tui.ts b/packages/tui/src/tui.ts index db08bb041..885f07af2 100644 --- a/packages/tui/src/tui.ts +++ b/packages/tui/src/tui.ts @@ -2239,12 +2239,12 @@ export class TUI extends Container { } #handleInput(data: string): void { - // Raw-mode Ctrl+C/Esc arrive as stdin data, not process signals. If the - // first key in a double-key gesture schedules an immediate slow repaint, - // the queued second key can sit behind that repaint long enough for the - // app-level double-press window to expire. Give the input queue one frame - // before ordinary paints; forced repaints still bypass this path. - this.#inputRenderGraceUntilMs = this.#renderScheduler.now() + TUI.#INPUT_RENDER_GRACE_MS; + // Ctrl+C/Esc use app-level double-press windows. Give those gestures one + // frame to drain queued input before an ordinary repaint; delaying every + // key would make idle navigation pay a full frame of latency. + if (matchesKey(data, "ctrl+c") || matchesKey(data, "escape")) { + this.#inputRenderGraceUntilMs = this.#renderScheduler.now() + TUI.#INPUT_RENDER_GRACE_MS; + } if (this.#inputListeners.size > 0) { let current = data; for (const listener of this.#inputListeners) { diff --git a/packages/tui/test/input-priority.test.ts b/packages/tui/test/input-priority.test.ts index 1ba28047d..9252d5f30 100644 --- a/packages/tui/test/input-priority.test.ts +++ b/packages/tui/test/input-priority.test.ts @@ -40,6 +40,18 @@ class BlockingDoubleInterruptComponent implements Component { } } +class NavigationProbe implements Component { + #selected = 0; + + handleInput(data: string): void { + if (data === "\x1b[B") this.#selected++; + } + + render(_width: number): readonly string[] { + return [`selected:${this.#selected}`]; + } +} + async function drainNextTick(): Promise { const nextTick = Promise.withResolvers(); process.nextTick(nextTick.resolve); @@ -102,4 +114,28 @@ describe("TUI input priority", () => { expect(component.interruptsHandled).toBe(2); expect(component.exitRequests).toBe(1); }); + + it("renders ordinary navigation without an interrupt-grace delay", async () => { + vi.useFakeTimers(); + setSystemTime(new Date(1_000)); + const terminal = new VirtualTerminal(40, 8); + const tui = new TUI(terminal, undefined, { renderScheduler: fakeTimerScheduler() }); + const component = new NavigationProbe(); + tui.addChild(component); + tui.setFocus(component); + + try { + tui.start(); + await drainNextTick(); + vi.advanceTimersByTime(40); + + terminal.sendInput("\x1b[B"); + await drainNextTick(); + await drainNextTick(); + + expect(terminal.getViewport().map(row => row.trimEnd())).toContain("selected:1"); + } finally { + tui.stop(); + } + }); }); From 93db3913d87bb2092a5f575af9387ae2e547b908 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 13:06:01 +0200 Subject: [PATCH 26/80] chore: update tips --- packages/coding-agent/CHANGELOG.md | 7 ++++++- packages/coding-agent/src/modes/components/tips.txt | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 74b448c2a..65a87910a 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -10,9 +10,14 @@ ### Changed - Streamlined list view styling by removing inline model role chips from row entries -- Clarified active selection state by only displaying cursor and highlight bands in focused panes +- Reworked /models hub selection visuals: the background highlight band is reserved for mouse hover, the keyboard position is a cursor glyph drawn only in the pane that owns the arrow keys, and the sidebar's active scope renders as a bold accent label - Removed the redundant "login" label from inactive (locked) provider entries in the Model Hub sidebar +### Fixed + +- Fixed PageUp/PageDown in the model browser wrapping past the list edges instead of clamping +- Fixed the hover highlight sticking to the last hovered model row when the pointer moved into the provider sidebar + ## [16.4.6] - 2026-07-12 ### Added diff --git a/packages/coding-agent/src/modes/components/tips.txt b/packages/coding-agent/src/modes/components/tips.txt index a2c55f06c..7e8c60848 100644 --- a/packages/coding-agent/src/modes/components/tips.txt +++ b/packages/coding-agent/src/modes/components/tips.txt @@ -21,4 +21,5 @@ Pair up live: `/collab` shares your session through an end-to-end encrypted rela Press ← ← to drill into a running or finished agent and inspect its tool calls and transcript Hit a Codex rate limit? `/usage reset` spends a saved reset credit to immediately restore your quota No native tool_calling? Inference provider botches parsing them? `PI_DIALECT=glm|kimi|anthropic…` rolls it locally for them! -Turn on `/advisor` to attach a second model that reviews every turn and quietly injects advice [NEW] \ No newline at end of file +Turn on `/advisor` to attach a second model that reviews every turn and quietly injects advice +Try starting your prompt with a ->, and writing a list (1. Do X, 2. Do Y) \ No newline at end of file From f933f02fcfaff075743835d7b305311dcdf58ec0 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 13:06:27 +0200 Subject: [PATCH 27/80] chore: bump version to 16.4.7 --- Cargo.lock | 10 ++-- Cargo.toml | 2 +- bun.lock | 68 +++++++++++++-------------- crates/pi-natives/src/lib.rs | 2 +- package.json | 24 +++++----- packages/agent/package.json | 2 +- packages/ai/package.json | 2 +- packages/catalog/package.json | 2 +- packages/coding-agent/CHANGELOG.md | 2 + packages/coding-agent/package.json | 2 +- packages/hashline/package.json | 2 +- packages/mnemopi/package.json | 2 +- packages/natives/native/index.d.ts | 2 +- packages/natives/native/index.js | 2 +- packages/natives/package.json | 2 +- packages/snapcompact/package.json | 2 +- packages/stats/CHANGELOG.md | 2 + packages/stats/package.json | 2 +- packages/swarm-extension/package.json | 2 +- packages/tui/CHANGELOG.md | 2 + packages/tui/package.json | 2 +- packages/utils/package.json | 2 +- packages/wire/package.json | 2 +- 23 files changed, 73 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c40f46dcc..a4fe6dba8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3264,7 +3264,7 @@ dependencies = [ [[package]] name = "pi-ast" -version = "16.4.6" +version = "16.4.7" dependencies = [ "anyhow", "ast-grep-core", @@ -3333,7 +3333,7 @@ dependencies = [ [[package]] name = "pi-iso" -version = "16.4.6" +version = "16.4.7" dependencies = [ "async-trait", "libc", @@ -3345,7 +3345,7 @@ dependencies = [ [[package]] name = "pi-natives" -version = "16.4.6" +version = "16.4.7" dependencies = [ "anyhow", "arboard", @@ -3398,7 +3398,7 @@ dependencies = [ [[package]] name = "pi-shell" -version = "16.4.6" +version = "16.4.7" dependencies = [ "anyhow", "brush-builtins", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "pi-walker" -version = "16.4.6" +version = "16.4.7" dependencies = [ "dashmap", "globset", diff --git a/Cargo.toml b/Cargo.toml index 53c08f355..2d82fa773 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["crates/vendor/brush-core", "crates/vendor/brush-builtins"] resolver = "3" [workspace.package] -version = "16.4.6" +version = "16.4.7" edition = "2024" license = "MIT" authors = ["Can Boluk"] diff --git a/bun.lock b/bun.lock index 867b96322..185d6908c 100644 --- a/bun.lock +++ b/bun.lock @@ -21,7 +21,7 @@ }, "packages/agent": { "name": "@oh-my-pi/pi-agent-core", - "version": "16.4.6", + "version": "16.4.7", "dependencies": { "@oh-my-pi/pi-ai": "catalog:", "@oh-my-pi/pi-catalog": "catalog:", @@ -39,7 +39,7 @@ }, "packages/ai": { "name": "@oh-my-pi/pi-ai", - "version": "16.4.6", + "version": "16.4.7", "dependencies": { "@bufbuild/protobuf": "catalog:", "@oh-my-pi/pi-catalog": "catalog:", @@ -55,7 +55,7 @@ }, "packages/catalog": { "name": "@oh-my-pi/pi-catalog", - "version": "16.4.6", + "version": "16.4.7", "dependencies": { "@bufbuild/protobuf": "catalog:", "@oh-my-pi/pi-utils": "catalog:", @@ -69,7 +69,7 @@ }, "packages/coding-agent": { "name": "@oh-my-pi/pi-coding-agent", - "version": "16.4.6", + "version": "16.4.7", "bin": { "omp": "src/cli.ts", }, @@ -138,7 +138,7 @@ }, "packages/hashline": { "name": "@oh-my-pi/hashline", - "version": "16.4.6", + "version": "16.4.7", "dependencies": { "diff": "catalog:", "lru-cache": "catalog:", @@ -149,7 +149,7 @@ }, "packages/mnemopi": { "name": "@oh-my-pi/pi-mnemopi", - "version": "16.4.6", + "version": "16.4.7", "bin": { "mnemopi": "src/cli.ts", }, @@ -175,7 +175,7 @@ }, "packages/natives": { "name": "@oh-my-pi/pi-natives", - "version": "16.4.6", + "version": "16.4.7", "devDependencies": { "@napi-rs/cli": "catalog:", "@types/bun": "catalog:", @@ -183,7 +183,7 @@ }, "packages/snapcompact": { "name": "@oh-my-pi/snapcompact", - "version": "16.4.6", + "version": "16.4.7", "dependencies": { "@oh-my-pi/pi-ai": "catalog:", "@oh-my-pi/pi-natives": "catalog:", @@ -196,7 +196,7 @@ }, "packages/stats": { "name": "@oh-my-pi/omp-stats", - "version": "16.4.6", + "version": "16.4.7", "bin": { "omp-stats": "./src/index.ts", }, @@ -222,7 +222,7 @@ }, "packages/swarm-extension": { "name": "@oh-my-pi/swarm-extension", - "version": "16.4.6", + "version": "16.4.7", "bin": { "omp-swarm": "src/cli.ts", }, @@ -248,7 +248,7 @@ }, "packages/tui": { "name": "@oh-my-pi/pi-tui", - "version": "16.4.6", + "version": "16.4.7", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "@oh-my-pi/pi-utils": "catalog:", @@ -289,7 +289,7 @@ }, "packages/utils": { "name": "@oh-my-pi/pi-utils", - "version": "16.4.6", + "version": "16.4.7", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "handlebars": "catalog:", @@ -302,7 +302,7 @@ }, "packages/wire": { "name": "@oh-my-pi/pi-wire", - "version": "16.4.6", + "version": "16.4.7", "devDependencies": { "@types/bun": "catalog:", }, @@ -342,18 +342,18 @@ "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.7.0", - "@oh-my-pi/hashline": "16.4.6", - "@oh-my-pi/omp-stats": "16.4.6", - "@oh-my-pi/pi-agent-core": "16.4.6", - "@oh-my-pi/pi-ai": "16.4.6", - "@oh-my-pi/pi-catalog": "16.4.6", - "@oh-my-pi/pi-coding-agent": "16.4.6", - "@oh-my-pi/pi-mnemopi": "16.4.6", - "@oh-my-pi/pi-natives": "16.4.6", - "@oh-my-pi/pi-tui": "16.4.6", - "@oh-my-pi/pi-utils": "16.4.6", - "@oh-my-pi/pi-wire": "16.4.6", - "@oh-my-pi/snapcompact": "16.4.6", + "@oh-my-pi/hashline": "16.4.7", + "@oh-my-pi/omp-stats": "16.4.7", + "@oh-my-pi/pi-agent-core": "16.4.7", + "@oh-my-pi/pi-ai": "16.4.7", + "@oh-my-pi/pi-catalog": "16.4.7", + "@oh-my-pi/pi-coding-agent": "16.4.7", + "@oh-my-pi/pi-mnemopi": "16.4.7", + "@oh-my-pi/pi-natives": "16.4.7", + "@oh-my-pi/pi-tui": "16.4.7", + "@oh-my-pi/pi-utils": "16.4.7", + "@oh-my-pi/pi-wire": "16.4.7", + "@oh-my-pi/snapcompact": "16.4.7", "@opentelemetry/api": "^1.9.1", "@opentelemetry/context-async-hooks": "^2.7.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.218.0", @@ -490,7 +490,7 @@ "@emnapi/core": ["@emnapi/core@1.11.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ=="], - "@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], + "@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA=="], @@ -1264,7 +1264,7 @@ "neo-async": ["neo-async@2.6.2", "", {}, "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="], - "node-releases": ["node-releases@2.0.50", "", {}, "sha512-J6l92tKHX6w8Jy5nO1Vuc01NoIiRGi/d6qBKVxh+IQ8Cr3b6HbVNfKiF8ZpFKufTwpwxMmce2W3iQZ861ZRyTg=="], + "node-releases": ["node-releases@2.0.51", "", {}, "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ=="], "nth-check": ["nth-check@3.0.1", "", { "dependencies": { "boolbase": "^2.0.0" } }, "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ=="], @@ -1394,7 +1394,7 @@ "string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], - "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], "strip-ansi": ["strip-ansi@7.2.0", "", { "dependencies": { "ansi-regex": "^6.2.2" } }, "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w=="], @@ -1446,7 +1446,7 @@ "vali-date": ["vali-date@1.0.0", "", {}, "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg=="], - "vite": ["vite@8.1.3", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", "postcss": "^8.5.16", "rolldown": "~1.1.3", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-Ds+gBRbj0lwRO2Y5hwnUBdxSwlAve9LeRyU4sNnAr0ewW0gWF0n5bgXgUzbgZ49MV9BVUAQUFYVcDUcilUExMA=="], + "vite": ["vite@8.1.4", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", "postcss": "^8.5.16", "rolldown": "~1.1.4", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-bTT9PsdWO+MQMNG9ZXIP/qM9wGh37DFxTV/sPq9cFpHr3w4jkgef032PkAL9jAqhk3Nz8NQw3O8n6/xFkqO4QQ=="], "vite-plugin-solid": ["vite-plugin-solid@2.11.12", "", { "dependencies": { "@babel/core": "^7.23.3", "@types/babel__core": "^7.20.4", "babel-preset-solid": "^1.8.4", "merge-anything": "^5.1.7", "solid-refresh": "^0.6.3", "vitefu": "^1.0.4" }, "peerDependencies": { "@testing-library/jest-dom": "^5.16.6 || ^5.17.0 || ^6.*", "solid-js": "^1.7.2", "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["@testing-library/jest-dom"] }, "sha512-FgjPcx2OwX9h6f28jli7A4bG7PP3te8uyakE5iqsmpq3Jqi1TWLgSroC9N6cMfGRU2zXsl4Q6ISvTr2VL0QHpA=="], @@ -1516,9 +1516,11 @@ "@opentelemetry/sdk-metrics/@opentelemetry/resources": ["@opentelemetry/resources@2.7.1", "", { "dependencies": { "@opentelemetry/core": "2.7.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ=="], - "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ=="], + "@rolldown/binding-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], - "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], + "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA=="], + + "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.2", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA=="], @@ -1570,8 +1572,6 @@ "roarr/sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], - "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], - "wrap-ansi/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], @@ -1594,8 +1594,6 @@ "htmlparser2/domutils/dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], - "jszip/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], - "@huggingface/transformers/onnxruntime-node/global-agent/matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="], "@huggingface/transformers/onnxruntime-node/global-agent/serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], diff --git a/crates/pi-natives/src/lib.rs b/crates/pi-natives/src/lib.rs index b3d6a640a..e57f51c9c 100644 --- a/crates/pi-natives/src/lib.rs +++ b/crates/pi-natives/src/lib.rs @@ -248,7 +248,7 @@ fn create_windows_napi_tokio_runtime() -> Option { /// MUST stay in sync with `VERSION_SENTINEL_EXPORT` in /// `packages/natives/native/index.js` (which derives the name from /// `package.json#version`). -#[napi(js_name = "__piNativesV16_4_6")] +#[napi(js_name = "__piNativesV16_4_7")] pub const fn pi_natives_version_sentinel() {} /// Native module entry point: install crash diagnostics before any tool can diff --git a/package.json b/package.json index e9c758e51..65f3d5793 100644 --- a/package.json +++ b/package.json @@ -25,18 +25,18 @@ "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.7.0", - "@oh-my-pi/hashline": "16.4.6", - "@oh-my-pi/omp-stats": "16.4.6", - "@oh-my-pi/pi-agent-core": "16.4.6", - "@oh-my-pi/pi-ai": "16.4.6", - "@oh-my-pi/pi-catalog": "16.4.6", - "@oh-my-pi/pi-coding-agent": "16.4.6", - "@oh-my-pi/pi-mnemopi": "16.4.6", - "@oh-my-pi/pi-natives": "16.4.6", - "@oh-my-pi/pi-tui": "16.4.6", - "@oh-my-pi/pi-utils": "16.4.6", - "@oh-my-pi/pi-wire": "16.4.6", - "@oh-my-pi/snapcompact": "16.4.6", + "@oh-my-pi/hashline": "16.4.7", + "@oh-my-pi/omp-stats": "16.4.7", + "@oh-my-pi/pi-agent-core": "16.4.7", + "@oh-my-pi/pi-ai": "16.4.7", + "@oh-my-pi/pi-catalog": "16.4.7", + "@oh-my-pi/pi-coding-agent": "16.4.7", + "@oh-my-pi/pi-mnemopi": "16.4.7", + "@oh-my-pi/pi-natives": "16.4.7", + "@oh-my-pi/pi-tui": "16.4.7", + "@oh-my-pi/pi-utils": "16.4.7", + "@oh-my-pi/pi-wire": "16.4.7", + "@oh-my-pi/snapcompact": "16.4.7", "@opentelemetry/api": "^1.9.1", "@opentelemetry/context-async-hooks": "^2.7.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.218.0", diff --git a/packages/agent/package.json b/packages/agent/package.json index d2233f3be..0ac34f9a2 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-agent-core", - "version": "16.4.6", + "version": "16.4.7", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/ai/package.json b/packages/ai/package.json index 3d25f92ff..9e28b73ba 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-ai", - "version": "16.4.6", + "version": "16.4.7", "description": "Unified LLM API with automatic model discovery and provider configuration", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/catalog/package.json b/packages/catalog/package.json index e5c62572d..3f23ba236 100644 --- a/packages/catalog/package.json +++ b/packages/catalog/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-catalog", - "version": "16.4.6", + "version": "16.4.7", "description": "Model catalog for omp: bundled model database, provider discovery descriptors, model identity, classification, and equivalence", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 65a87910a..737235f6f 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.7] - 2026-07-12 + ### Added - Enabled Home and End keyboard navigation in the model browser diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 5a808ccde..8b72da063 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-coding-agent", - "version": "16.4.6", + "version": "16.4.7", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/hashline/package.json b/packages/hashline/package.json index f218a2ad9..8a890adff 100644 --- a/packages/hashline/package.json +++ b/packages/hashline/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/hashline", - "version": "16.4.6", + "version": "16.4.7", "description": "Hashline: a compact, line-anchored patch language and applier. Pluggable FS/IO so it works over disk, in-memory, or any custom backend.", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/mnemopi/package.json b/packages/mnemopi/package.json index d0a7ef18b..40a543674 100644 --- a/packages/mnemopi/package.json +++ b/packages/mnemopi/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-mnemopi", - "version": "16.4.6", + "version": "16.4.7", "description": "Local SQLite memory engine for Oh My Pi agents", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/natives/native/index.d.ts b/packages/natives/native/index.d.ts index 4e0394861..d693eca63 100644 --- a/packages/natives/native/index.d.ts +++ b/packages/natives/native/index.d.ts @@ -170,7 +170,7 @@ export declare function __ompInstallTokioRuntime(): void * `packages/natives/native/index.js` (which derives the name from * `package.json#version`). */ -export declare function __piNativesV16_4_6(): void +export declare function __piNativesV16_4_7(): void /** * Apply ast-grep rewrite rules to matching files; honors `dryRun` and returns diff --git a/packages/natives/native/index.js b/packages/natives/native/index.js index 58f09faab..5b03bfece 100644 --- a/packages/natives/native/index.js +++ b/packages/natives/native/index.js @@ -24,7 +24,7 @@ export const Shell = nativeBindings.Shell; // functions export const __ompInstallTokioRuntime = nativeBindings.__ompInstallTokioRuntime; -export const __piNativesV16_4_6 = nativeBindings.__piNativesV16_4_6; +export const __piNativesV16_4_7 = nativeBindings.__piNativesV16_4_7; export const astEdit = nativeBindings.astEdit; export const astGrep = nativeBindings.astGrep; export const astMatch = nativeBindings.astMatch; diff --git a/packages/natives/package.json b/packages/natives/package.json index 8555cc095..3ec513cda 100644 --- a/packages/natives/package.json +++ b/packages/natives/package.json @@ -1,6 +1,6 @@ { "name": "@oh-my-pi/pi-natives", - "version": "16.4.6", + "version": "16.4.7", "description": "Native Rust bindings for grep, clipboard, image processing, syntax highlighting, PTY, and shell operations via N-API", "type": "module", "homepage": "https://omp.sh", diff --git a/packages/snapcompact/package.json b/packages/snapcompact/package.json index e22d389da..811e2944a 100644 --- a/packages/snapcompact/package.json +++ b/packages/snapcompact/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/snapcompact", - "version": "16.4.6", + "version": "16.4.7", "description": "Bitmap-frame context compression for vision-capable LLMs", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/stats/CHANGELOG.md b/packages/stats/CHANGELOG.md index 76f3de847..7cf491436 100644 --- a/packages/stats/CHANGELOG.md +++ b/packages/stats/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.7] - 2026-07-12 + ### Fixed - Fixed a `SQLITE_CONSTRAINT_NOTNULL` crash (`messages.stop_reason`) aborting the entire session sync when a persisted assistant message lacks a `stopReason`. Malformed entries — missing stop reason, token counts, or message timestamp — are now coerced at the parser boundary, and entries with no usage or model attribution are skipped instead of failing the batch insert. diff --git a/packages/stats/package.json b/packages/stats/package.json index e104e9f00..73f9a7cc3 100644 --- a/packages/stats/package.json +++ b/packages/stats/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/omp-stats", - "version": "16.4.6", + "version": "16.4.7", "description": "Local observability dashboard for pi AI usage statistics", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/swarm-extension/package.json b/packages/swarm-extension/package.json index 3e74a0598..ef4d3857a 100644 --- a/packages/swarm-extension/package.json +++ b/packages/swarm-extension/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/swarm-extension", - "version": "16.4.6", + "version": "16.4.7", "description": "Swarm orchestration extension for omp", "homepage": "https://omp.sh", "author": "Derek Rynd", diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index bc6b81c73..3143afbfc 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.7] - 2026-07-12 + ### Fixed - Fixed keyboard navigation paying an extra frame of input latency after idle; the queue-drain grace now applies only to Ctrl+C and Escape double-press gestures. diff --git a/packages/tui/package.json b/packages/tui/package.json index d1ed89a22..3bc7341bb 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-tui", - "version": "16.4.6", + "version": "16.4.7", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/utils/package.json b/packages/utils/package.json index 0093a8dde..109ec1ead 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-utils", - "version": "16.4.6", + "version": "16.4.7", "description": "Shared utilities for pi packages", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/wire/package.json b/packages/wire/package.json index eb392d0c3..a91d7bb34 100644 --- a/packages/wire/package.json +++ b/packages/wire/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-wire", - "version": "16.4.6", + "version": "16.4.7", "description": "Shared wire protocol types for Oh My Pi packages", "homepage": "https://omp.sh", "author": "Can Boluk", From b6f83021c937b1dee56243dcc2f50349aa292de4 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 13:21:45 +0200 Subject: [PATCH 28/80] fix(coding-agent): ensured top-level declarations persist in async cells - Updated import rewriting to identify and publish `var` and `function` declarations to the global scope when a cell contains top-level `await`. - Prevented these declarations from being trapped within the async wrapper's function scope, allowing them to remain accessible to subsequent evaluation cells. --- packages/coding-agent/CHANGELOG.md | 5 +++ .../src/eval/js/shared/rewrite-imports.ts | 44 +++++++++++++------ .../core/js-static-import-rewrite.test.ts | 36 +++++++++++++++ 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 737235f6f..7de11cc4d 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +### Fixed + +- Improved search reliability for Perplexity provider by forcing retrieval for all queries +- Fixed JS eval cells losing top-level `function` and `var` declarations across cells when the defining cell contained top-level `await` — the async wrapper scoped them to the cell's IIFE instead of publishing them to the worker global + ## [16.4.7] - 2026-07-12 ### Added diff --git a/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts b/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts index a5abd1897..265b74cb4 100644 --- a/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts +++ b/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts @@ -48,6 +48,15 @@ type BabelClassDeclaration = { }; type BabelLexicalDecl = BabelVariableDeclaration | BabelClassDeclaration; +type BabelFunctionDeclaration = { + type: "FunctionDeclaration"; + start: number; + end: number; + id: { start: number; end: number; name: string } | null; +}; + +/** Top-level declarations whose bindings must survive the cell (demoted and/or published). */ +type BabelPublishableDecl = BabelLexicalDecl | BabelFunctionDeclaration; type BabelExpressionStatement = { type: "ExpressionStatement"; @@ -322,7 +331,7 @@ function collectBindingNames(pattern: unknown, names: string[]): void { } } -function getLexicalBindingNames(node: BabelLexicalDecl): string[] { +function getLexicalBindingNames(node: BabelPublishableDecl): string[] { const names: string[] = []; if (node.type === "VariableDeclaration") { for (const declaration of node.declarations ?? []) collectBindingNames(declaration.id, names); @@ -348,40 +357,49 @@ function appendGlobalBindingPublish(source: string, names: readonly string[]): s * let { a, b } = obj; -> var { a, b } = obj; * class Foo extends Bar {} -> var Foo = class extends Bar {}; * - * When the source must run inside the async wrapper, demoted `var`s would normally become - * function-scoped. In that mode we publish each top-level binding back to the wrapper's - * lexical `this`, which is the worker global object. + * When the source must run inside the async wrapper (top-level `await`), demoted `var`s — + * and the user's own top-level `var` and `function` declarations — would be scoped to the + * wrapper function and die with the cell. In that mode we publish every top-level binding + * back to the wrapper's lexical `this`, which is the worker global object. * - * Nested declarations (inside functions, blocks, classes) are left alone \u2014 they're + * Nested declarations (inside functions, blocks, classes) are left alone — they're * scoped to their enclosing function/block regardless of `var` vs `let`/`const`. */ async function demoteTopLevelLexicals(code: string, options: { publishGlobals?: boolean } = {}): Promise { - if (!/\b(?:const|let|class)\b/.test(code)) return code; + const publishGlobals = options.publishGlobals === true; + const fastPath = publishGlobals ? /\b(?:const|let|class|var|function)\b/ : /\b(?:const|let|class)\b/; + if (!fastPath.test(code)) return code; const ast = await parseProgram(code); if (!ast) { return code; } - const targets: BabelLexicalDecl[] = []; + const targets: Array<{ node: BabelPublishableDecl; demote: boolean }> = []; for (const node of ast.program.body) { if (node.type === "VariableDeclaration") { const decl = node as unknown as BabelVariableDeclaration; - if (decl.kind === "const" || decl.kind === "let") targets.push(decl); + if (decl.kind === "const" || decl.kind === "let") targets.push({ node: decl, demote: true }); + else if (publishGlobals) targets.push({ node: decl, demote: false }); } else if (node.type === "ClassDeclaration") { const decl = node as unknown as BabelClassDeclaration; - if (decl.id) targets.push(decl); + if (decl.id) targets.push({ node: decl, demote: true }); + } else if (publishGlobals && node.type === "FunctionDeclaration") { + const decl = node as unknown as BabelFunctionDeclaration; + if (decl.id) targets.push({ node: decl, demote: false }); } } if (targets.length === 0) return code; - targets.sort((a, b) => b.start - a.start); + targets.sort((a, b) => b.node.start - a.node.start); let result = code; - for (const node of targets) { + for (const { node, demote } of targets) { const segment = result.slice(node.start, node.end); - const bindingNames = options.publishGlobals ? getLexicalBindingNames(node) : []; + const bindingNames = publishGlobals ? getLexicalBindingNames(node) : []; let replacement: string; - if (node.type === "VariableDeclaration") { + if (!demote) { + replacement = segment; + } else if (node.type === "VariableDeclaration") { replacement = `var${segment.slice(node.kind.length)}`; } else { const id = node.id; diff --git a/packages/coding-agent/test/core/js-static-import-rewrite.test.ts b/packages/coding-agent/test/core/js-static-import-rewrite.test.ts index 90fff846e..3321a85f3 100644 --- a/packages/coding-agent/test/core/js-static-import-rewrite.test.ts +++ b/packages/coding-agent/test/core/js-static-import-rewrite.test.ts @@ -182,3 +182,39 @@ describe("rewriteImports", () => { expect(wrapped.source).not.toContain(`${IMPORT} type`); }); }); + +// Cross-cell persistence: cells run through indirect eval, so bindings survive only if +// they land on globalThis. Cells with top-level `await` run inside an async IIFE, where +// declarations are function-scoped — wrapCode must publish them back to the global object +// or the next cell sees ReferenceError. +describe("wrapCode cross-cell persistence", () => { + it("publishes top-level function declarations and their sibling consts from async-wrapped cells", async () => { + const globals = globalThis as Record; + const wrapped = await wrapCode( + "async function ompPersistedFn(n) { return (await Promise.resolve(n)) + 1; }\nconst ompPersistedTotal = await ompPersistedFn(41);", + ); + expect(wrapped.asyncWrapped).toBe(true); + try { + await indirectEval(wrapped.source); + const fn = globals.ompPersistedFn as (n: number) => Promise; + expect(typeof fn).toBe("function"); + expect(await fn(1)).toBe(2); + expect(globals.ompPersistedTotal).toBe(42); + } finally { + delete globals.ompPersistedFn; + delete globals.ompPersistedTotal; + } + }); + + it("publishes explicit top-level var declarations from async-wrapped cells", async () => { + const globals = globalThis as Record; + const wrapped = await wrapCode("await Promise.resolve();\nvar ompPersistedVar = 5;"); + expect(wrapped.asyncWrapped).toBe(true); + try { + await indirectEval(wrapped.source); + expect(globals.ompPersistedVar).toBe(5); + } finally { + delete globals.ompPersistedVar; + } + }); +}); From dabe233c627ea68ce0987b041c96bcf6dfd57f33 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 13:14:06 +0200 Subject: [PATCH 29/80] feat(coding-agent/web): improved perplexity results --- .../src/web/search/providers/perplexity.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/src/web/search/providers/perplexity.ts b/packages/coding-agent/src/web/search/providers/perplexity.ts index 1aa789978..757d49a8b 100644 --- a/packages/coding-agent/src/web/search/providers/perplexity.ts +++ b/packages/coding-agent/src/web/search/providers/perplexity.ts @@ -581,11 +581,27 @@ async function callPerplexityAsk( search_recency_filter: params.search_recency_filter ?? null, is_incognito: true, use_schematized_api: true, - skip_search_enabled: true, + // `true` (the native app's default) lets the backend classifier skip + // retrieval for queries it deems answerable from memory — the model then + // runs ungrounded and refuses with "I don't currently have live access". + // We are a search tool; always retrieve. + skip_search_enabled: false, + // Belt and braces with `skip_search_enabled: false`: the web client sets + // this to force retrieval even when the skip classifier fires. + always_search_override: true, + prompt_source: "user", + source: "default", + local_search_enabled: false, + // Declare no tool-approval UI and no local (Comet) browser agent, so the + // stream never stalls waiting for a confirmation we cannot render. + should_ask_for_mcp_tool_confirmation: false, + supports_tool_approval_modal: false, + force_enable_browser_agent: false, + is_local_browser_available: false, + is_local_browser_allowed: false, }; if (auth.type === "anonymous") { requestParams.send_back_text_in_streaming_api = true; - requestParams.source = "default"; } const response = await (params.fetch ?? fetch)(PERPLEXITY_OAUTH_ASK_URL, { From bb35e791890d33327ff184b1e94621d074b5bad4 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 13:40:43 +0200 Subject: [PATCH 30/80] fix(ast): auto-wrap multi-node patterns instead of erroring A pattern like `"@types/bun": $V` parses to multiple root AST nodes and was rejected with "Multiple AST nodes are detected". When no explicit selector is given, retry the fragment wrapped in a minimal single-node context (JSON: `{ }` selecting `pair`), quoting bare metavars so JSON parses. Falls back to the original error if the wrapped form still fails, so bad patterns are unchanged. Applies to both ast_edit (compile_pattern) and ast_grep (compile_search_patterns). --- crates/pi-ast/src/ops.rs | 120 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 6 deletions(-) diff --git a/crates/pi-ast/src/ops.rs b/crates/pi-ast/src/ops.rs index 8592bb36f..250d10406 100644 --- a/crates/pi-ast/src/ops.rs +++ b/crates/pi-ast/src/ops.rs @@ -100,21 +100,129 @@ pub fn compile_pattern( strictness: &MatchStrictness, lang: SupportLang, ) -> Result { - let mut compiled = if let Some(selector) = selector.map(str::trim).filter(|s| !s.is_empty()) { - Pattern::contextual(pattern, selector, lang) + let selector = selector.map(str::trim).filter(|s| !s.is_empty()); + let mut compiled = if let Some(selector) = selector { + Pattern::contextual(pattern, selector, lang).map_err(|err| anyhow!("Invalid pattern: {err}"))? } else { - Pattern::try_new(pattern, lang) - } - .map_err(|err| anyhow!("Invalid pattern: {err}"))?; + match Pattern::try_new(pattern, lang) { + Ok(compiled) => compiled, + // A fragment like `"key": $V` parses to multiple root nodes and is + // rejected as `MultipleNode`; auto-wrap it in a single-node context + // before giving up. Any other error, or a failed fallback, keeps the + // original message so genuinely-bad patterns behave as before. + Err(err @ PatternError::MultipleNode(_)) => { + match compile_wrapped_fallback(pattern, strictness, lang) { + Some(compiled) => return Ok(compiled), + None => return Err(anyhow!("Invalid pattern: {err}")), + } + }, + Err(err) => return Err(anyhow!("Invalid pattern: {err}")), + } + }; compiled.strictness = strictness.clone(); Ok(compiled) } +/// Language-specific wrapper template used to turn a multi-node fragment into a +/// single selectable node. `None` for languages without a template — those keep +/// the original `MultipleNode` error. +fn wrapper_template(lang: SupportLang) -> Option<(&'static str, &'static str, &'static str)> { + // (prefix, suffix, selector-kind); the fragment is spliced between prefix/suffix. + match lang { + SupportLang::Json => Some(("{", "}", "pair")), + _ => None, + } +} + +/// Retry a fragment that failed as `MultipleNode` by wrapping it in a minimal +/// valid context and selecting the node kind that spans it. Returns the compiled +/// pattern (with `strictness` applied) or `None` if this language has no template +/// or the wrapped form still fails to compile. +fn compile_wrapped_fallback( + pattern: &str, + strictness: &MatchStrictness, + lang: SupportLang, +) -> Option { + let (prefix, suffix, selector) = wrapper_template(lang)?; + // JSON only accepts a bare `$V` inside a string, so quote value-position + // metavars; ast-grep still reads the quoted `"$V"` as capture `V`. + let prepared = if lang == SupportLang::Json { + quote_bare_metavars(pattern) + } else { + pattern.to_string() + }; + let context = format!("{prefix} {prepared} {suffix}"); + let mut compiled = Pattern::contextual(&context, selector, lang).ok()?; + compiled.strictness = strictness.clone(); + Some(compiled) +} + +/// Wrap bare `$NAME` / `$$$NAME` metavars in double quotes so a JSON wrapper +/// parses. Metavars already inside a string literal (including `"$V"`) are left +/// untouched; a quote toggles in/out of string context. +fn quote_bare_metavars(pattern: &str) -> String { + let bytes = pattern.as_bytes(); + let mut out = String::with_capacity(pattern.len() + 4); + let mut in_string = false; + let mut index = 0; + while index < bytes.len() { + let byte = bytes[index]; + if byte == b'"' && (index == 0 || bytes[index - 1] != b'\\') { + in_string = !in_string; + out.push('"'); + index += 1; + continue; + } + if byte == b'$' && !in_string { + // Consume `$`, an optional `$$` ellipsis, then the identifier. + let start = index; + index += 1; + if bytes[index..].starts_with(b"$$") { + index += 2; + } + while index < bytes.len() + && (bytes[index].is_ascii_alphanumeric() || bytes[index] == b'_') + { + index += 1; + } + out.push('"'); + out.push_str(&pattern[start..index]); + out.push('"'); + continue; + } + // Copy this byte's full UTF-8 char so multi-byte content is preserved. + let char_end = next_char_boundary(bytes, index); + out.push_str(&pattern[index..char_end]); + index = char_end; + } + out +} + +/// Byte index of the end of the UTF-8 character starting at `index`. +fn next_char_boundary(bytes: &[u8], index: usize) -> usize { + let mut end = index + 1; + while end < bytes.len() && (bytes[end] & 0b1100_0000) == 0b1000_0000 { + end += 1; + } + end +} + pub fn compile_search_patterns( pattern: &str, language: SupportLang, ) -> Result, PatternError> { - let mut compiled = vec![Pattern::try_new(pattern, language)?]; + let mut compiled = match Pattern::try_new(pattern, language) { + Ok(compiled) => vec![compiled], + // Multi-node fragments (e.g. `"key": $V`) get the same auto-wrap fallback + // as the edit path; other errors propagate unchanged. + Err(err @ PatternError::MultipleNode(_)) => { + match compile_wrapped_fallback(pattern, &MatchStrictness::Smart, language) { + Some(compiled) => vec![compiled], + None => return Err(err), + } + }, + Err(err) => return Err(err), + }; if language == SupportLang::Rust { let trimmed = pattern.trim_end(); if let Some(contextual) = compile_rust_contextual_pattern(trimmed) { From fc35e17cb9a697cd7c7a66ea8e9ff9bf9fc88293 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 20:46:29 +0200 Subject: [PATCH 31/80] fix(pi-ast): satisfied rustfmt and clippy const-fn lint in ops.rs - Reflowed compile_pattern/wrapper_template comments and while-condition to rustfmt's preferred layout. - Made wrapper_template a const fn to satisfy clippy::missing_const_for_fn under -D warnings. --- crates/pi-ast/src/ops.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/pi-ast/src/ops.rs b/crates/pi-ast/src/ops.rs index 250d10406..2149f52cd 100644 --- a/crates/pi-ast/src/ops.rs +++ b/crates/pi-ast/src/ops.rs @@ -102,7 +102,8 @@ pub fn compile_pattern( ) -> Result { let selector = selector.map(str::trim).filter(|s| !s.is_empty()); let mut compiled = if let Some(selector) = selector { - Pattern::contextual(pattern, selector, lang).map_err(|err| anyhow!("Invalid pattern: {err}"))? + Pattern::contextual(pattern, selector, lang) + .map_err(|err| anyhow!("Invalid pattern: {err}"))? } else { match Pattern::try_new(pattern, lang) { Ok(compiled) => compiled, @@ -126,8 +127,9 @@ pub fn compile_pattern( /// Language-specific wrapper template used to turn a multi-node fragment into a /// single selectable node. `None` for languages without a template — those keep /// the original `MultipleNode` error. -fn wrapper_template(lang: SupportLang) -> Option<(&'static str, &'static str, &'static str)> { - // (prefix, suffix, selector-kind); the fragment is spliced between prefix/suffix. +const fn wrapper_template(lang: SupportLang) -> Option<(&'static str, &'static str, &'static str)> { + // (prefix, suffix, selector-kind); the fragment is spliced between + // prefix/suffix. match lang { SupportLang::Json => Some(("{", "}", "pair")), _ => None, @@ -135,9 +137,9 @@ fn wrapper_template(lang: SupportLang) -> Option<(&'static str, &'static str, &' } /// Retry a fragment that failed as `MultipleNode` by wrapping it in a minimal -/// valid context and selecting the node kind that spans it. Returns the compiled -/// pattern (with `strictness` applied) or `None` if this language has no template -/// or the wrapped form still fails to compile. +/// valid context and selecting the node kind that spans it. Returns the +/// compiled pattern (with `strictness` applied) or `None` if this language has +/// no template or the wrapped form still fails to compile. fn compile_wrapped_fallback( pattern: &str, strictness: &MatchStrictness, @@ -180,8 +182,7 @@ fn quote_bare_metavars(pattern: &str) -> String { if bytes[index..].starts_with(b"$$") { index += 2; } - while index < bytes.len() - && (bytes[index].is_ascii_alphanumeric() || bytes[index] == b'_') + while index < bytes.len() && (bytes[index].is_ascii_alphanumeric() || bytes[index] == b'_') { index += 1; } From 6bd51d4ad32aefe59081eb335af5ef09953dc2a8 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 20:46:29 +0200 Subject: [PATCH 32/80] refactor(coding-agent): decoupled tip weight test from tips.txt data - pickWeightedTip now takes the tip list and a uniform sample, exported for tests. - Weighted-selection test sweeps a synthetic tip list, so shipping zero [NEW] tips no longer fails the suite. --- .../src/modes/components/welcome.ts | 27 +++++++++---------- .../test/modes/components/welcome.test.ts | 15 +++++------ 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/packages/coding-agent/src/modes/components/welcome.ts b/packages/coding-agent/src/modes/components/welcome.ts index 673056be4..fdb4dfc3e 100644 --- a/packages/coding-agent/src/modes/components/welcome.ts +++ b/packages/coding-agent/src/modes/components/welcome.ts @@ -44,20 +44,19 @@ const NEW_GLOW_PERIOD_MS = 1500; * affordance surfaces this many times as often. */ const NEW_TIP_WEIGHT = 4; -/** Per-tip selection weights, parallel to {@link TIPS}. */ -const TIP_WEIGHTS: readonly number[] = TIPS.map(tip => (NEW_TIP_MARKER.test(tip) ? NEW_TIP_WEIGHT : 1)); -const TIP_WEIGHT_TOTAL = TIP_WEIGHTS.reduce((sum, weight) => sum + weight, 0); - -/** Pick a tip at random, biased toward "[NEW]" tips by {@link NEW_TIP_WEIGHT}. - * Returns "" when no tips are embedded. */ -function pickWeightedTip(): string { - if (TIPS.length === 0) return ""; - let r = Math.random() * TIP_WEIGHT_TOTAL; - for (let i = 0; i < TIPS.length; i++) { - r -= TIP_WEIGHTS[i] ?? 1; - if (r < 0) return TIPS[i] ?? ""; +/** Pick a tip from `tips`, biased toward "[NEW]" tips by {@link NEW_TIP_WEIGHT}; + * `r` is a uniform sample in [0, 1). Returns "" when `tips` is empty. + * Exported for tests. */ +export function pickWeightedTip(tips: readonly string[], r: number): string { + if (tips.length === 0) return ""; + const weights = tips.map(tip => (NEW_TIP_MARKER.test(tip) ? NEW_TIP_WEIGHT : 1)); + const total = weights.reduce((sum, weight) => sum + weight, 0); + let acc = r * total; + for (let i = 0; i < tips.length; i++) { + acc -= weights[i] ?? 1; + if (acc < 0) return tips[i] ?? ""; } - return TIPS[TIPS.length - 1] ?? ""; + return tips[tips.length - 1] ?? ""; } type ColorEncoding = "ansi-16m" | "ansi-256"; @@ -161,7 +160,7 @@ export class WelcomeComponent implements Component { if (theme.getSymbolPreset() === "unicode" && Math.random() < 0.1) { this.#selectedTip = "Please use nerdfont 😭."; } else { - this.#selectedTip = pickWeightedTip(); + this.#selectedTip = pickWeightedTip(TIPS, Math.random()); } } return this.#selectedTip || undefined; diff --git a/packages/coding-agent/test/modes/components/welcome.test.ts b/packages/coding-agent/test/modes/components/welcome.test.ts index 901c184e2..f1d5997d1 100644 --- a/packages/coding-agent/test/modes/components/welcome.test.ts +++ b/packages/coding-agent/test/modes/components/welcome.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "bun:test"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; -import { WelcomeComponent } from "@oh-my-pi/pi-coding-agent/modes/components/welcome"; +import { pickWeightedTip, WelcomeComponent } from "@oh-my-pi/pi-coding-agent/modes/components/welcome"; import { initTheme, theme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; describe("WelcomeComponent tips", () => { @@ -37,17 +37,15 @@ describe("WelcomeComponent tips", () => { }); it("weights [NEW] tips above ordinary tips in selection", () => { - // Skip the nerdfont gate so the only Math.random() call is the weighted pick. - vi.spyOn(theme, "getSymbolPreset").mockReturnValue("nerd"); - let r = 0; - vi.spyOn(Math, "random").mockImplementation(() => r); + // Data-independent: tips.txt may legitimately carry zero "[NEW]" tips, so + // exercise the weighting contract on a synthetic list. + const tips = ["plain one", "shiny thing [NEW]", "plain two"] as const; const counts = new Map(); const samples = 10_000; for (let i = 0; i < samples; i++) { - r = (i + 0.5) / samples; // sweep the selection domain uniformly - const tip = new WelcomeComponent("1.0.0", "model", "provider").tip; - if (tip) counts.set(tip, (counts.get(tip) ?? 0) + 1); + const tip = pickWeightedTip(tips, (i + 0.5) / samples); // sweep the selection domain uniformly + counts.set(tip, (counts.get(tip) ?? 0) + 1); } let newMax = 0; @@ -61,5 +59,6 @@ describe("WelcomeComponent tips", () => { // uniform selection domain than any single ordinary tip. expect(newMax).toBeGreaterThan(0); expect(newMax).toBeGreaterThan(ordinaryMax); + expect(pickWeightedTip([], 0.5)).toBe(""); }); }); From 01d3fc9b6be922d2209c3211b2063e60565d7398 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 20:48:09 +0200 Subject: [PATCH 33/80] chore: bump version to 16.4.8 --- Cargo.lock | 26 +++++------ Cargo.toml | 2 +- bun.lock | 66 ++++++++++++++------------- crates/pi-natives/src/lib.rs | 2 +- package.json | 24 +++++----- packages/agent/package.json | 2 +- packages/ai/package.json | 2 +- packages/catalog/package.json | 2 +- packages/coding-agent/CHANGELOG.md | 2 + packages/coding-agent/package.json | 2 +- packages/hashline/package.json | 2 +- packages/mnemopi/package.json | 2 +- packages/natives/native/index.d.ts | 2 +- packages/natives/native/index.js | 2 +- packages/natives/package.json | 2 +- packages/snapcompact/package.json | 2 +- packages/stats/package.json | 2 +- packages/swarm-extension/package.json | 2 +- packages/tui/package.json | 2 +- packages/utils/package.json | 2 +- packages/wire/package.json | 2 +- 21 files changed, 79 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a4fe6dba8..590488592 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2616,9 +2616,9 @@ dependencies = [ [[package]] name = "napi" -version = "3.10.3" +version = "3.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c71997d6f7ad4a756966e452426848ac27d3b37a295302d63afbbcce0270f93" +checksum = "59b7fbd5f12adbf51ddec954d4ef9cecb3542a9d53bce2fe0653696c4cd06d73" dependencies = [ "bitflags 2.13.0", "ctor", @@ -2638,9 +2638,9 @@ checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1" [[package]] name = "napi-derive" -version = "3.5.9" +version = "3.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ba572deef53e2c386759a8c2014175a62679d74ff83adc205c8bc0e0285727" +checksum = "b0fe526e81c105d3640516fcde83909dd1afe757c0d7a15af58830b5bc0fb9a1" dependencies = [ "convert_case", "ctor", @@ -2652,9 +2652,9 @@ dependencies = [ [[package]] name = "napi-derive-backend" -version = "5.1.1" +version = "5.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd961eb2aa8965e3f29722d754f3a86907eb1984e2fbcbe3fe87b9a02d6bfba" +checksum = "514281397bcddd9ea9a876c7a21a57bff2374237a000ca9a64ea0211ec1993e2" dependencies = [ "convert_case", "proc-macro2", @@ -2665,9 +2665,9 @@ dependencies = [ [[package]] name = "napi-sys" -version = "3.2.2" +version = "3.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f5bcdf71abd3a50d00b49c1c2c75251cb3c913777d6139cd37dabc093a5e400" +checksum = "73e43cf2eb0bd1bf95a43c07c076ebd2da5d1e015a71c3d201faeffffcc0ecac" dependencies = [ "libloading", ] @@ -3264,7 +3264,7 @@ dependencies = [ [[package]] name = "pi-ast" -version = "16.4.7" +version = "16.4.8" dependencies = [ "anyhow", "ast-grep-core", @@ -3333,7 +3333,7 @@ dependencies = [ [[package]] name = "pi-iso" -version = "16.4.7" +version = "16.4.8" dependencies = [ "async-trait", "libc", @@ -3345,7 +3345,7 @@ dependencies = [ [[package]] name = "pi-natives" -version = "16.4.7" +version = "16.4.8" dependencies = [ "anyhow", "arboard", @@ -3398,7 +3398,7 @@ dependencies = [ [[package]] name = "pi-shell" -version = "16.4.7" +version = "16.4.8" dependencies = [ "anyhow", "brush-builtins", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "pi-walker" -version = "16.4.7" +version = "16.4.8" dependencies = [ "dashmap", "globset", diff --git a/Cargo.toml b/Cargo.toml index 2d82fa773..c603960b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["crates/vendor/brush-core", "crates/vendor/brush-builtins"] resolver = "3" [workspace.package] -version = "16.4.7" +version = "16.4.8" edition = "2024" license = "MIT" authors = ["Can Boluk"] diff --git a/bun.lock b/bun.lock index 185d6908c..3778bbec5 100644 --- a/bun.lock +++ b/bun.lock @@ -21,7 +21,7 @@ }, "packages/agent": { "name": "@oh-my-pi/pi-agent-core", - "version": "16.4.7", + "version": "16.4.8", "dependencies": { "@oh-my-pi/pi-ai": "catalog:", "@oh-my-pi/pi-catalog": "catalog:", @@ -39,7 +39,7 @@ }, "packages/ai": { "name": "@oh-my-pi/pi-ai", - "version": "16.4.7", + "version": "16.4.8", "dependencies": { "@bufbuild/protobuf": "catalog:", "@oh-my-pi/pi-catalog": "catalog:", @@ -55,7 +55,7 @@ }, "packages/catalog": { "name": "@oh-my-pi/pi-catalog", - "version": "16.4.7", + "version": "16.4.8", "dependencies": { "@bufbuild/protobuf": "catalog:", "@oh-my-pi/pi-utils": "catalog:", @@ -69,7 +69,7 @@ }, "packages/coding-agent": { "name": "@oh-my-pi/pi-coding-agent", - "version": "16.4.7", + "version": "16.4.8", "bin": { "omp": "src/cli.ts", }, @@ -138,7 +138,7 @@ }, "packages/hashline": { "name": "@oh-my-pi/hashline", - "version": "16.4.7", + "version": "16.4.8", "dependencies": { "diff": "catalog:", "lru-cache": "catalog:", @@ -149,7 +149,7 @@ }, "packages/mnemopi": { "name": "@oh-my-pi/pi-mnemopi", - "version": "16.4.7", + "version": "16.4.8", "bin": { "mnemopi": "src/cli.ts", }, @@ -175,7 +175,7 @@ }, "packages/natives": { "name": "@oh-my-pi/pi-natives", - "version": "16.4.7", + "version": "16.4.8", "devDependencies": { "@napi-rs/cli": "catalog:", "@types/bun": "catalog:", @@ -183,7 +183,7 @@ }, "packages/snapcompact": { "name": "@oh-my-pi/snapcompact", - "version": "16.4.7", + "version": "16.4.8", "dependencies": { "@oh-my-pi/pi-ai": "catalog:", "@oh-my-pi/pi-natives": "catalog:", @@ -196,7 +196,7 @@ }, "packages/stats": { "name": "@oh-my-pi/omp-stats", - "version": "16.4.7", + "version": "16.4.8", "bin": { "omp-stats": "./src/index.ts", }, @@ -222,7 +222,7 @@ }, "packages/swarm-extension": { "name": "@oh-my-pi/swarm-extension", - "version": "16.4.7", + "version": "16.4.8", "bin": { "omp-swarm": "src/cli.ts", }, @@ -248,7 +248,7 @@ }, "packages/tui": { "name": "@oh-my-pi/pi-tui", - "version": "16.4.7", + "version": "16.4.8", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "@oh-my-pi/pi-utils": "catalog:", @@ -289,7 +289,7 @@ }, "packages/utils": { "name": "@oh-my-pi/pi-utils", - "version": "16.4.7", + "version": "16.4.8", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "handlebars": "catalog:", @@ -302,7 +302,7 @@ }, "packages/wire": { "name": "@oh-my-pi/pi-wire", - "version": "16.4.7", + "version": "16.4.8", "devDependencies": { "@types/bun": "catalog:", }, @@ -342,18 +342,18 @@ "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.7.0", - "@oh-my-pi/hashline": "16.4.7", - "@oh-my-pi/omp-stats": "16.4.7", - "@oh-my-pi/pi-agent-core": "16.4.7", - "@oh-my-pi/pi-ai": "16.4.7", - "@oh-my-pi/pi-catalog": "16.4.7", - "@oh-my-pi/pi-coding-agent": "16.4.7", - "@oh-my-pi/pi-mnemopi": "16.4.7", - "@oh-my-pi/pi-natives": "16.4.7", - "@oh-my-pi/pi-tui": "16.4.7", - "@oh-my-pi/pi-utils": "16.4.7", - "@oh-my-pi/pi-wire": "16.4.7", - "@oh-my-pi/snapcompact": "16.4.7", + "@oh-my-pi/hashline": "16.4.8", + "@oh-my-pi/omp-stats": "16.4.8", + "@oh-my-pi/pi-agent-core": "16.4.8", + "@oh-my-pi/pi-ai": "16.4.8", + "@oh-my-pi/pi-catalog": "16.4.8", + "@oh-my-pi/pi-coding-agent": "16.4.8", + "@oh-my-pi/pi-mnemopi": "16.4.8", + "@oh-my-pi/pi-natives": "16.4.8", + "@oh-my-pi/pi-tui": "16.4.8", + "@oh-my-pi/pi-utils": "16.4.8", + "@oh-my-pi/pi-wire": "16.4.8", + "@oh-my-pi/snapcompact": "16.4.8", "@opentelemetry/api": "^1.9.1", "@opentelemetry/context-async-hooks": "^2.7.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.218.0", @@ -496,11 +496,11 @@ "@huggingface/blake3-jit": ["@huggingface/blake3-jit@0.0.2", "", {}, "sha512-Bq7B5qabyjrJfhBsl85Jd2QBtf+HzRD7h7A9GfN2lzrrsABhOa5evVPgzoCTxR7Ub0QFj7YDK1YkYRWBU25+2w=="], - "@huggingface/hub": ["@huggingface/hub@2.13.2", "", { "dependencies": { "@huggingface/tasks": "^0.21.13", "@huggingface/xetchunk-wasm": "^0.1.0" }, "optionalDependencies": { "cli-progress": "^3.12.0" }, "bin": { "hfjs": "dist/cli.js" } }, "sha512-I1Pbn9UcBHCt0M/SbZkKzYr3JVnbJp4YIFv74eVRH2Y22yT9Jpm6DtfM74tcjNLbC7SyFZ433znUw0szdFd4Fg=="], + "@huggingface/hub": ["@huggingface/hub@2.13.3", "", { "dependencies": { "@huggingface/tasks": "^0.21.24", "@huggingface/xetchunk-wasm": "^0.1.0" }, "optionalDependencies": { "cli-progress": "^3.12.0" }, "bin": { "hfjs": "dist/cli.js" } }, "sha512-qAEbZ0x1vUQIUSComJ5EPgII+eI3nJB3gTPNTzjir6Kk/aCHHvPfCNMreS4iUg6gT17YDkvt7XBTndzh7617KQ=="], "@huggingface/jinja": ["@huggingface/jinja@0.5.9", "", {}, "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw=="], - "@huggingface/tasks": ["@huggingface/tasks@0.21.20", "", {}, "sha512-pGw7pQI8TYSbrOVfNRdrYeISDEYOvQzOCEBceYjEKOH2wOmooc6eGfbYRhAALDgNKZCSISPYYFszRxHta3FaNw=="], + "@huggingface/tasks": ["@huggingface/tasks@0.21.25", "", {}, "sha512-u71rRx80Ynzy2oEDpRTSgOLyrzAV86gtnt4ajGHHI+1SQmsqmIUyzZG4Wi0TavDKdRsTa5QC/REDeJ3cFvkxvA=="], "@huggingface/tokenizers": ["@huggingface/tokenizers@0.1.3", "", {}, "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA=="], @@ -1224,13 +1224,13 @@ "lru-cache": ["lru-cache@11.5.1", "", {}, "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A=="], - "lucide-react": ["lucide-react@1.23.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw=="], + "lucide-react": ["lucide-react@1.24.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-YT6mBD8lGKkg4nM39enlm94/sfJIiW0YKUT60fBy4YK8tai31ylg1VhGNWxkpSKHo9UagfnZqwIff3HTDQwXeA=="], "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], "mammoth": ["mammoth@1.12.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.6", "argparse": "~1.0.3", "base64-js": "^1.5.1", "bluebird": "~3.4.0", "dingbat-to-unicode": "^1.0.1", "jszip": "^3.7.1", "lop": "^0.4.2", "path-is-absolute": "^1.0.0", "underscore": "^1.13.1", "xmlbuilder": "^10.0.0" }, "bin": { "mammoth": "bin/mammoth" } }, "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w=="], - "marked": ["marked@18.0.5", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w=="], + "marked": ["marked@18.0.6", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-MrV5puXBfuiy6wl6DLaq3BtIJQAJToAd5zt/ZKhRfGRAuFPALE7/4Y7jnxRQoEgK/pBgurGqLyAuRgZ2xOjr6w=="], "matcher": ["matcher@4.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ=="], @@ -1306,7 +1306,7 @@ "postcss": ["postcss@8.5.16", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg=="], - "prettier": ["prettier@3.9.4", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-yWG/o/4oJfo036EKAfK6ACAoDOfHeRHx4tuxkfBZiauURiaSmYwlpOr5LQqKtIkRD2z1PLteme2WoxEnj4tHTg=="], + "prettier": ["prettier@3.9.5", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-/FVl766LpUfB5vXgCYOYa0MeV/441Ia99AeICQIQFTY/Nw0roZwULcXpku5i1/m5kt/baz+s4Zogspd839HSMg=="], "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], @@ -1394,7 +1394,7 @@ "string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], - "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], "strip-ansi": ["strip-ansi@7.2.0", "", { "dependencies": { "ansi-regex": "^6.2.2" } }, "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w=="], @@ -1572,6 +1572,8 @@ "roarr/sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], + "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + "wrap-ansi/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], @@ -1594,6 +1596,8 @@ "htmlparser2/domutils/dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + "jszip/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "@huggingface/transformers/onnxruntime-node/global-agent/matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="], "@huggingface/transformers/onnxruntime-node/global-agent/serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], diff --git a/crates/pi-natives/src/lib.rs b/crates/pi-natives/src/lib.rs index e57f51c9c..2ef74efb6 100644 --- a/crates/pi-natives/src/lib.rs +++ b/crates/pi-natives/src/lib.rs @@ -248,7 +248,7 @@ fn create_windows_napi_tokio_runtime() -> Option { /// MUST stay in sync with `VERSION_SENTINEL_EXPORT` in /// `packages/natives/native/index.js` (which derives the name from /// `package.json#version`). -#[napi(js_name = "__piNativesV16_4_7")] +#[napi(js_name = "__piNativesV16_4_8")] pub const fn pi_natives_version_sentinel() {} /// Native module entry point: install crash diagnostics before any tool can diff --git a/package.json b/package.json index 65f3d5793..5865a13bc 100644 --- a/package.json +++ b/package.json @@ -25,18 +25,18 @@ "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.7.0", - "@oh-my-pi/hashline": "16.4.7", - "@oh-my-pi/omp-stats": "16.4.7", - "@oh-my-pi/pi-agent-core": "16.4.7", - "@oh-my-pi/pi-ai": "16.4.7", - "@oh-my-pi/pi-catalog": "16.4.7", - "@oh-my-pi/pi-coding-agent": "16.4.7", - "@oh-my-pi/pi-mnemopi": "16.4.7", - "@oh-my-pi/pi-natives": "16.4.7", - "@oh-my-pi/pi-tui": "16.4.7", - "@oh-my-pi/pi-utils": "16.4.7", - "@oh-my-pi/pi-wire": "16.4.7", - "@oh-my-pi/snapcompact": "16.4.7", + "@oh-my-pi/hashline": "16.4.8", + "@oh-my-pi/omp-stats": "16.4.8", + "@oh-my-pi/pi-agent-core": "16.4.8", + "@oh-my-pi/pi-ai": "16.4.8", + "@oh-my-pi/pi-catalog": "16.4.8", + "@oh-my-pi/pi-coding-agent": "16.4.8", + "@oh-my-pi/pi-mnemopi": "16.4.8", + "@oh-my-pi/pi-natives": "16.4.8", + "@oh-my-pi/pi-tui": "16.4.8", + "@oh-my-pi/pi-utils": "16.4.8", + "@oh-my-pi/pi-wire": "16.4.8", + "@oh-my-pi/snapcompact": "16.4.8", "@opentelemetry/api": "^1.9.1", "@opentelemetry/context-async-hooks": "^2.7.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.218.0", diff --git a/packages/agent/package.json b/packages/agent/package.json index 0ac34f9a2..ef3ea4a70 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-agent-core", - "version": "16.4.7", + "version": "16.4.8", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/ai/package.json b/packages/ai/package.json index 9e28b73ba..2ebedffb1 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-ai", - "version": "16.4.7", + "version": "16.4.8", "description": "Unified LLM API with automatic model discovery and provider configuration", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/catalog/package.json b/packages/catalog/package.json index 3f23ba236..98580554d 100644 --- a/packages/catalog/package.json +++ b/packages/catalog/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-catalog", - "version": "16.4.7", + "version": "16.4.8", "description": "Model catalog for omp: bundled model database, provider discovery descriptors, model identity, classification, and equivalence", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 7de11cc4d..10f7883ed 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [16.4.8] - 2026-07-12 + ### Fixed - Improved search reliability for Perplexity provider by forcing retrieval for all queries diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 8b72da063..57a83ecfa 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-coding-agent", - "version": "16.4.7", + "version": "16.4.8", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/hashline/package.json b/packages/hashline/package.json index 8a890adff..208dcaaac 100644 --- a/packages/hashline/package.json +++ b/packages/hashline/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/hashline", - "version": "16.4.7", + "version": "16.4.8", "description": "Hashline: a compact, line-anchored patch language and applier. Pluggable FS/IO so it works over disk, in-memory, or any custom backend.", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/mnemopi/package.json b/packages/mnemopi/package.json index 40a543674..3bb0b77fa 100644 --- a/packages/mnemopi/package.json +++ b/packages/mnemopi/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-mnemopi", - "version": "16.4.7", + "version": "16.4.8", "description": "Local SQLite memory engine for Oh My Pi agents", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/natives/native/index.d.ts b/packages/natives/native/index.d.ts index d693eca63..ebdba17bd 100644 --- a/packages/natives/native/index.d.ts +++ b/packages/natives/native/index.d.ts @@ -170,7 +170,7 @@ export declare function __ompInstallTokioRuntime(): void * `packages/natives/native/index.js` (which derives the name from * `package.json#version`). */ -export declare function __piNativesV16_4_7(): void +export declare function __piNativesV16_4_8(): void /** * Apply ast-grep rewrite rules to matching files; honors `dryRun` and returns diff --git a/packages/natives/native/index.js b/packages/natives/native/index.js index 5b03bfece..d5a41e427 100644 --- a/packages/natives/native/index.js +++ b/packages/natives/native/index.js @@ -24,7 +24,7 @@ export const Shell = nativeBindings.Shell; // functions export const __ompInstallTokioRuntime = nativeBindings.__ompInstallTokioRuntime; -export const __piNativesV16_4_7 = nativeBindings.__piNativesV16_4_7; +export const __piNativesV16_4_8 = nativeBindings.__piNativesV16_4_8; export const astEdit = nativeBindings.astEdit; export const astGrep = nativeBindings.astGrep; export const astMatch = nativeBindings.astMatch; diff --git a/packages/natives/package.json b/packages/natives/package.json index 3ec513cda..ed10b62f8 100644 --- a/packages/natives/package.json +++ b/packages/natives/package.json @@ -1,6 +1,6 @@ { "name": "@oh-my-pi/pi-natives", - "version": "16.4.7", + "version": "16.4.8", "description": "Native Rust bindings for grep, clipboard, image processing, syntax highlighting, PTY, and shell operations via N-API", "type": "module", "homepage": "https://omp.sh", diff --git a/packages/snapcompact/package.json b/packages/snapcompact/package.json index 811e2944a..0779a81fc 100644 --- a/packages/snapcompact/package.json +++ b/packages/snapcompact/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/snapcompact", - "version": "16.4.7", + "version": "16.4.8", "description": "Bitmap-frame context compression for vision-capable LLMs", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/stats/package.json b/packages/stats/package.json index 73f9a7cc3..91cdf6e29 100644 --- a/packages/stats/package.json +++ b/packages/stats/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/omp-stats", - "version": "16.4.7", + "version": "16.4.8", "description": "Local observability dashboard for pi AI usage statistics", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/swarm-extension/package.json b/packages/swarm-extension/package.json index ef4d3857a..e82a8ad0a 100644 --- a/packages/swarm-extension/package.json +++ b/packages/swarm-extension/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/swarm-extension", - "version": "16.4.7", + "version": "16.4.8", "description": "Swarm orchestration extension for omp", "homepage": "https://omp.sh", "author": "Derek Rynd", diff --git a/packages/tui/package.json b/packages/tui/package.json index 3bc7341bb..6d132f385 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-tui", - "version": "16.4.7", + "version": "16.4.8", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/utils/package.json b/packages/utils/package.json index 109ec1ead..f102195e5 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-utils", - "version": "16.4.7", + "version": "16.4.8", "description": "Shared utilities for pi packages", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/wire/package.json b/packages/wire/package.json index a91d7bb34..21e9c8e49 100644 --- a/packages/wire/package.json +++ b/packages/wire/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-wire", - "version": "16.4.7", + "version": "16.4.8", "description": "Shared wire protocol types for Oh My Pi packages", "homepage": "https://omp.sh", "author": "Can Boluk", From bd7d395222b128ce383be942329867251f75b5f0 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 16:04:07 +0200 Subject: [PATCH 34/80] feat(coding-agent/tools): added predicate polling support to wait() - Enhanced `wait()` in browser tools to accept a predicate function in addition to milliseconds. - Implemented automatic polling with configurable timeout and interval, resolving with the first truthy value. - Throws a descriptive `ToolError` on timeout instead of waiting for the full execution deadline. - Added comprehensive unit tests to verify polling behavior, cancellation, and error handling. --- .../coding-agent/src/prompts/tools/browser.md | 2 +- .../src/tools/browser/cmux/cmux-tab.ts | 5 +- .../src/tools/browser/run-cancellation.ts | 51 ++++++++++++++++--- .../src/tools/browser/tab-worker.ts | 5 +- .../tools/browser-run-cancellation.test.ts | 40 ++++++++++++++- 5 files changed, 91 insertions(+), 12 deletions(-) diff --git a/packages/coding-agent/src/prompts/tools/browser.md b/packages/coding-agent/src/prompts/tools/browser.md index e0f6d22ff..30d193b24 100644 --- a/packages/coding-agent/src/prompts/tools/browser.md +++ b/packages/coding-agent/src/prompts/tools/browser.md @@ -5,7 +5,7 @@ Drives real Chromium tab; full puppeteer access via JS. - Three actions: - `open` — acquire/reuse named tab (`name` defaults `"main"`). Optional `url` (navigate once ready), `viewport`, `dialogs: "accept" | "dismiss"` (auto-handle `alert`/`confirm`/`beforeunload`; else page hangs till you wire `page.on('dialog', …)`). - `close` — release tab by `name`, or all with `all: true`. `kill: true` also kills spawned-app process trees. - - `run` — execute JS in existing tab. `code` = async function body; `page`, `browser`, `tab`, `display`, `assert`, `wait` in scope. Return value JSON-stringified into result; `display(value)` accumulates text/images. + - `run` — execute JS in existing tab. `code` = async function body; `page`, `browser`, `tab`, `display`, `assert`, `wait` in scope. Return value JSON-stringified into result; `display(value)` accumulates text/images. `wait(ms)` sleeps; `wait(fn, { timeout?, interval? })` polls `fn` (sync or async) until truthy and resolves with that value (default 30s timeout, 100ms interval; named error on timeout) — use it instead of in-page polling Promises inside `tab.evaluate`. - Tabs survive `run` calls and in-process subagents — open once, reuse. - Browser kinds (`app` on `open`): - default (no `app`) → headless Chromium with stealth patches. diff --git a/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts b/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts index 262f4d19d..b4bcdbf70 100644 --- a/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts +++ b/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts @@ -12,7 +12,7 @@ import { ToolAbortError, ToolError, throwIfAborted } from "../../tool-errors"; import { type AriaSnapshotOptions, buildAriaSnapshotScript } from "../aria/aria-snapshot"; import { DEFAULT_VIEWPORT } from "../launch"; import { extractReadableFromHtml, type ReadableFormat } from "../readable"; -import { bindBrowserRunFacade, waitForBrowserRun } from "../run-cancellation"; +import { bindBrowserRunFacade, type WaitPredicateOptions, waitForBrowserRun } from "../run-cancellation"; import { cloneSafe, RunOutput } from "../run-output"; import type { Observation, ReadyInfo, RunResultOk, ScreenshotResult, SessionSnapshot } from "../tab-protocol"; import { @@ -1352,7 +1352,8 @@ export async function runCmuxCode(tab: CmuxTab, opts: RunCmuxCodeOptions): Promi assert: (cond: unknown, text?: string): void => { if (!cond) throw new ToolError(text ?? "Assertion failed"); }, - wait: (ms: number): Promise => waitForBrowserRun(ms, signal), + wait: (msOrPredicate: number | (() => unknown), opts?: WaitPredicateOptions): Promise => + waitForBrowserRun(msOrPredicate, signal, opts), }); const hooks: RuntimeHooks = { diff --git a/packages/coding-agent/src/tools/browser/run-cancellation.ts b/packages/coding-agent/src/tools/browser/run-cancellation.ts index 3d5057752..41460524d 100644 --- a/packages/coding-agent/src/tools/browser/run-cancellation.ts +++ b/packages/coding-agent/src/tools/browser/run-cancellation.ts @@ -1,5 +1,5 @@ import { untilAborted } from "@oh-my-pi/pi-utils"; -import { throwIfAborted } from "../tool-errors"; +import { ToolError, throwIfAborted } from "../tool-errors"; /** * Marks a run-scoped promise as observed without changing its behavior for awaited callers. @@ -16,12 +16,51 @@ export function markHandled(promise: Promise): Promise { return promise; } -/** Sleeps inside evaluated browser code while honoring the owning run's cancellation signal. */ -export function waitForBrowserRun(ms: number, signal: AbortSignal): Promise { - const promise = (async (): Promise => { - throwIfAborted(signal); - await untilAborted(signal, () => Bun.sleep(ms)); +/** Options for the predicate form of the run-scoped `wait()` helper. */ +export interface WaitPredicateOptions { + /** Max time to poll before failing, in ms (default 30_000). */ + timeout?: number; + /** Poll interval in ms (default 100, floor 10). */ + interval?: number; +} + +/** + * Run-scoped `wait()` helper for evaluated browser code, honoring the owning run's + * cancellation signal. + * + * - `wait(ms)` sleeps for `ms` milliseconds. + * - `wait(fn, { timeout?, interval? })` polls `fn` (sync or async) until it returns a + * truthy value and resolves with that value; throws a named `ToolError` on timeout + * instead of stalling into the whole-cell deadline. Predicate errors propagate. + */ +export function waitForBrowserRun( + msOrPredicate: number | (() => unknown), + signal: AbortSignal, + opts?: WaitPredicateOptions, +): Promise { + const promise = (async (): Promise => { throwIfAborted(signal); + if (typeof msOrPredicate === "number") { + await untilAborted(signal, async () => await Bun.sleep(msOrPredicate)); + throwIfAborted(signal); + return undefined; + } + if (typeof msOrPredicate !== "function") { + throw new ToolError("wait(...) expects milliseconds (number) or a predicate function to poll"); + } + const timeout = + opts?.timeout !== undefined && Number.isFinite(opts.timeout) && opts.timeout > 0 ? opts.timeout : 30_000; + const interval = Math.max(opts?.interval ?? 100, 10); + const deadline = Date.now() + timeout; + for (;;) { + const value = await untilAborted(signal, async () => await msOrPredicate()); + throwIfAborted(signal); + if (value) return value; + if (Date.now() + interval > deadline) { + throw new ToolError(`wait(predicate) timed out after ${timeout}ms — predicate never returned truthy`); + } + await untilAborted(signal, async () => await Bun.sleep(interval)); + } })(); return markHandled(promise); } diff --git a/packages/coding-agent/src/tools/browser/tab-worker.ts b/packages/coding-agent/src/tools/browser/tab-worker.ts index 546cc8e35..973d624a2 100644 --- a/packages/coding-agent/src/tools/browser/tab-worker.ts +++ b/packages/coding-agent/src/tools/browser/tab-worker.ts @@ -35,7 +35,7 @@ import { loadPuppeteerInWorker, } from "./launch"; import { extractReadableFromHtml, type ReadableFormat } from "./readable"; -import { markHandled, waitForBrowserRun } from "./run-cancellation"; +import { markHandled, type WaitPredicateOptions, waitForBrowserRun } from "./run-cancellation"; import { cloneSafe, RunOutput } from "./run-output"; import type { Observation, @@ -761,7 +761,8 @@ export class WorkerCore { assert: (cond: unknown, text?: string): void => { if (!cond) throw new ToolError(text ?? "Assertion failed"); }, - wait: (ms: number): Promise => waitForBrowserRun(ms, signal), + wait: (msOrPredicate: number | (() => unknown), opts?: WaitPredicateOptions): Promise => + waitForBrowserRun(msOrPredicate, signal, opts), }); const { promise: cancelRejection, reject: rejectCancel } = Promise.withResolvers(); const onCancel = (): void => { diff --git a/packages/coding-agent/test/tools/browser-run-cancellation.test.ts b/packages/coding-agent/test/tools/browser-run-cancellation.test.ts index 4d451fa49..876b56817 100644 --- a/packages/coding-agent/test/tools/browser-run-cancellation.test.ts +++ b/packages/coding-agent/test/tools/browser-run-cancellation.test.ts @@ -57,6 +57,44 @@ describe("browser run cancellation", () => { await expect(wait).rejects.toThrow("browser run ended"); }); + it("resolves wait(predicate) with the first truthy value", async () => { + vi.useRealTimers(); + const controller = new AbortController(); + let calls = 0; + + const wait = waitForBrowserRun(() => (++calls >= 3 ? "ready" : null), controller.signal, { interval: 10 }); + + await expect(wait).resolves.toBe("ready"); + expect(calls).toBe(3); + }); + + it("fails wait(predicate) with a named timeout error instead of stalling", async () => { + vi.useRealTimers(); + const controller = new AbortController(); + + const wait = waitForBrowserRun(() => false, controller.signal, { timeout: 50, interval: 10 }); + + await expect(wait).rejects.toThrow("wait(predicate) timed out after 50ms"); + }); + + it("rejects wait(predicate) when the run aborts mid-poll", async () => { + vi.useRealTimers(); + const controller = new AbortController(); + + const wait = waitForBrowserRun(() => false, controller.signal, { timeout: 5000 }); + controller.abort(new Error("browser run ended")); + + await expect(wait).rejects.toThrow("browser run ended"); + }); + + it("rejects wait() input that is neither milliseconds nor a predicate", async () => { + const controller = new AbortController(); + + await expect(waitForBrowserRun("soon" as never, controller.signal)).rejects.toThrow( + "wait(...) expects milliseconds (number) or a predicate function to poll", + ); + }); + it("does not emit unhandledRejection for an unawaited wait aborted by run teardown", async () => { const controller = new AbortController(); @@ -124,7 +162,7 @@ describe("browser run cancellation", () => { once: true, }); runtime.setRunScope({ - wait: (ms: number): Promise => waitForBrowserRun(ms, signal), + wait: (ms: number): Promise => waitForBrowserRun(ms, signal), tab: bindBrowserRunFacade( { goto: async (url: string): Promise => { From 69865b609e8c3af94cc1ba3f5e45762986c3b1c2 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 16:04:32 +0200 Subject: [PATCH 35/80] feat(coding-agent/prompts): updated system prompt verification guidelines - Shifted testing expectations from mandatory inclusion to proof-based verification depending on the task type. - Clarified that experiments, UI changes, and investigations do not require tests unless specific conditions are met. - Reorganized the verification workflow to prioritize behavioral smoke testing over automated test execution for non-permanent changes. --- packages/coding-agent/CHANGELOG.md | 3 +++ .../src/prompts/system/system-prompt.md | 16 +++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 10f7883ed..e47320413 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -3,6 +3,9 @@ ## [Unreleased] ## [16.4.8] - 2026-07-12 +### Changed + +- Refined agent workflow to prioritize smoke testing and reduce mandatory upfront test generation ### Fixed diff --git a/packages/coding-agent/src/prompts/system/system-prompt.md b/packages/coding-agent/src/prompts/system/system-prompt.md index 1e45733ed..5289091c5 100644 --- a/packages/coding-agent/src/prompts/system/system-prompt.md +++ b/packages/coding-agent/src/prompts/system/system-prompt.md @@ -192,7 +192,7 @@ EXECUTION WORKFLOW # 3. Decompose - Update todos as you go; skip them for trivial requests. Marking a todo done is a transition: start the next in the same turn. -- Plan only what makes the request work. Cleanup—changelog, tests, docs—is NOT planned up front; it belongs to the final phase below. +- Plan only what makes the request work. Cleanup—changelog, docs, removing scaffolding—is NOT planned up front; it belongs to the final phase below. Tests are cleanup only for permanent feature/bug-fix work (see Cleanup). # 4. Implement - Fix problems at the source. Remove obsolete code—no leftover comments, aliases, or re-exports. @@ -202,14 +202,16 @@ EXECUTION WORKFLOW {{#has tools "ask"}}- Ask before destructive commands or deleting code you didn't write.{{else}}- Don't run destructive git commands or delete code you didn't write.{{/has}} # 5. Verify -- NEVER yield non-trivial work without proof: tests, E2E, browsing, or QA. -- Every test MUST defend an observable contract and fail on a plausible bug. -- Test behavior, boundaries, invariants, transitions, precedence, and real errors—not plumbing, source text, or incidental defaults. -- Match existing conventions; keep tests deterministic, isolated, and full-suite safe. -- Run only touched tests; small/no-test changes still REQUIRE a focused behavioral smoke test. +- NEVER yield non-trivial work without proof that the deliverable works. The proof method depends on the ask: + - **Experiment / investigation** → run it. The output IS the proof. No tests. + - **UI change** → drive it in browser. Visual confirmation IS the proof. No tests unless the existing suite breaks and the break is real. + - **Bug fix** → reproduce the bug, apply the fix, confirm the reproduction no longer triggers. + - **Permanent feature / API change** → existing tests that cover the changed contract. Add a test only when the change introduces a new observable contract not already covered, or the user asked for one. +- Smoke test: run the thing, not a test file. Launch it, exercise the changed path, observe the result. +- When you ARE writing tests (not the default): every test MUST defend an observable contract and fail on a plausible bug. Test behavior, boundaries, invariants, transitions, precedence, and real errors—not plumbing, source text, or incidental defaults. Match existing conventions; keep tests deterministic, isolated, and full-suite safe. # 6. Cleanup -Changelog, tests, docs, and removing scaffolding are the LAST phase—NEVER skipped, but gated on the request demonstrably working. +Changelog and removing scaffolding are the LAST phase—NEVER skipped, but gated on the request demonstrably working. Tests and docs are cleanup ONLY when the work is a permanent feature change or bug fix, not for experiments or one-off investigations. - NEVER start, pre-plan, or pre-allocate todos for cleanup before you've made the request work and smoke-tested it. Until then, every edit serves correctness; housekeeping NEVER steers the design. - Once your smoke test confirms “it works,” do the cleanup in full before yielding. From 0d07da529a78a9928546430d1c3d8cc953652929 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 16:27:12 +0200 Subject: [PATCH 36/80] feat(coding-agent-tools): implemented browser execution safety controls - Implemented cell budget clamping for timeouts to prevent stalled browser operations from exceeding execution limits. - Added `recover` capabilities for tab workers to clear blocking dialogs and safely terminate hung navigations. - Introduced `//!world=main` support for `tab.evaluate` via Puppeteer patch to allow execution in main execution contexts. - Improved failure attribution for tab terminations by tracking dialogs, stalled operations, and specific termination reasons. --- packages/coding-agent/CHANGELOG.md | 11 ++ .../coding-agent/src/prompts/tools/browser.md | 8 +- .../src/tools/browser/cmux/cmux-tab.ts | 20 ++- .../src/tools/browser/run-cancellation.ts | 25 ++- .../src/tools/browser/tab-protocol.ts | 6 + .../src/tools/browser/tab-supervisor.ts | 18 ++- .../src/tools/browser/tab-worker.ts | 148 +++++++++++++++--- .../test/tools/browser-tab-timeouts.test.ts | 26 +++ patches/puppeteer-core@25.3.0.patch | 47 ++++-- 9 files changed, 262 insertions(+), 47 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index e47320413..0df0c3212 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -3,12 +3,23 @@ ## [Unreleased] ## [16.4.8] - 2026-07-12 +### Added + +- Added a predicate form to the browser run's `wait()` helper: `wait(fn, { timeout?, interval? })` polls the function (sync or async) until truthy and resolves with that value, failing with a named timeout error (deadline clamped under the cell budget so it always beats the opaque whole-cell timeout) instead of Bun's `sleep expects a number` or a whole-cell stall from in-page polling Promises; both `wait` forms now register in the stall diagnosis of cell timeouts + ### Changed +- Improved tab recovery after timeouts by automatically clearing pending navigation and JS dialogs +- Made `tab.goto` navigation failures catchable with a named error instead of triggering a whole-cell timeout +- Added support for `//!world=main` in `tab.evaluate` to access page-level globals +- Enhanced cell timeout messages to include identification of stalled operations and blocking JS dialogs +- Browser `run` on a tab the supervisor force-killed now reports the kill reason instead of a bare "not alive" - Refined agent workflow to prioritize smoke testing and reduce mandatory upfront test generation ### Fixed +- Fixed the `//!world=main` evaluate directive being silently ignored for string expressions passed through `page.evaluate`/`tab.evaluate`: puppeteer's source-URL tagging boxed the string into a `String` object before the world check +- Fixed tab reuse issues where hung navigation or unhandled modals would cause initialization to stall and trigger a force-kill - Improved search reliability for Perplexity provider by forcing retrieval for all queries - Fixed JS eval cells losing top-level `function` and `var` declarations across cells when the defining cell contained top-level `await` — the async wrapper scoped them to the cell's IIFE instead of publishing them to the worker global diff --git a/packages/coding-agent/src/prompts/tools/browser.md b/packages/coding-agent/src/prompts/tools/browser.md index 30d193b24..0e96346c9 100644 --- a/packages/coding-agent/src/prompts/tools/browser.md +++ b/packages/coding-agent/src/prompts/tools/browser.md @@ -5,7 +5,7 @@ Drives real Chromium tab; full puppeteer access via JS. - Three actions: - `open` — acquire/reuse named tab (`name` defaults `"main"`). Optional `url` (navigate once ready), `viewport`, `dialogs: "accept" | "dismiss"` (auto-handle `alert`/`confirm`/`beforeunload`; else page hangs till you wire `page.on('dialog', …)`). - `close` — release tab by `name`, or all with `all: true`. `kill: true` also kills spawned-app process trees. - - `run` — execute JS in existing tab. `code` = async function body; `page`, `browser`, `tab`, `display`, `assert`, `wait` in scope. Return value JSON-stringified into result; `display(value)` accumulates text/images. `wait(ms)` sleeps; `wait(fn, { timeout?, interval? })` polls `fn` (sync or async) until truthy and resolves with that value (default 30s timeout, 100ms interval; named error on timeout) — use it instead of in-page polling Promises inside `tab.evaluate`. + - `run` — execute JS in existing tab. `code` = async function body; `page`, `browser`, `tab`, `display`, `assert`, `wait` in scope. Return value JSON-stringified into result; `display(value)` accumulates text/images. `wait(ms)` sleeps; `wait(fn, { timeout?, interval? })` polls `fn` (sync or async) until truthy and resolves with that value (default 100ms interval; deadline min(30s, cell budget − 1s), named error on timeout) — use it instead of in-page polling Promises inside `tab.evaluate`. - Tabs survive `run` calls and in-process subagents — open once, reuse. - Browser kinds (`app` on `open`): - default (no `app`) → headless Chromium with stealth patches. @@ -13,7 +13,7 @@ Drives real Chromium tab; full puppeteer access via JS. - `app.cdp_url` → connect to existing CDP endpoint (e.g. `http://127.0.0.1:9222`). - `app.target` (with `path`/`cdp_url`) — substring on url+title picks BrowserWindow. - `tab` helpers; drop to raw puppeteer `page` for anything uncovered: - - `tab.goto(url, { waitUntil? })` — navigate. + - `tab.goto(url, { waitUntil? })` — navigate. A hung load fails ~1s before the cell budget with a named, catchable error and the pending navigation is stopped; for slow pages raise `timeout` or use `waitUntil: "domcontentloaded"`. - `tab.observe({ includeAll?, viewportOnly? })` — accessibility snapshot: `{ url, title, viewport, scroll, elements: [{ id, role, name, value, states, … }] }`. Ids stable until next observe/goto. - `tab.ariaSnapshot(selector?, { depth?, boxes? })` — Playwright-format ARIA-tree YAML (nested roles + accessible names + `/url`/`/placeholder`), scoped to `selector` or the whole document. Every node carries a `[ref=eN]` id; `[cursor=pointer]` flags clickables. Captures dense, hierarchical structure/text that `observe()`'s flat list flattens away. Refs renumber from e1 each call and stay valid until the next `ariaSnapshot()`. - `tab.ref("e5")` — `[ref=eN]` from the last ariaSnapshot → element handle with the common action methods (`.click()`, `.type()`, `.fill()`, `.hover()`, `.evaluate()`, …); the primary way to act on a ref. For convenience `aria-ref=e5` also works inline in `tab.click`/`type`/`fill`/`waitFor`/`scrollIntoView` (e.g. `tab.click("aria-ref=e5")`). @@ -27,10 +27,10 @@ Drives real Chromium tab; full puppeteer access via JS. - `tab.waitForUrl(pattern, { timeout? })` — substring or `RegExp` (matches SPA pushState nav); returns matched URL. - `tab.waitForResponse(pattern, { timeout? })` — substring, `RegExp`, or `(response) => boolean`; returns puppeteer `HTTPResponse` (`.text()`/`.json()`/`.status()`/`.headers()`). - `tab.waitForNavigation({ waitUntil?, timeout? })` — resolves on the next navigation. Start it BEFORE the click/submit that triggers it; after `tab.goto` (which already waits) use `tab.waitForUrl`/`tab.waitForSelector` instead. - - `tab.evaluate(fn, …args)` — `page.evaluate` for ad-hoc DOM reads. + - `tab.evaluate(fn, …args)` — `page.evaluate` for ad-hoc DOM reads. Runs in an ISOLATED world: DOM, `document.title`, `document.fonts` are shared, but the page's OWN JS globals (`window.myFlag`, app state, framework internals) are NOT visible and reads return `undefined`. To touch page globals, start the function body with the main-world directive: `tab.evaluate(() => { //!world=main` `return window.__done; })` (or `/*!world=main*/`; also honored by string expressions and `page.waitForFunction`). - `tab.screenshot({ selector?, fullPage?, save?, silent? })` — capture + attach for viewing (`silent: true` skips). Pass `save` only when a later step needs the file. - `tab.extract(format = "markdown")` — readable page content (`"markdown"` | `"text"`); throws when nothing readable. -- Selectors: CSS + puppeteer handlers `aria/Sign in`, `text/Continue`, `xpath/…`, `pierce/…`; also Playwright-style `p-aria/…`, `p-text/…`. Playwright-only engines/pseudos (`:has-text()`, `:visible`, …) are rejected — use `text/…` or `aria/…`. A stalled action/wait fails fast with a named `tab.` error carrying a match-count diagnosis, never the whole-cell timeout; a selector matching nothing fails in ~2s (pass an explicit `{ timeout }` to `waitFor`/`waitForSelector` to wait out slow-appearing elements). +- Selectors: CSS + puppeteer handlers `aria/Sign in`, `text/Continue`, `xpath/…`, `pierce/…`; also Playwright-style `p-aria/…`, `p-text/…`. Playwright-only engines/pseudos (`:has-text()`, `:visible`, …) are rejected — use `text/…` or `aria/…`. A stalled action/wait fails fast with a named `tab.` error carrying a match-count diagnosis, never the whole-cell timeout; a selector matching nothing fails in ~2s (pass an explicit `{ timeout }` to `waitFor`/`waitForSelector` to wait out slow-appearing elements). A whole-cell timeout names the stalled op (including `wait(...)`) and any unhandled dialog blocking the page. diff --git a/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts b/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts index b4bcdbf70..950fceeb4 100644 --- a/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts +++ b/packages/coding-agent/src/tools/browser/cmux/cmux-tab.ts @@ -12,7 +12,12 @@ import { ToolAbortError, ToolError, throwIfAborted } from "../../tool-errors"; import { type AriaSnapshotOptions, buildAriaSnapshotScript } from "../aria/aria-snapshot"; import { DEFAULT_VIEWPORT } from "../launch"; import { extractReadableFromHtml, type ReadableFormat } from "../readable"; -import { bindBrowserRunFacade, type WaitPredicateOptions, waitForBrowserRun } from "../run-cancellation"; +import { + bindBrowserRunFacade, + resolvePredicateTimeout, + type WaitPredicateOptions, + waitForBrowserRun, +} from "../run-cancellation"; import { cloneSafe, RunOutput } from "../run-output"; import type { Observation, ReadyInfo, RunResultOk, ScreenshotResult, SessionSnapshot } from "../tab-protocol"; import { @@ -1352,8 +1357,17 @@ export async function runCmuxCode(tab: CmuxTab, opts: RunCmuxCodeOptions): Promi assert: (cond: unknown, text?: string): void => { if (!cond) throw new ToolError(text ?? "Assertion failed"); }, - wait: (msOrPredicate: number | (() => unknown), opts?: WaitPredicateOptions): Promise => - waitForBrowserRun(msOrPredicate, signal, opts), + wait: (msOrPredicate: number | (() => unknown), waitOpts?: WaitPredicateOptions): Promise => + waitForBrowserRun( + msOrPredicate, + signal, + typeof msOrPredicate === "number" + ? waitOpts + : { + timeout: resolvePredicateTimeout(opts.timeoutMs, waitOpts?.timeout), + interval: waitOpts?.interval, + }, + ), }); const hooks: RuntimeHooks = { diff --git a/packages/coding-agent/src/tools/browser/run-cancellation.ts b/packages/coding-agent/src/tools/browser/run-cancellation.ts index 41460524d..d2313af14 100644 --- a/packages/coding-agent/src/tools/browser/run-cancellation.ts +++ b/packages/coding-agent/src/tools/browser/run-cancellation.ts @@ -16,14 +16,33 @@ export function markHandled(promise: Promise): Promise { return promise; } +/** Headroom subtracted from the cell budget so an in-run deadline fires before the opaque whole-cell timeout. */ +export const CELL_BUDGET_SLACK_MS = 1_000; + +/** Default poll deadline for `wait(predicate)` before clamping to the cell budget. */ +export const DEFAULT_PREDICATE_TIMEOUT_MS = 30_000; + /** Options for the predicate form of the run-scoped `wait()` helper. */ export interface WaitPredicateOptions { - /** Max time to poll before failing, in ms (default 30_000). */ + /** Max time to poll before failing, in ms (default 30s, clamped to the cell budget). */ timeout?: number; /** Poll interval in ms (default 100, floor 10). */ interval?: number; } +/** + * Effective `wait(predicate)` deadline for a given cell budget. Always strictly below + * the cell budget so the named `wait(predicate) timed out` error wins the race against + * the opaque whole-cell "Browser code execution timed out". `0`/`Infinity` ("disable") + * map to the largest bounded deadline; negative/NaN garbage falls back to the default. + */ +export function resolvePredicateTimeout(cellTimeoutMs: number, explicit?: number): number { + const budgetBound = Math.max(1, cellTimeoutMs - CELL_BUDGET_SLACK_MS); + if (explicit === 0 || explicit === Number.POSITIVE_INFINITY) return budgetBound; + if (explicit !== undefined && Number.isFinite(explicit) && explicit > 0) return Math.min(explicit, budgetBound); + return Math.min(DEFAULT_PREDICATE_TIMEOUT_MS, budgetBound); +} + /** * Run-scoped `wait()` helper for evaluated browser code, honoring the owning run's * cancellation signal. @@ -49,7 +68,9 @@ export function waitForBrowserRun( throw new ToolError("wait(...) expects milliseconds (number) or a predicate function to poll"); } const timeout = - opts?.timeout !== undefined && Number.isFinite(opts.timeout) && opts.timeout > 0 ? opts.timeout : 30_000; + opts?.timeout !== undefined && Number.isFinite(opts.timeout) && opts.timeout > 0 + ? opts.timeout + : DEFAULT_PREDICATE_TIMEOUT_MS; const interval = Math.max(opts?.interval ?? 100, 10); const deadline = Date.now() + timeout; for (;;) { diff --git a/packages/coding-agent/src/tools/browser/tab-protocol.ts b/packages/coding-agent/src/tools/browser/tab-protocol.ts index d2f5d51bd..96c946c4a 100644 --- a/packages/coding-agent/src/tools/browser/tab-protocol.ts +++ b/packages/coding-agent/src/tools/browser/tab-protocol.ts @@ -59,6 +59,12 @@ export type WorkerInitPayload = safeDir: string; targetId: string; dialogs?: "accept" | "dismiss"; + /** + * Post-timeout recycle: before adopting the page, dismiss any open JS dialog and + * stop a pending navigation so a blocked target cannot stall worker init (which + * previously force-killed the tab). Never set for first-time Electron attach. + */ + recover?: boolean; }; export type ToolReply = { ok: true; value: unknown } | { ok: false; error: RunErrorPayload }; diff --git a/packages/coding-agent/src/tools/browser/tab-supervisor.ts b/packages/coding-agent/src/tools/browser/tab-supervisor.ts index 921fb23ff..0d0a7324a 100644 --- a/packages/coding-agent/src/tools/browser/tab-supervisor.ts +++ b/packages/coding-agent/src/tools/browser/tab-supervisor.ts @@ -131,6 +131,10 @@ const tabs = new Map(); // awaits) cannot interleave and leak a worker + browser refCount. const acquireChains = new Map>(); const GRACE_MS = 750; +// Names of tabs the supervisor force-killed (timeout past grace, failed recycle), +// mapped to the kill reason. Lets the next `run` on that name explain WHY the tab +// vanished instead of a bare "not alive". Cleared when the name is opened again. +const killedTabs = new Map(); export function getTab(name: string): TabSession | undefined { return tabs.get(name); @@ -161,6 +165,7 @@ async function acquireTabImpl( if (opts.signal?.aborted) { throw new ToolAbortError("Browser tab open aborted"); } + killedTabs.delete(name); // Temporary refCount hold so releasing an existing tab on the SAME browser // below cannot drop it to refCount 0 and dispose the instance we are about // to reuse (e.g. reopening the sole tab with a different dialogs policy). @@ -386,7 +391,14 @@ async function runInTabWithSnapshot( snapshot: SessionSnapshot, ): Promise { const tab = tabs.get(name); - if (!tab || tab.state === "dead") throw new ToolError(`Tab ${JSON.stringify(name)} is not alive. Reopen it.`); + if (!tab || tab.state === "dead") { + const killed = killedTabs.get(name); + throw new ToolError( + killed + ? `Tab ${JSON.stringify(name)} was killed: ${killed}. Reopen it.` + : `Tab ${JSON.stringify(name)} is not alive. Open it first with action:"open".`, + ); + } if (tab.pending.size > 0) throw new ToolError(`Tab ${JSON.stringify(name)} is busy`); const id = Snowflake.next(); const { promise, resolve, reject } = Promise.withResolvers(); @@ -712,6 +724,9 @@ async function recycleTimedOutWorkerTab(tab: WorkerTabSession, timeoutMs: number safeDir: getPuppeteerDir(), targetId: tab.targetId, dialogs: tab.dialogPolicy, + // Unblock a wedged page (open JS dialog, hung navigation) before adopting it — + // otherwise init stalls, times out, and the tab gets force-killed. + recover: true, }; let worker = await spawnTabWorker(); try { @@ -743,6 +758,7 @@ async function recycleTimedOutWorkerTab(tab: WorkerTabSession, timeoutMs: number async function forceKillTab(name: string, reason: string): Promise { const tab = tabs.get(name); if (!tab) return; + killedTabs.set(name, reason); tab.state = "dead"; const error = postmortem.markExpectedCleanupError(new ToolError(reason)); for (const pending of tab.pending.values()) pending.reject(error); diff --git a/packages/coding-agent/src/tools/browser/tab-worker.ts b/packages/coding-agent/src/tools/browser/tab-worker.ts index 973d624a2..b51734c1f 100644 --- a/packages/coding-agent/src/tools/browser/tab-worker.ts +++ b/packages/coding-agent/src/tools/browser/tab-worker.ts @@ -6,6 +6,7 @@ import { postmortem, Snowflake, untilAborted } from "@oh-my-pi/pi-utils"; import type { HTMLElement } from "linkedom"; import type { Browser, + CDPSession, Dialog, ElementHandle, ElementScreenshotOptions, @@ -35,7 +36,13 @@ import { loadPuppeteerInWorker, } from "./launch"; import { extractReadableFromHtml, type ReadableFormat } from "./readable"; -import { markHandled, type WaitPredicateOptions, waitForBrowserRun } from "./run-cancellation"; +import { + CELL_BUDGET_SLACK_MS, + markHandled, + resolvePredicateTimeout, + type WaitPredicateOptions, + waitForBrowserRun, +} from "./run-cancellation"; import { cloneSafe, RunOutput } from "./run-output"; import type { Observation, @@ -105,6 +112,11 @@ const PLAYWRIGHT_ONLY_SELECTOR_RE = type DialogPolicy = "accept" | "dismiss"; type DragTarget = string | { readonly x: number; readonly y: number }; type ActionabilityResult = { ok: true; x: number; y: number } | { ok: false; reason: string }; +/** Last JS dialog seen on the page; kept for timeout attribution until handled or navigation. */ +interface OpenDialogInfo { + type: string; + message: string; +} /** * Per-op fail-fast ceilings for `tab.*` helpers. All are kept strictly under the cell @@ -126,7 +138,7 @@ type ActionabilityResult = { ok: true; x: number; y: number } | { ok: false; rea const QUICK_OP_TIMEOUT_MS = 20_000; const ACTION_OP_TIMEOUT_MS = 8_000; /** Headroom subtracted from the cell budget so a per-op deadline fires before it. */ -const OP_DEADLINE_SLACK_MS = 1_000; +const OP_DEADLINE_SLACK_MS = CELL_BUDGET_SLACK_MS; /** * A selector op whose selector has matched nothing for this long fails fast with the * zero-match hint instead of burning the rest of its deadline: a wrong selector or a @@ -595,6 +607,7 @@ export class WorkerCore { #mode?: WorkerInitPayload["mode"]; #dialogPolicy?: DialogPolicy; #dialogHandler?: (dialog: Dialog) => void; + #openDialog?: OpenDialogInfo; constructor(transport: Transport) { this.#transport = transport; @@ -648,6 +661,7 @@ export class WorkerCore { }); if (payload.mode === "headless") { this.#page = await this.#browser.newPage(); + this.#observeDialogs(); await applyStealthPatches(this.#browser, this.#page, { browserSession: null, override: null }); await applyViewport(this.#page, payload.viewport); if (payload.dialogs) this.#applyDialogPolicy(payload.dialogs); @@ -659,7 +673,15 @@ export class WorkerCore { }); } } else { - this.#page = await this.#findAttachedPage(payload.targetId); + const target = await this.#findAttachedTarget(payload.targetId); + // Post-timeout recycle: unblock the target BEFORE adopting the page — an open + // modal dialog or hung navigation can stall `target.page()` / ready info, and a + // stalled init used to time out and force-kill the tab. + if (payload.recover) await this.#recoverAttachedTarget(target); + const page = await target.page(); + if (!page) throw new ToolError(`Target ${payload.targetId} is no longer available on the attached browser`); + this.#page = page; + this.#observeDialogs(); if (payload.dialogs) this.#applyDialogPolicy(payload.dialogs); } this.#targetId = await targetIdForPage(this.#page); @@ -669,17 +691,53 @@ export class WorkerCore { } } - async #findAttachedPage(targetId: string): Promise { + async #findAttachedTarget(targetId: string): Promise { if (!this.#browser) throw new ToolError("Browser is not connected"); for (const target of this.#browser.targets()) { if ((await targetIdForTarget(target).catch(() => "")) !== targetId) continue; - const page = await target.page(); - if (!page) break; - return page; + return target; } throw new ToolError(`Target ${targetId} is no longer available on the attached browser`); } + /** + * Best-effort unblocking of a wedged target during post-timeout recovery: dismiss any + * open JS dialog and stop a pending navigation over a raw CDP session (created on the + * target, not the page, so it works while the page itself is unresponsive). Every step + * tolerates "nothing to do". + */ + async #recoverAttachedTarget(target: Target): Promise { + let session: CDPSession | undefined; + try { + session = await target.createCDPSession(); + await session.send("Page.enable").catch(() => undefined); + await session.send("Page.handleJavaScriptDialog", { accept: false }).catch(() => undefined); + await session.send("Page.stopLoading").catch(() => undefined); + } catch (error) { + this.#log("debug", "Recovery CDP session failed; proceeding with attach", { + error: error instanceof Error ? error.message : String(error), + }); + } finally { + await session?.detach().catch(() => undefined); + } + } + + /** + * Record JS dialogs for timeout attribution without handling them (semantics of an + * unset `dialogs` policy are unchanged — the page stays blocked until user code or + * the policy handler acts). Cleared when the policy handler settles the dialog or a + * main-frame navigation proves the modal is gone. + */ + #observeDialogs(): void { + const page = this.#requirePage(); + page.on("dialog", dialog => { + this.#openDialog = { type: dialog.type(), message: dialog.message() }; + }); + page.on("framenavigated", frame => { + if (frame === page.mainFrame()) this.#openDialog = undefined; + }); + } + async #currentReadyInfo(): Promise { const page = this.#requirePage(); const targetId = this.#targetId ?? (await targetIdForPage(page)); @@ -698,11 +756,15 @@ export class WorkerCore { if (this.#dialogHandler) page.off("dialog", this.#dialogHandler); const handler = (dialog: Dialog): void => { const action = policy === "accept" ? dialog.accept() : dialog.dismiss(); - void action.catch(err => - this.#log("debug", "Dialog auto-handler failed", { - policy, - error: err instanceof Error ? err.message : String(err), - }), + void action.then( + () => { + this.#openDialog = undefined; + }, + err => + this.#log("debug", "Dialog auto-handler failed", { + policy, + error: err instanceof Error ? err.message : String(err), + }), ); }; page.on("dialog", handler); @@ -761,8 +823,20 @@ export class WorkerCore { assert: (cond: unknown, text?: string): void => { if (!cond) throw new ToolError(text ?? "Assertion failed"); }, - wait: (msOrPredicate: number | (() => unknown), opts?: WaitPredicateOptions): Promise => - waitForBrowserRun(msOrPredicate, signal, opts), + // Both wait forms register in the in-flight map so a cell that dies while + // sleeping/polling names the culprit instead of a bare whole-cell timeout. + wait: (msOrPredicate: number | (() => unknown), opts?: WaitPredicateOptions): Promise => { + const label = typeof msOrPredicate === "number" ? `wait(${msOrPredicate}ms)` : "wait(predicate)"; + const resolved = + typeof msOrPredicate === "number" + ? undefined + : { timeout: resolvePredicateTimeout(msg.timeoutMs, opts?.timeout), interval: opts?.interval }; + return markHandled( + this.#runOp(active, label, signal, Number.POSITIVE_INFINITY, sig => + waitForBrowserRun(msOrPredicate, sig, resolved), + ), + ); + }, }); const { promise: cancelRejection, reject: rejectCancel } = Promise.withResolvers(); const onCancel = (): void => { @@ -772,9 +846,13 @@ export class WorkerCore { : new ToolAbortError(undefined, { cause: signal.reason }); if (timeoutSignal.aborted) { const stalled = describeInflight(active.inflight); + const dialog = this.#openDialog; + const dialogNote = dialog + ? `; a ${dialog.type}(${JSON.stringify(dialog.message.slice(0, 80))}) dialog opened during this run and may still block the page — reopen the tab with dialogs:"accept"|"dismiss" or handle page.on('dialog')` + : ""; rejectCancel( new ToolError( - `Browser code execution timed out after ${msg.timeoutMs}ms${stalled ? ` (stalled on ${stalled})` : ""}`, + `Browser code execution timed out after ${msg.timeoutMs}ms${stalled ? ` (stalled on ${stalled})` : ""}${dialogNote}`, ), ); } else { @@ -986,7 +1064,7 @@ export class WorkerCore { active: ActiveRun, ): TabApi { const page = this.#requirePage(); - const { quickOpMs, actionOpMs } = resolveOpTimeouts(timeoutMs); + const { budgetBound, quickOpMs, actionOpMs } = resolveOpTimeouts(timeoutMs); const waitMs = (explicit?: number): number => resolveWaitTimeout(timeoutMs, explicit); const INF = Number.POSITIVE_INFINITY; const op = ( @@ -1004,10 +1082,24 @@ export class WorkerCore { goto: (url, opts) => op(`tab.goto(${JSON.stringify(url)})`, INF, async sig => { this.#clearElementCache(); - // Default to "load" because dev servers with HMR/WS never reach networkidle. - await untilAborted(sig, () => - page.goto(url, { waitUntil: opts?.waitUntil ?? "load", timeout: timeoutMs }), - ); + try { + // Default to "load" because dev servers with HMR/WS never reach networkidle. + // budgetBound (not the full cell) so a hung navigation fails named and + // catchable inside the run instead of dying with the whole cell. + await untilAborted(sig, () => + page.goto(url, { waitUntil: opts?.waitUntil ?? "load", timeout: budgetBound }), + ); + } catch (err) { + if (err instanceof Error && err.name === "TimeoutError") { + // Abandon the hung navigation NOW — a still-pending load stalls every + // later op on this page and cascades into more opaque timeouts. + await this.#stopLoading(); + throw new ToolError( + `tab.goto(${JSON.stringify(url)}) timed out after ${budgetBound}ms; pending navigation stopped — retry with a longer tool timeout or waitUntil:"domcontentloaded"`, + ); + } + throw err; + } }), observe: opts => op("tab.observe()", quickOpMs, sig => this.#collectObservation({ ...opts, signal: sig })), ariaSnapshot: (selector, opts) => @@ -1554,6 +1646,22 @@ export class WorkerCore { for (const handle of handles) void handle.dispose().catch(() => undefined); } + /** Best-effort `Page.stopLoading` so an abandoned navigation cannot stall later ops. */ + async #stopLoading(): Promise { + try { + const session = await this.#requirePage().createCDPSession(); + try { + await session.send("Page.stopLoading"); + } finally { + await session.detach().catch(() => undefined); + } + } catch (error) { + this.#log("debug", "Page.stopLoading failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + async #close(): Promise { this.#unsub(); this.#clearElementCache(); diff --git a/packages/coding-agent/test/tools/browser-tab-timeouts.test.ts b/packages/coding-agent/test/tools/browser-tab-timeouts.test.ts index 57468b411..ed66b8fd9 100644 --- a/packages/coding-agent/test/tools/browser-tab-timeouts.test.ts +++ b/packages/coding-agent/test/tools/browser-tab-timeouts.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "bun:test"; +import { resolvePredicateTimeout } from "@oh-my-pi/pi-coding-agent/tools/browser/run-cancellation"; import { normalizeSelector, resolveOpTimeouts, @@ -77,6 +78,31 @@ describe("browser wait-helper timeout resolution", () => { }); }); +describe("browser wait(predicate) deadline resolution", () => { + it("keeps the default deadline strictly under the cell budget so the named error wins", () => { + // Default cell (30s): the old 30s predicate default tied the cell timer and lost the + // race, surfacing the opaque whole-cell timeout instead of the named wait error. + for (const cell of [5_000, 30_000, 120_000]) { + expect(resolvePredicateTimeout(cell)).toBeLessThan(cell); + } + expect(resolvePredicateTimeout(120_000)).toBe(30_000); + expect(resolvePredicateTimeout(5_000)).toBe(4_000); + }); + + it("honors an explicit deadline but clamps it under the cell budget", () => { + expect(resolvePredicateTimeout(30_000, 5_000)).toBe(5_000); + expect(resolvePredicateTimeout(30_000, 90_000)).toBe(29_000); + expect(resolvePredicateTimeout(120_000, 90_000)).toBe(90_000); + }); + + it("maps disable sentinels to the largest bounded deadline and garbage to the default", () => { + expect(resolvePredicateTimeout(30_000, 0)).toBe(29_000); + expect(resolvePredicateTimeout(30_000, Number.POSITIVE_INFINITY)).toBe(29_000); + expect(resolvePredicateTimeout(30_000, -5)).toBe(29_000); + expect(resolvePredicateTimeout(30_000, Number.NaN)).toBe(29_000); + }); +}); + describe("browser selector guard", () => { it("rejects Playwright-only selector engines with an actionable message", () => { expect(() => normalizeSelector('button:has-text("Allow all")')).toThrow(/Playwright-only/); diff --git a/patches/puppeteer-core@25.3.0.patch b/patches/puppeteer-core@25.3.0.patch index bc93ba914..b2ececd9f 100644 --- a/patches/puppeteer-core@25.3.0.patch +++ b/patches/puppeteer-core@25.3.0.patch @@ -1,16 +1,24 @@ +diff --git a/node_modules/puppeteer-core/.bun-tag-2e714b457f0bd8e8 b/.bun-tag-2e714b457f0bd8e8 +new file mode 100644 +index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/puppeteer-core/.bun-tag-a797aeb3ca2bd69f b/.bun-tag-a797aeb3ca2bd69f new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/puppeteer/api/ElementHandle.js b/lib/puppeteer/api/ElementHandle.js -index 81454fe2af73518f7e76f04cbe3a7f170d735fa5..6f9eedb9fc1344901705bc83cb3cf415e6271e28 100644 +index 81454fe2af73518f7e76f04cbe3a7f170d735fa5..ee6b042c87861f00d7fcbb6e3eb6e87af232ebbd 100644 --- a/lib/puppeteer/api/ElementHandle.js +++ b/lib/puppeteer/api/ElementHandle.js -@@ -102,6 +102,31 @@ import { throwIfDisposed } from '../util/decorators.js'; +@@ -102,6 +102,36 @@ import { throwIfDisposed } from '../util/decorators.js'; import { _isElementHandle } from './ElementHandleSymbol.js'; import { JSHandle } from './JSHandle.js'; import { NodeLocator } from './locators/locators.js'; +const MAIN_WORLD_DIRECTIVE = /^\s*(?:(?:\/\/!world=main(?=$|\s))|(?:\/\*!world=main\s*\*\/))/; +const shouldEvaluateInMainWorld = (pageFunction) => { ++ if (pageFunction instanceof String) { ++ // withSourcePuppeteerURLIfNone tags string page functions via Object.assign, ++ // which boxes the primitive; unbox so the directive check still sees a string. ++ pageFunction = pageFunction.toString(); ++ } + if (typeof pageFunction !== 'function' && typeof pageFunction !== 'string') { + return false; + } @@ -33,11 +41,11 @@ index 81454fe2af73518f7e76f04cbe3a7f170d735fa5..6f9eedb9fc1344901705bc83cb3cf415 + const arrowIndex = source.indexOf('=>'); + const bodyStart = source.indexOf('{', arrowIndex >= 0 ? arrowIndex : 0); + return bodyStart >= 0 && MAIN_WORLD_DIRECTIVE.test(source.slice(bodyStart + 1)); -+}; ++} /** * A given method will have it's `this` replaced with an isolated version of * `this` when decorated with this decorator. -@@ -299,6 +324,7 @@ let ElementHandle = (() => { +@@ -299,6 +329,7 @@ let ElementHandle = (() => { * trying to adopt it multiple times */ isolatedHandle = __runInitializers(this, _instanceExtraInitializers); @@ -45,7 +53,7 @@ index 81454fe2af73518f7e76f04cbe3a7f170d735fa5..6f9eedb9fc1344901705bc83cb3cf415 /** * @internal */ -@@ -335,19 +361,37 @@ let ElementHandle = (() => { +@@ -335,19 +366,37 @@ let ElementHandle = (() => { async getProperties() { return await this.handle.getProperties(); } @@ -85,7 +93,7 @@ index 81454fe2af73518f7e76f04cbe3a7f170d735fa5..6f9eedb9fc1344901705bc83cb3cf415 } /** * @internal -@@ -371,7 +415,7 @@ let ElementHandle = (() => { +@@ -371,7 +420,7 @@ let ElementHandle = (() => { * @internal */ async dispose() { @@ -94,7 +102,7 @@ index 81454fe2af73518f7e76f04cbe3a7f170d735fa5..6f9eedb9fc1344901705bc83cb3cf415 } /** * @internal -@@ -555,15 +599,27 @@ let ElementHandle = (() => { +@@ -555,15 +604,27 @@ let ElementHandle = (() => { async $$eval(selector, pageFunction, ...args) { const env_2 = { stack: [], error: void 0, hasError: false }; try { @@ -127,15 +135,20 @@ index 81454fe2af73518f7e76f04cbe3a7f170d735fa5..6f9eedb9fc1344901705bc83cb3cf415 ]); return result; diff --git a/lib/puppeteer/api/Frame.js b/lib/puppeteer/api/Frame.js -index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4265ca15f 100644 +index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..62c3b7ca8ec987cc62ceacec69acf6545bb59b86 100644 --- a/lib/puppeteer/api/Frame.js +++ b/lib/puppeteer/api/Frame.js -@@ -119,6 +119,31 @@ export var FrameEvent; +@@ -119,6 +119,36 @@ export var FrameEvent; export const throwIfDetached = throwIfDisposed(frame => { return `Attempted to use detached Frame '${frame._id}'.`; }); +const MAIN_WORLD_DIRECTIVE = /^\s*(?:(?:\/\/!world=main(?=$|\s))|(?:\/\*!world=main\s*\*\/))/; +const shouldEvaluateInMainWorld = (pageFunction) => { ++ if (pageFunction instanceof String) { ++ // withSourcePuppeteerURLIfNone tags string page functions via Object.assign, ++ // which boxes the primitive; unbox so the directive check still sees a string. ++ pageFunction = pageFunction.toString(); ++ } + if (typeof pageFunction !== 'function' && typeof pageFunction !== 'string') { + return false; + } @@ -158,11 +171,11 @@ index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4 + const arrowIndex = source.indexOf('=>'); + const bodyStart = source.indexOf('{', arrowIndex >= 0 ? arrowIndex : 0); + return bodyStart >= 0 && MAIN_WORLD_DIRECTIVE.test(source.slice(bodyStart + 1)); -+}; ++} /** * Represents a DOM frame. * -@@ -277,12 +302,21 @@ let Frame = (() => { +@@ -277,12 +307,21 @@ let Frame = (() => { super(); } #_document; @@ -186,7 +199,7 @@ index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4 return document; }); } -@@ -295,6 +329,7 @@ let Frame = (() => { +@@ -295,6 +334,7 @@ let Frame = (() => { */ clearDocumentHandle() { this.#_document = undefined; @@ -194,7 +207,7 @@ index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4 } /** * @returns The frame element associated with this frame (if any). -@@ -345,8 +380,9 @@ let Frame = (() => { +@@ -345,8 +385,9 @@ let Frame = (() => { * See {@link Page.evaluateHandle} for details. */ async evaluateHandle(pageFunction, ...args) { @@ -205,7 +218,7 @@ index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4 } /** * Behaves identically to {@link Page.evaluate} except it's run within -@@ -355,8 +391,9 @@ let Frame = (() => { +@@ -355,8 +396,9 @@ let Frame = (() => { * See {@link Page.evaluate} for details. */ async evaluate(pageFunction, ...args) { @@ -216,7 +229,7 @@ index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4 } /** * @internal -@@ -458,9 +495,10 @@ let Frame = (() => { +@@ -458,9 +500,10 @@ let Frame = (() => { * @returns A promise to the result of the function. */ async $eval(selector, pageFunction, ...args) { @@ -228,7 +241,7 @@ index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4 return await document.$eval(selector, pageFunction, ...args); } /** -@@ -498,9 +536,10 @@ let Frame = (() => { +@@ -498,9 +541,10 @@ let Frame = (() => { * @returns A promise to the result of the function. */ async $$eval(selector, pageFunction, ...args) { @@ -240,7 +253,7 @@ index 8698d2a5a976b277344fe42e90ffe985bdf0fbd7..01614143cba96055d0f0c22a1eb903d4 return await document.$$eval(selector, pageFunction, ...args); } /** -@@ -577,7 +616,8 @@ let Frame = (() => { +@@ -577,7 +621,8 @@ let Frame = (() => { * @returns the promise which resolve when the `pageFunction` returns a truthy value. */ async waitForFunction(pageFunction, options = {}, ...args) { From 900ffef0624d769d773a0544fc8b2e9ca8bdd23c Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 19:49:12 +0200 Subject: [PATCH 37/80] feat(settings): changed default text verbosity from high to medium - Codex request transformer fallback now defaults to medium as well. --- .../ai/src/providers/openai-codex/request-transformer.ts | 2 +- packages/coding-agent/src/config/settings-schema.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ai/src/providers/openai-codex/request-transformer.ts b/packages/ai/src/providers/openai-codex/request-transformer.ts index 9131cde81..d21e816b4 100644 --- a/packages/ai/src/providers/openai-codex/request-transformer.ts +++ b/packages/ai/src/providers/openai-codex/request-transformer.ts @@ -425,7 +425,7 @@ export async function transformRequestBody( body.text = { ...body.text, - verbosity: options.textVerbosity || "high", + verbosity: options.textVerbosity || "medium", }; const include = Array.isArray(options.include) ? [...options.include] : []; diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index e07b14045..e702bcc20 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -1239,7 +1239,7 @@ export const SETTINGS_SCHEMA = { textVerbosity: { type: "enum", values: ["low", "medium", "high"] as const, - default: "high", + default: "medium", ui: { tab: "model", group: "Sampling", @@ -1247,8 +1247,8 @@ export const SETTINGS_SCHEMA = { description: "OpenAI Responses and Codex response verbosity (low, medium, or high)", options: [ { value: "low", label: "Low", description: "Prefer concise responses" }, - { value: "medium", label: "Medium", description: "Balance brevity and detail" }, - { value: "high", label: "High", description: "Prefer detailed responses (default)" }, + { value: "medium", label: "Medium", description: "Balance brevity and detail (default)" }, + { value: "high", label: "High", description: "Prefer detailed responses" }, ], }, }, From 8c8afaf47790d872c8aff912cacc1ba7c50ac190 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 12 Jul 2026 20:04:01 +0200 Subject: [PATCH 38/80] fix(browser): made tab evaluation use the main world --- packages/coding-agent/CHANGELOG.md | 4 +- .../coding-agent/src/prompts/tools/browser.md | 4 +- .../src/tools/browser/tab-worker.ts | 14 ++++++- .../test/tools/browser-tab-evaluate.test.ts | 38 +++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 packages/coding-agent/test/tools/browser-tab-evaluate.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 0df0c3212..fb1584975 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -11,14 +11,14 @@ - Improved tab recovery after timeouts by automatically clearing pending navigation and JS dialogs - Made `tab.goto` navigation failures catchable with a named error instead of triggering a whole-cell timeout -- Added support for `//!world=main` in `tab.evaluate` to access page-level globals +- Made `tab.evaluate` run in the page's main JavaScript world so page-defined globals are available without a directive - Enhanced cell timeout messages to include identification of stalled operations and blocking JS dialogs - Browser `run` on a tab the supervisor force-killed now reports the kill reason instead of a bare "not alive" - Refined agent workflow to prioritize smoke testing and reduce mandatory upfront test generation ### Fixed -- Fixed the `//!world=main` evaluate directive being silently ignored for string expressions passed through `page.evaluate`/`tab.evaluate`: puppeteer's source-URL tagging boxed the string into a `String` object before the world check +- Fixed the `//!world=main` directive being silently ignored for string expressions passed to raw Puppeteer evaluation APIs - Fixed tab reuse issues where hung navigation or unhandled modals would cause initialization to stall and trigger a force-kill - Improved search reliability for Perplexity provider by forcing retrieval for all queries - Fixed JS eval cells losing top-level `function` and `var` declarations across cells when the defining cell contained top-level `await` — the async wrapper scoped them to the cell's IIFE instead of publishing them to the worker global diff --git a/packages/coding-agent/src/prompts/tools/browser.md b/packages/coding-agent/src/prompts/tools/browser.md index 0e96346c9..27121f2b9 100644 --- a/packages/coding-agent/src/prompts/tools/browser.md +++ b/packages/coding-agent/src/prompts/tools/browser.md @@ -27,10 +27,10 @@ Drives real Chromium tab; full puppeteer access via JS. - `tab.waitForUrl(pattern, { timeout? })` — substring or `RegExp` (matches SPA pushState nav); returns matched URL. - `tab.waitForResponse(pattern, { timeout? })` — substring, `RegExp`, or `(response) => boolean`; returns puppeteer `HTTPResponse` (`.text()`/`.json()`/`.status()`/`.headers()`). - `tab.waitForNavigation({ waitUntil?, timeout? })` — resolves on the next navigation. Start it BEFORE the click/submit that triggers it; after `tab.goto` (which already waits) use `tab.waitForUrl`/`tab.waitForSelector` instead. - - `tab.evaluate(fn, …args)` — `page.evaluate` for ad-hoc DOM reads. Runs in an ISOLATED world: DOM, `document.title`, `document.fonts` are shared, but the page's OWN JS globals (`window.myFlag`, app state, framework internals) are NOT visible and reads return `undefined`. To touch page globals, start the function body with the main-world directive: `tab.evaluate(() => { //!world=main` `return window.__done; })` (or `/*!world=main*/`; also honored by string expressions and `page.waitForFunction`). + - `tab.evaluate(fn, …args)` — run ad-hoc code in the page's MAIN world. DOM and page-defined globals (`window.myFlag`) are visible; mutations affect the page. - `tab.screenshot({ selector?, fullPage?, save?, silent? })` — capture + attach for viewing (`silent: true` skips). Pass `save` only when a later step needs the file. - `tab.extract(format = "markdown")` — readable page content (`"markdown"` | `"text"`); throws when nothing readable. -- Selectors: CSS + puppeteer handlers `aria/Sign in`, `text/Continue`, `xpath/…`, `pierce/…`; also Playwright-style `p-aria/…`, `p-text/…`. Playwright-only engines/pseudos (`:has-text()`, `:visible`, …) are rejected — use `text/…` or `aria/…`. A stalled action/wait fails fast with a named `tab.` error carrying a match-count diagnosis, never the whole-cell timeout; a selector matching nothing fails in ~2s (pass an explicit `{ timeout }` to `waitFor`/`waitForSelector` to wait out slow-appearing elements). A whole-cell timeout names the stalled op (including `wait(...)`) and any unhandled dialog blocking the page. +- Selectors: CSS + puppeteer handlers `aria/Sign in`, `text/Continue`, `xpath/…`, `pierce/…`; also Playwright-style `p-aria/…`, `p-text/…`. Playwright-only engines/pseudos (`:has-text()`, `:visible`, …) are rejected — use `text/…` or `aria/…`. A stalled action/wait fails fast with a named `tab.` error carrying a match-count diagnosis, never the whole-cell timeout; a selector matching nothing fails in ~2s (pass an explicit `{ timeout }` to `waitFor`/`waitForSelector` to wait out slow-appearing elements). A whole-cell timeout names the stalled op (including `wait(…)`) and any unhandled dialog blocking the page. diff --git a/packages/coding-agent/src/tools/browser/tab-worker.ts b/packages/coding-agent/src/tools/browser/tab-worker.ts index b51734c1f..55bb31eb2 100644 --- a/packages/coding-agent/src/tools/browser/tab-worker.ts +++ b/packages/coding-agent/src/tools/browser/tab-worker.ts @@ -57,6 +57,13 @@ import type { WorkerInitPayload, } from "./tab-protocol"; +declare module "puppeteer-core" { + interface Frame { + /** Puppeteer's main JavaScript realm, retained by our pinned runtime patch. */ + mainRealm(): Realm; + } +} + declare global { interface Element extends HTMLElement {} function getComputedStyle(element: Element): Record; @@ -1257,8 +1264,11 @@ export class WorkerCore { op("tab.evaluate()", INF, sig => untilAborted(sig, () => typeof fn === "string" - ? page.evaluate(fn) - : page.evaluate(fn as (...a: unknown[]) => unknown, ...args), + ? page.mainFrame().mainRealm().evaluate(fn) + : page + .mainFrame() + .mainRealm() + .evaluate(fn as (...a: unknown[]) => unknown, ...args), ), ) as never, scrollIntoView: selector => diff --git a/packages/coding-agent/test/tools/browser-tab-evaluate.test.ts b/packages/coding-agent/test/tools/browser-tab-evaluate.test.ts new file mode 100644 index 000000000..63f8daf39 --- /dev/null +++ b/packages/coding-agent/test/tools/browser-tab-evaluate.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "bun:test"; +import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import type { ToolSession } from "@oh-my-pi/pi-coding-agent/sdk"; +import { BrowserTool } from "@oh-my-pi/pi-coding-agent/tools/browser"; + +function makeSession(): ToolSession { + return { + cwd: "/tmp/test", + hasUI: false, + getSessionFile: () => null, + getSessionSpawns: () => "*", + settings: Settings.isolated({ "browser.headless": true }), + }; +} + +describe("browser tab evaluation", () => { + it("runs tab.evaluate in the page's main JavaScript world", async () => { + const tool = new BrowserTool(makeSession()); + const name = `main-world-${process.pid}`; + + try { + await tool.execute("open", { + action: "open", + name, + url: "data:text/html,", + }); + const result = await tool.execute("run", { + action: "run", + name, + code: "return await tab.evaluate(() => globalThis.__ompMainWorld);", + }); + + expect(result.content).toEqual([{ type: "text", text: "42" }]); + } finally { + await tool.execute("close", { action: "close", name, kill: true }); + } + }); +}); From d4ffb4b64cecffd1cdf66f5d99a5b4d3ea4794f8 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 00:06:03 +0200 Subject: [PATCH 39/80] feat(coding-agent/tools): updated status event rendering to show tail window - Refactor status event display to show the most recent events as the live edge. - Replace the bottom-trailing "more" indicator with a leading "earlier" marker when events exceed the display limit. - Dynamically adjust the expanded display size based on the available preview window rows. --- packages/coding-agent/CHANGELOG.md | 4 +++ .../coding-agent/src/tools/eval-render.ts | 30 ++++++++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index fb1584975..69aa9acf6 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- Updated status event log to prioritize the most recent entries in the display window + ## [16.4.8] - 2026-07-12 ### Added diff --git a/packages/coding-agent/src/tools/eval-render.ts b/packages/coding-agent/src/tools/eval-render.ts index 5e7f73090..b42468c11 100644 --- a/packages/coding-agent/src/tools/eval-render.ts +++ b/packages/coding-agent/src/tools/eval-render.ts @@ -414,37 +414,39 @@ function formatStatusEventExpanded(event: EvalStatusEvent, theme: Theme): string return lines; } -/** Render status events as tree lines. */ +/** + * Render status events as tree lines. Shows a tail window (newest events are + * the live edge for `log()` progress loops) behind an "… N earlier" marker, + * matching the code/output tail-window convention. Collapsed keeps a small + * fixed window; expanded widens to the viewport-sized preview window. + */ function renderStatusEvents(events: EvalStatusEvent[], theme: Theme, expanded: boolean): string[] { if (events.length === 0) return []; - const maxCollapsed = 3; - const maxExpanded = 10; - const displayCount = expanded ? Math.min(events.length, maxExpanded) : Math.min(events.length, maxCollapsed); + const max = expanded ? Math.max(10, previewWindowRows()) : 3; + const hidden = Math.max(0, events.length - max); + const visible = hidden > 0 ? events.slice(hidden) : events; const lines: string[] = []; - for (let i = 0; i < displayCount; i++) { - const isLast = i === displayCount - 1 && (expanded || events.length <= maxCollapsed); + if (hidden > 0) { + lines.push(`${theme.fg("dim", theme.tree.branch)} ${theme.fg("dim", `… ${hidden} earlier`)}`); + } + for (let i = 0; i < visible.length; i++) { + const isLast = i === visible.length - 1; const branch = isLast ? theme.tree.last : theme.tree.branch; if (expanded) { - const eventLines = formatStatusEventExpanded(events[i], theme); + const eventLines = formatStatusEventExpanded(visible[i], theme); lines.push(`${theme.fg("dim", branch)} ${eventLines[0]}`); const continueBranch = isLast ? " " : `${theme.tree.vertical} `; for (let j = 1; j < eventLines.length; j++) { lines.push(`${theme.fg("dim", continueBranch)}${eventLines[j]}`); } } else { - lines.push(`${theme.fg("dim", branch)} ${formatStatusEvent(events[i], theme)}`); + lines.push(`${theme.fg("dim", branch)} ${formatStatusEvent(visible[i], theme)}`); } } - if (!expanded && events.length > maxCollapsed) { - lines.push(`${theme.fg("dim", theme.tree.last)} ${theme.fg("dim", `… ${events.length - maxCollapsed} more`)}`); - } else if (expanded && events.length > maxExpanded) { - lines.push(`${theme.fg("dim", theme.tree.last)} ${theme.fg("dim", `… ${events.length - maxExpanded} more`)}`); - } - return lines; } From 8dbc43b6e869e7b4e3f18f5615135cd7fc46aa10 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 00:06:36 +0200 Subject: [PATCH 40/80] feat(coding-agent-modes): implemented floating model selection system - Introduced ModelPickerComponent to enable temporary, floating model selection overlays. - Centralized role management logic using a new resolveRoleAssignments function in the browser. - Streamlined model hub by removing legacy pick-mode logic and defaulting to fullscreen views. - Synchronized model picker state with the registry to ensure accurate offline refreshes. --- packages/coding-agent/CHANGELOG.md | 4 + .../src/modes/components/index.ts | 1 + .../src/modes/components/model-browser.ts | 84 ++++++++- .../src/modes/components/model-hub.ts | 139 +++----------- .../src/modes/components/model-picker.ts | 156 ++++++++++++++++ .../modes/controllers/selector-controller.ts | 86 ++++++--- packages/coding-agent/test/model-hub.test.ts | 118 ++---------- .../coding-agent/test/model-picker.test.ts | 172 ++++++++++++++++++ 8 files changed, 516 insertions(+), 244 deletions(-) create mode 100644 packages/coding-agent/src/modes/components/model-picker.ts create mode 100644 packages/coding-agent/test/model-picker.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 69aa9acf6..abeb200e5 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added a compact session-only model picker (Alt+P) for quick model switching without changing roles + ### Changed - Updated status event log to prioritize the most recent entries in the display window diff --git a/packages/coding-agent/src/modes/components/index.ts b/packages/coding-agent/src/modes/components/index.ts index 769bb9ddd..e1fe04035 100644 --- a/packages/coding-agent/src/modes/components/index.ts +++ b/packages/coding-agent/src/modes/components/index.ts @@ -20,6 +20,7 @@ export * from "./login-dialog"; export * from "./logout-account-selector"; export * from "./model-browser"; export * from "./model-hub"; +export * from "./model-picker"; export * from "./oauth-selector"; export * from "./queue-mode-selector"; export * from "./read-tool-group"; diff --git a/packages/coding-agent/src/modes/components/model-browser.ts b/packages/coding-agent/src/modes/components/model-browser.ts index 1ae63ee24..84ae9f3aa 100644 --- a/packages/coding-agent/src/modes/components/model-browser.ts +++ b/packages/coding-agent/src/modes/components/model-browser.ts @@ -22,10 +22,11 @@ import { visibleWidth, } from "@oh-my-pi/pi-tui"; import { formatNumber } from "@oh-my-pi/pi-utils"; -import { getRoleInfo, MODEL_ROLE_IDS } from "../../config/model-roles"; +import { getModelMatchPreferences, resolveModelRoleValue } from "../../config/model-resolver"; +import { getKnownRoleIds, getRoleInfo, MODEL_ROLE_IDS } from "../../config/model-roles"; import type { Settings } from "../../config/settings"; import type { ModelPerfStats } from "../../session/agent-storage"; -import { AUTO_THINKING, type ConfiguredThinkingLevel } from "../../thinking"; +import { AUTO_THINKING, type ConfiguredThinkingLevel, parseConfiguredThinkingLevel } from "../../thinking"; import { theme } from "../theme/theme"; import { matchesSelectCancel, @@ -54,6 +55,67 @@ export interface RoleAssignment { /** Map of role id to its resolved assignment (absent roles are unresolved). */ export type RoleAssignments = Record; +/** + * Resolve every known role to its display assignment: configured role values + * resolve against `allModels`; unconfigured roles fall back to auto-selection + * over `autoCandidates` (skipped when empty). Shared by the /models hub and + * the alt+p session picker. + */ +export function resolveRoleAssignments( + settings: Settings, + allModels: ReadonlyArray, + autoCandidates: ReadonlyArray, +): RoleAssignments { + const resolvedThinkingLevel = ( + role: string, + resolved: { explicitThinkingLevel: boolean; thinkingLevel?: ConfiguredThinkingLevel }, + ): ConfiguredThinkingLevel => { + if (resolved.explicitThinkingLevel && resolved.thinkingLevel !== undefined) { + return resolved.thinkingLevel; + } + if (role === "default") { + return parseConfiguredThinkingLevel(settings.get("defaultThinkingLevel")) ?? ThinkingLevel.Inherit; + } + return ThinkingLevel.Inherit; + }; + + const roles: RoleAssignments = {}; + const matchPreferences = getModelMatchPreferences(settings); + const knownRoles = getKnownRoleIds(settings); + const configuredRoles = new Set(); + const catalog = [...allModels]; + + for (const role of knownRoles) { + const roleValue = settings.getModelRole(role); + if (!roleValue) continue; + configuredRoles.add(role); + const resolved = resolveModelRoleValue(roleValue, catalog, { settings, matchPreferences }); + if (resolved.model) { + roles[role] = { + model: resolved.model, + thinkingLevel: resolvedThinkingLevel(role, resolved), + autoSelected: false, + }; + } + } + + if (autoCandidates.length > 0) { + const candidates = [...autoCandidates]; + for (const role of knownRoles) { + if (configuredRoles.has(role)) continue; + const resolved = resolveModelRoleValue(`pi/${role}`, candidates, { settings, matchPreferences }); + if (!resolved.model) continue; + roles[role] = { + model: resolved.model, + thinkingLevel: resolvedThinkingLevel(role, resolved), + autoSelected: true, + }; + } + } + + return roles; +} + /** Wrap raw models into browser items. */ export function buildBrowserItems(models: ReadonlyArray): ModelBrowserItem[] { return models.map(model => ({ @@ -256,7 +318,6 @@ export interface ModelBrowserOptions { disableOverContext?: boolean; /** Host-provided empty-state text (e.g. provider discovery status). */ emptyText?: () => string | undefined; - initialQuery?: string; } /** Rendered rows before the list window: search row + blank. */ @@ -295,6 +356,8 @@ export class ModelBrowser implements Component { #windowCount = 0; /** Whether the host pane owns arrow keys; drives cursor strength and the selected-row band. */ #focused = true; + /** `provider/id` of the session's active model; marked in rows and detail. */ + #currentSelector: string | undefined; /** Enter or click-on-selected. */ onActivate?: (item: ModelBrowserItem) => void; @@ -310,9 +373,11 @@ export class ModelBrowser implements Component { this.#currentContextTokens = Number.isFinite(tokens) && tokens > 0 ? Math.floor(tokens) : 0; this.#disableOverContext = options.disableOverContext ?? false; this.#emptyText = options.emptyText; - if (options.initialQuery) { - this.#searchInput.setValue(options.initialQuery); - } + } + + /** Mark `selector` as the session's active model (undefined clears the mark). */ + setCurrentSelector(selector: string | undefined): void { + this.#currentSelector = selector; } /** Replace the scope's base items; the live query re-applies and selection is pinned by selector. */ @@ -641,10 +706,12 @@ export class ModelBrowser implements Component { const prefix = selected && this.#focused ? `${theme.fg("accent", theme.nav.cursor)} ` : " "; const providerPrefix = this.#showProvider ? theme.fg("dim", `${item.provider}/`) : ""; const name = selected ? theme.fg("accent", item.id) : item.id; + const currentMark = + item.selector === this.#currentSelector ? ` ${theme.fg("success", theme.status.enabled)}` : ""; const overLimit = disabled ? ` ${theme.status.disabled} context>${formatNumber(item.model.contextWindow ?? 0).toLowerCase()}` : ""; - let left = `${prefix}${providerPrefix}${name}${overLimit}`; + let left = `${prefix}${providerPrefix}${name}${currentMark}${overLimit}`; // Perf column collapses entirely when no visible row has measurements. const perfCol = @@ -691,6 +758,9 @@ export class ModelBrowser implements Component { } const chips: string[] = []; + if (selected.selector === this.#currentSelector) { + chips.push(theme.fg("success", `${theme.status.enabled} current`)); + } const seen = new Set(); const pushRole = (role: string) => { if (seen.has(role)) return; diff --git a/packages/coding-agent/src/modes/components/model-hub.ts b/packages/coding-agent/src/modes/components/model-hub.ts index 8ddf6ec08..d662f8ed8 100644 --- a/packages/coding-agent/src/modes/components/model-hub.ts +++ b/packages/coding-agent/src/modes/components/model-hub.ts @@ -6,7 +6,8 @@ * {@link ModelBrowser} body. The Roles view manages assignments directly: * pick a role, pick a model, adjust thinking in an inline strip, or clear the * role back to auto-selection. Locked providers forward to the /login flow. - * Fully mouse-navigable (hover, wheel, click). + * Fully mouse-navigable (hover, wheel, click). Session-only switching lives + * in the compact alt+p picker ({@link ./model-picker}). */ import { ThinkingLevel } from "@oh-my-pi/pi-agent-core"; import type { Model } from "@oh-my-pi/pi-ai"; @@ -27,15 +28,9 @@ import { visibleWidth, } from "@oh-my-pi/pi-tui"; import type { ModelRegistry } from "../../config/model-registry"; -import { getModelMatchPreferences, resolveModelRoleValue } from "../../config/model-resolver"; import { getKnownRoleIds, getRoleInfo } from "../../config/model-roles"; import type { Settings } from "../../config/settings"; -import { - AUTO_THINKING, - type ConfiguredThinkingLevel, - getConfiguredThinkingLevelMetadata, - parseConfiguredThinkingLevel, -} from "../../thinking"; +import { AUTO_THINKING, type ConfiguredThinkingLevel, getConfiguredThinkingLevelMetadata } from "../../thinking"; import { theme } from "../theme/theme"; import { matchesSelectCancel, matchesSelectDown, matchesSelectUp } from "../utils/keybinding-matchers"; import { @@ -43,15 +38,13 @@ import { ModelBrowser, type ModelBrowserItem, type RoleAssignments, + resolveRoleAssignments, sortModelItems, thinkingLevelGlyph, } from "./model-browser"; import { bottomBorder, dividerSplit, row, splitBodyWidth, splitRow, topBorderSplit } from "./overlay-box"; import { renderSegmentTrack } from "./segment-track"; -/** `roles` is the full /models hub; `pick` is a one-shot session/embedded picker. */ -export type ModelHubMode = "roles" | "pick"; - /** * A row of the Roles view: a role, a model/wildcard chain-key header, one of a * chain's fallback entries, or the trailing "+ New role…". Fallback rows under @@ -89,8 +82,6 @@ export interface ModelHubCallbacks { onUnassign: (role: string) => void; /** Persist a `retry.fallbackChains` entry — keyed by a role, `provider/model-id`, or `provider/*`; an empty chain clears the key. */ onFallbackChainChange?: (role: string, chain: string[]) => void; - /** Pick-mode activation: session-only switch or embedded pick. */ - onPick?: (model: Model, selector: string) => void; /** Locked provider activation: forward to the /login flow. */ onLoginRequest?: (providerId: string) => void; /** Persist a new quick-switch cycle order (the ctrl+p role cycle). */ @@ -99,14 +90,8 @@ export interface ModelHubCallbacks { } export interface ModelHubOptions { - mode?: ModelHubMode; - /** Session token count; in pick mode, models with smaller context windows are disabled. */ - currentContextTokens?: number; /** Preselect this provider's sidebar entry (e.g. when reopening after /login). */ initialProviderId?: string; - /** Status-row hint shown in pick mode. */ - pickerHint?: string; - initialQuery?: string; } interface SidebarEntry { @@ -158,8 +143,6 @@ const RECENT_LIMIT = 15; const SIDEBAR_MIN_WIDTH = 18; const SIDEBAR_MAX_WIDTH = 26; -const PICK_MODE_HINT = "Session-only switch — role models stay unchanged"; - /** * Providers already auto-refreshed this process. Selecting a provider fetches * its live model list at most once per application lifetime (surviving hub @@ -182,8 +165,6 @@ export class ModelHubComponent implements Component { #registry: ModelRegistry; #scopedModels: ReadonlyArray; #callbacks: ModelHubCallbacks; - #mode: ModelHubMode; - #pickerHint: string; #browser: ModelBrowser; #roles: RoleAssignments = {}; @@ -250,14 +231,9 @@ export class ModelHubComponent implements Component { this.#registry = registry; this.#scopedModels = scopedModels; this.#callbacks = callbacks; - this.#mode = options.mode ?? "roles"; - this.#pickerHint = options.pickerHint ?? PICK_MODE_HINT; this.#browser = new ModelBrowser(settings, { - currentContextTokens: options.currentContextTokens, - disableOverContext: this.#mode === "pick", emptyText: () => this.#emptyStateMessage(), - initialQuery: options.initialQuery, }); this.#browser.onActivate = item => this.#activateItem(item); this.#browser.onCancel = () => this.#callbacks.onCancel(); @@ -275,10 +251,6 @@ export class ModelHubComponent implements Component { this.#setActiveEntry("all"); } - if (this.#mode === "pick") { - this.#focus = "list"; - } - // Reconcile with cached discovery state in the background. A --models // scope is registry-independent, so the offline reload would only repeat // the synchronous hydration above. @@ -314,62 +286,10 @@ export class ModelHubComponent implements Component { return getKnownRoleIds(this.#settings).filter(role => !getRoleInfo(role, this.#settings).hidden); } - #getResolvedRoleThinkingLevel( - role: string, - resolved: { explicitThinkingLevel: boolean; thinkingLevel?: ConfiguredThinkingLevel }, - ): ConfiguredThinkingLevel { - if (resolved.explicitThinkingLevel && resolved.thinkingLevel !== undefined) { - return resolved.thinkingLevel; - } - if (role === "default") { - return parseConfiguredThinkingLevel(this.#settings.get("defaultThinkingLevel")) ?? ThinkingLevel.Inherit; - } - return ThinkingLevel.Inherit; - } - /** Resolve every known role: configured values first, auto-selection for the rest. */ #reloadRoles(autoCandidates: ReadonlyArray): void { - const nextRoles: RoleAssignments = {}; - const allModels = this.#scopedModels.length > 0 ? [...autoCandidates] : this.#registry.getAll(); - const matchPreferences = getModelMatchPreferences(this.#settings); - const knownRoles = getKnownRoleIds(this.#settings); - const configuredRoles = new Set(); - - for (const role of knownRoles) { - const roleValue = this.#settings.getModelRole(role); - if (!roleValue) continue; - configuredRoles.add(role); - const resolved = resolveModelRoleValue(roleValue, allModels, { - settings: this.#settings, - matchPreferences, - }); - if (resolved.model) { - nextRoles[role] = { - model: resolved.model, - thinkingLevel: this.#getResolvedRoleThinkingLevel(role, resolved), - autoSelected: false, - }; - } - } - - if (autoCandidates.length > 0) { - const candidates = [...autoCandidates]; - for (const role of knownRoles) { - if (configuredRoles.has(role)) continue; - const resolved = resolveModelRoleValue(`pi/${role}`, candidates, { - settings: this.#settings, - matchPreferences, - }); - if (!resolved.model) continue; - nextRoles[role] = { - model: resolved.model, - thinkingLevel: this.#getResolvedRoleThinkingLevel(role, resolved), - autoSelected: true, - }; - } - } - - this.#roles = nextRoles; + const allModels = this.#scopedModels.length > 0 ? autoCandidates : this.#registry.getAll(); + this.#roles = resolveRoleAssignments(this.#settings, allModels, autoCandidates); } /** Rebuild items, roles, and the sidebar from the registry's in-memory state. */ @@ -475,16 +395,15 @@ export class ModelHubComponent implements Component { // Roles leads the fixed section so downward hops from Recent head into // model scopes instead of being captured by the roles view. - const fixed: SidebarEntry[] = []; - if (this.#mode === "roles") { - fixed.push({ + const fixed: SidebarEntry[] = [ + { id: "roles", kind: "roles", label: "Roles", annotation: `${assignedCount}/${visibleRoles.length}`, - }); - } - fixed.push({ id: "all", kind: "all", label: "All models", annotation: String(availableModels.length) }); + }, + { id: "all", kind: "all", label: "All models", annotation: String(availableModels.length) }, + ]; this.#fixedEntries = fixed; this.#unlockedProviderEntries = [...unlocked] @@ -811,10 +730,6 @@ export class ModelHubComponent implements Component { // ═══════════════════════════════════════════════════════════════════════ #activateItem(item: ModelBrowserItem): void { - if (this.#mode === "pick") { - this.#callbacks.onPick?.(item.model, item.selector); - return; - } if (this.#assigning) { const target = this.#assigning; this.#assigning = null; @@ -917,7 +832,7 @@ export class ModelHubComponent implements Component { const strip = this.#strip; this.#strip = null; this.#chipRanges = []; - if (strip?.kind === "thinking" && strip.returnToRoles && this.#mode === "roles") { + if (strip?.kind === "thinking" && strip.returnToRoles) { this.#setActiveEntry("roles"); this.#focus = "list"; } @@ -1030,14 +945,12 @@ export class ModelHubComponent implements Component { } this.#setFallbackChain(target.role, chain); this.#browser.setQuery(""); - if (this.#mode === "roles") { - this.#setActiveEntry("roles"); - this.#focus = "list"; - const rowIndex = this.#rolesRows.findIndex( - row => row.kind === "fallback" && row.role === target.role && row.selector === selector, - ); - if (rowIndex >= 0) this.#roleIndex = rowIndex; - } + this.#setActiveEntry("roles"); + this.#focus = "list"; + const rowIndex = this.#rolesRows.findIndex( + row => row.kind === "fallback" && row.role === target.role && row.selector === selector, + ); + if (rowIndex >= 0) this.#roleIndex = rowIndex; } /** Persist `role`'s chain through the host callback and rebuild dependent state. */ @@ -1076,10 +989,8 @@ export class ModelHubComponent implements Component { #cancelAssign(): void { this.#assigning = null; this.#browser.setQuery(""); - if (this.#mode === "roles") { - this.#setActiveEntry("roles"); - this.#focus = "list"; - } + this.#setActiveEntry("roles"); + this.#focus = "list"; } // ═══════════════════════════════════════════════════════════════════════ @@ -1655,7 +1566,7 @@ export class ModelHubComponent implements Component { let text: string; switch (entry.kind) { case "recent": - text = this.#mode === "pick" ? this.#pickerHint : `Recently used models${scopedSuffix}`; + text = `Recently used models${scopedSuffix}`; break; case "roles": text = "Model roles — f adds a retry fallback, cleared roles fall back to auto-selection"; @@ -1670,7 +1581,7 @@ export class ModelHubComponent implements Component { } break; default: - text = this.#mode === "pick" ? this.#pickerHint : `All available models${scopedSuffix}`; + text = `All available models${scopedSuffix}`; break; } if (this.#configError && entry.kind !== "provider") { @@ -1895,9 +1806,6 @@ export class ModelHubComponent implements Component { } const arrows = this.#focus === "scope" ? "↑/↓ providers · → models" : "↑/↓ models · ← providers"; const refresh = entry.kind === "provider" ? " · F5 refresh" : ""; - if (this.#mode === "pick") { - return `Enter use for this session · ${arrows} · type to search${refresh} · Esc close`; - } return `Enter assign roles · ${arrows} · type to search${refresh} · Esc close`; } @@ -1992,9 +1900,8 @@ export class ModelHubComponent implements Component { const sidebarLines = this.#renderSidebar(sidebarWidth, contentRows); - const title = this.#mode === "pick" ? "Switch Model" : "Models"; const out: string[] = []; - out.push(topBorderSplit(width, title, sidebarWidth)); + out.push(topBorderSplit(width, "Models", sidebarWidth)); this.#contentRowStart = out.length; for (let i = 0; i < contentRows; i++) { out.push(splitRow(sidebarLines[i] ?? "", bodyLines[i] ?? "", width, sidebarWidth)); diff --git a/packages/coding-agent/src/modes/components/model-picker.ts b/packages/coding-agent/src/modes/components/model-picker.ts new file mode 100644 index 000000000..be284792d --- /dev/null +++ b/packages/coding-agent/src/modes/components/model-picker.ts @@ -0,0 +1,156 @@ +/** + * Compact session-model picker (alt+p / `/switch`): a bottom-anchored + * floating overlay hosting just a {@link ModelBrowser} — no provider sidebar, + * no role management. Picking a model switches the current session only; + * role assignment lives in the fullscreen /models hub ({@link ./model-hub}). + */ +import type { Model } from "@oh-my-pi/pi-ai"; +import type { Component, TUI } from "@oh-my-pi/pi-tui"; +import type { ModelRegistry } from "../../config/model-registry"; +import type { Settings } from "../../config/settings"; +import { theme } from "../theme/theme"; +import { buildBrowserItems, ModelBrowser, resolveRoleAssignments, sortModelItems } from "./model-browser"; +import type { ScopedModelItem } from "./model-hub"; +import { bottomBorder, row, topBorder } from "./overlay-box"; + +export interface ModelPickerCallbacks { + /** A model was chosen for a session-only switch. `selector` is `provider/id`. */ + onPick: (model: Model, selector: string) => void; + /** The picker was dismissed. */ + onCancel: () => void; +} + +export interface ModelPickerOptions { + /** Session token count; models with smaller context windows are disabled. */ + currentContextTokens?: number; + /** `provider/id` of the session's active model; highlighted and preselected. */ + currentSelector?: string; +} + +/** Fixed chrome rows: top border, status row, footer, bottom border. */ +const CHROME_ROWS = 4; +/** Rows the browser renders around its list window (search + blank, blank + two detail rows). */ +const BROWSER_FRAME_ROWS = 5; +/** Minimum rows for the browser list window on short terminals. */ +const MIN_VISIBLE = 5; +/** Fraction of the terminal height the floating overlay occupies. */ +const HEIGHT_FRACTION = 0.4; + +const STATUS_HINT = "Session-only switch — role models stay unchanged"; +const FOOTER_HINT = "↑/↓ models · Enter use for this session · type to search · Esc close"; + +/** + * The alt+p picker component. Hosted as a non-fullscreen bottom-anchored + * overlay (`ui.showOverlay(..., { anchor: "bottom-center" })`); keyboard-only, + * since mouse tracking is reserved for fullscreen overlays. + */ +export class ModelPickerComponent implements Component { + #tui: TUI; + #settings: Settings; + #registry: ModelRegistry; + #scopedModels: ReadonlyArray; + #browser: ModelBrowser; + #configError: string | undefined; + + constructor( + tui: TUI, + settings: Settings, + registry: ModelRegistry, + scopedModels: ReadonlyArray, + callbacks: ModelPickerCallbacks, + options: ModelPickerOptions = {}, + ) { + this.#tui = tui; + this.#settings = settings; + this.#registry = registry; + this.#scopedModels = scopedModels; + + this.#browser = new ModelBrowser(settings, { + currentContextTokens: options.currentContextTokens, + disableOverContext: true, + }); + this.#browser.onActivate = item => callbacks.onPick(item.model, item.selector); + this.#browser.onCancel = () => callbacks.onCancel(); + this.#browser.setCurrentSelector(options.currentSelector); + + // Hydrate synchronously from the current registry snapshot so the first + // Enter after opening acts on cached models instead of being dropped + // while the offline refresh promise is still pending. + this.#syncFromRegistryState(); + if (options.currentSelector) { + this.#browser.selectSelector(options.currentSelector); + } + + // Reconcile with cached discovery state in the background. A --models + // scope is registry-independent, so the offline reload would only repeat + // the synchronous hydration above. + if (this.#scopedModels.length === 0) { + this.#registry + .refresh("offline") + .then(() => this.#syncFromRegistryState()) + .catch(error => { + this.#configError = error instanceof Error ? error.message : String(error); + }) + .finally(() => this.#tui.requestRender()); + } + } + + invalidate(): void {} + + /** Rebuild items and role chips from the registry's in-memory state (selection is pinned by selector). */ + #syncFromRegistryState(): void { + let models: ReadonlyArray; + if (this.#scopedModels.length > 0) { + models = this.#scopedModels.map(scoped => scoped.model); + this.#configError = undefined; + } else { + const loadError = this.#registry.getError(); + this.#configError = loadError ? String(loadError) : undefined; + try { + models = this.#registry.getAvailable(); + } catch (error) { + this.#configError = error instanceof Error ? error.message : String(error); + models = []; + } + } + + const allModels = this.#scopedModels.length > 0 ? models : this.#registry.getAll(); + const roles = resolveRoleAssignments(this.#settings, allModels, models); + const storage = this.#settings.getStorage(); + const mruOrder = storage?.getModelUsageOrder() ?? []; + const items = buildBrowserItems(models); + sortModelItems(items, { roles, mruOrder }); + this.#browser.setRoles(roles); + this.#browser.setMruOrder(mruOrder); + this.#browser.setPerfStats(storage?.getModelPerf() ?? new Map()); + this.#browser.setItems(items); + } + + handleInput(data: string): void { + // Mouse tracking is off outside fullscreen overlays; drop any stray SGR + // reports instead of feeding them to the search input. + if (data.startsWith("\x1b[<")) return; + this.#browser.handleInput(data); + } + + render(width: number): string[] { + const termRows = Math.max(16, this.#tui.terminal?.rows || process.stdout.rows || 40); + const listBudget = Math.floor(termRows * HEIGHT_FRACTION) - CHROME_ROWS - BROWSER_FRAME_ROWS; + this.#browser.setMaxVisible(Math.max(MIN_VISIBLE, listBudget)); + + const inner = Math.max(1, width - 4); + const status = this.#configError + ? theme.fg("error", ` ${this.#configError}`) + : theme.fg("muted", ` ${STATUS_HINT}`); + + const out: string[] = []; + out.push(topBorder(width, "Switch Model")); + out.push(row(status, width)); + for (const line of this.#browser.render(inner)) { + out.push(row(line, width)); + } + out.push(row(theme.fg("dim", FOOTER_HINT), width)); + out.push(bottomBorder(width)); + return out; + } +} diff --git a/packages/coding-agent/src/modes/controllers/selector-controller.ts b/packages/coding-agent/src/modes/controllers/selector-controller.ts index e101bbc5f..e49ad1827 100644 --- a/packages/coding-agent/src/modes/controllers/selector-controller.ts +++ b/packages/coding-agent/src/modes/controllers/selector-controller.ts @@ -67,7 +67,8 @@ import { ExtensionDashboard } from "../components/extensions"; import { HistorySearchComponent } from "../components/history-search"; import { LoginDialogComponent } from "../components/login-dialog"; import { LogoutAccountSelectorComponent } from "../components/logout-account-selector"; -import { ModelHubComponent, type ModelHubMode } from "../components/model-hub"; +import { ModelHubComponent } from "../components/model-hub"; +import { ModelPickerComponent } from "../components/model-picker"; import { OAuthSelectorComponent } from "../components/oauth-selector"; import { PluginSelectorComponent } from "../components/plugin-selector"; import { ResetUsageSelectorComponent } from "../components/reset-usage-selector"; @@ -586,7 +587,65 @@ export class SelectorController { } showModelSelector(options?: { temporaryOnly?: boolean }): void { - this.#showModelHub({ mode: options?.temporaryOnly ? "pick" : "roles" }); + if (options?.temporaryOnly) { + this.#showModelPicker(); + return; + } + this.#showModelHub({}); + } + + /** + * Compact session-only model picker (alt+p / `/switch`): a floating + * bottom-anchored overlay over the transcript — just the model list, no + * provider sidebar, no role management. The current model is highlighted + * and preselected. + */ + #showModelPicker(): void { + const currentContextTokens = this.ctx.session.getContextUsage()?.tokens ?? 0; + const current = this.ctx.session.model; + let overlayHandle: OverlayHandle | undefined; + let closed = false; + const done = () => { + if (closed) return; + closed = true; + overlayHandle?.hide(); + this.focusActiveEditorArea(); + this.ctx.ui.requestRender(); + }; + const picker = new ModelPickerComponent( + this.ctx.ui, + this.ctx.settings, + this.ctx.session.modelRegistry, + this.ctx.session.scopedModels, + { + onPick: async (model, selector) => { + try { + // Session-only: update agent state but don't persist the model to settings. + await this.ctx.session.setModelTemporary(model); + this.ctx.statusLine.invalidate(); + this.ctx.updateEditorBorderColor(); + const roleSelectorHint = this.ctx.keybindings.getKeys("app.model.select")[0] ?? "Alt+M"; + this.ctx.showStatus(`Session-only model: ${selector}. Use ${roleSelectorHint} or /model for roles.`); + done(); + } catch (error) { + this.ctx.showError(error instanceof Error ? error.message : String(error)); + } + }, + onCancel: done, + }, + { + currentContextTokens, + currentSelector: current ? `${current.provider}/${current.id}` : undefined, + }, + ); + overlayHandle = this.ctx.ui.showOverlay(picker, { + anchor: "bottom-center", + width: "100%", + maxHeight: "100%", + margin: 0, + }); + this.ctx.ui.setFocus(picker); + this.ctx.ui.requestRender(); } /** @@ -595,13 +654,13 @@ export class SelectorController { * untouched underneath. `initialProviderId` preselects a provider's sidebar * entry — used when reopening the hub after a /login round-trip. */ - #showModelHub(hubOptions: { mode: ModelHubMode; initialProviderId?: string }): void { + #showModelHub(hubOptions: { initialProviderId?: string }): void { const currentContextTokens = this.ctx.session.getContextUsage()?.tokens ?? 0; let overlayHandle: OverlayHandle | undefined; let hub: ModelHubComponent | undefined; let closed = false; const done = () => { - // Re-entrant guard: cancel paths (Esc, pick, login forward) may race; + // Re-entrant guard: cancel paths (Esc, login forward) may race; // the overlay must hide exactly once. if (closed) return; closed = true; @@ -686,21 +745,6 @@ export class SelectorController { this.ctx.showError(error instanceof Error ? error.message : String(error)); } }, - onPick: async (model, selector) => { - try { - // Session-only: update agent state but don't persist the model to settings. - await this.ctx.session.setModelTemporary(model); - this.ctx.statusLine.invalidate(); - this.ctx.updateEditorBorderColor(); - const roleSelectorHint = this.ctx.keybindings.getKeys("app.model.select")[0] ?? "Alt+M"; - this.ctx.showStatus( - `Session-only model: ${selector ?? model.id}. Use ${roleSelectorHint} or /model for roles.`, - ); - done(); - } catch (error) { - this.ctx.showError(error instanceof Error ? error.message : String(error)); - } - }, onLoginRequest: providerId => { done(); void this.#loginThenReopenModelHub(providerId); @@ -718,8 +762,6 @@ export class SelectorController { onCancel: () => done(), }, { - mode: hubOptions.mode, - currentContextTokens, initialProviderId: hubOptions.initialProviderId, }, ); @@ -738,7 +780,7 @@ export class SelectorController { async #loginThenReopenModelHub(providerId: string): Promise { const succeeded = await this.#handleOAuthLogin(providerId); if (succeeded) { - this.#showModelHub({ mode: "roles", initialProviderId: providerId }); + this.#showModelHub({ initialProviderId: providerId }); } } diff --git a/packages/coding-agent/test/model-hub.test.ts b/packages/coding-agent/test/model-hub.test.ts index 674d87435..553a14178 100644 --- a/packages/coding-agent/test/model-hub.test.ts +++ b/packages/coding-agent/test/model-hub.test.ts @@ -75,7 +75,6 @@ interface HubHarness { hub: ModelHubComponent; onAssign: ReturnType; onUnassign: ReturnType; - onPick: ReturnType; onLoginRequest: ReturnType; onCancel: ReturnType; onFallbackChainChange: Mock<(role: string, chain: string[]) => void>; @@ -98,7 +97,6 @@ function createHub(options: { const ui = { requestRender: vi.fn(), terminal: { rows: 40 } } as unknown as TUI; const onAssign = vi.fn(); const onUnassign = vi.fn(); - const onPick = vi.fn(); const onLoginRequest = vi.fn(); const onCancel = vi.fn(); // Mirror the controller: persist chain edits so the hub's re-read sees them. @@ -119,7 +117,6 @@ function createHub(options: { { onAssign: options.callbacks?.onAssign ?? onAssign, onUnassign: options.callbacks?.onUnassign ?? onUnassign, - onPick: options.callbacks?.onPick ?? onPick, onLoginRequest: options.callbacks?.onLoginRequest ?? onLoginRequest, onCycleOrderChange: options.callbacks?.onCycleOrderChange, onFallbackChainChange: options.callbacks?.onFallbackChainChange ?? onFallbackChainChange, @@ -128,12 +125,13 @@ function createHub(options: { options.hub, ); openHubs.push(hub); - return { hub, onAssign, onUnassign, onPick, onLoginRequest, onCancel, onFallbackChainChange }; + return { hub, onAssign, onUnassign, onLoginRequest, onCancel, onFallbackChainChange }; } const DOWN = "\x1b[B"; const UP = "\x1b[A"; const LEFT = "\x1b[D"; +const ESC = "\x1b"; describe("ModelHub", () => { beforeAll(async () => { @@ -608,86 +606,10 @@ describe("ModelHub", () => { }); }); - describe("pick mode", () => { - test("disables models below the current context size and picks the first enabled one", () => { - const small = makeModel("test", "a-small", 4096); - const large = makeModel("test", "b-large", 128_000); - const { hub, onPick } = createHub({ - models: [small, large], - scoped: true, - hub: { mode: "pick", currentContextTokens: 6000 }, - }); - installTestTheme(); - - const rendered = normalize(hub.render(220)); - expect(rendered).toContain("a-small"); - expect(rendered).toContain("context>4.1k"); - expect(rendered).toContain("Session-only switch"); - - hub.handleInput("\n"); - expect(onPick).toHaveBeenCalledTimes(1); - expect(onPick.mock.calls[0]?.[0]).toBe(large); - }); - - test("uses cached models for Enter while the offline refresh is still pending", () => { - const cached = makeModel("test", "cached-fast"); - const refreshGate = Promise.withResolvers(); - const refresh = vi.fn(() => refreshGate.promise); - const { hub, onPick } = createHub({ - models: [cached], - registry: { refresh }, - hub: { mode: "pick" }, - }); - installTestTheme(); - - hub.handleInput("\n"); - expect(onPick).toHaveBeenCalledTimes(1); - expect(onPick.mock.calls[0]?.[0]).toBe(cached); - expect(refresh).toHaveBeenCalledTimes(1); - refreshGate.resolve(); - }); - - test("focuses list mode initially in pick mode", () => { - const model = makeModel("test", "test-model"); - const { hub } = createHub({ - models: [model], - hub: { mode: "pick" }, - }); - const rendered = normalize(hub.render(220)); - expect(rendered).toContain("↑/↓ models · ← providers"); - }); - - test("focuses scope mode initially in roles mode", () => { - const model = makeModel("test", "test-model"); - const { hub } = createHub({ - models: [model], - hub: { mode: "roles" }, - }); - const rendered = normalize(hub.render(220)); - expect(rendered).toContain("↑/↓ providers · → models"); - }); - - test("keeps the highlighted model when a background refresh reorders the list", async () => { - const modelBb = makeModel("test", "bb-model"); - const modelCc = makeModel("test", "cc-model"); - const modelAa = makeModel("test", "aa-model"); - let available = [modelBb, modelCc]; - const refreshGate = Promise.withResolvers(); - const { hub, onPick } = createHub({ - models: () => available, - registry: { refresh: () => refreshGate.promise }, - hub: { mode: "pick" }, - }); - installTestTheme(); - - hub.handleInput(DOWN); // highlight cc-model - available = [modelAa, modelBb, modelCc]; - refreshGate.resolve(); - await Bun.sleep(0); - - hub.handleInput("\n"); - expect(onPick.mock.calls[0]?.[0]?.id).toBe("cc-model"); - }); + test("focuses the scope pane initially", () => { + const { hub } = createHub({ models: [makeModel("test", "test-model")] }); + const rendered = normalize(hub.render(220)); + expect(rendered).toContain("↑/↓ providers · → models"); }); describe("mouse wheel", () => { @@ -700,12 +622,15 @@ describe("ModelHub", () => { test("wheel pans the model list without moving the selection and clamps at the ends", () => { const models = Array.from({ length: 40 }, (_, i) => makeModel("test", `model-${String(i).padStart(2, "0")}`)); - const { hub, onPick } = createHub({ models, scoped: true, hub: { mode: "pick" } }); + const { hub } = createHub({ models, scoped: true }); const before = normalize(hub.render(220)); // establishes mouse geometry + // Enter opens the role strip for the selected model — its footer + // (` → …`) identifies the selection. hub.handleInput("\n"); - expect(onPick).toHaveBeenCalledTimes(1); - const initiallySelected = onPick.mock.calls[0]?.[0]; + const initialStrip = footerLine(hub.render(220)); + expect(initialStrip).toContain("→"); + hub.handleInput(ESC); // close the strip // Panning reveals rows that were below the fold... for (let i = 0; i < 8; i++) hub.handleInput(WHEEL_DOWN_BODY); @@ -715,10 +640,10 @@ describe("ModelHub", () => { const revealed = [...modelIdsIn(panned)].filter(id => !beforeIds.has(id)); expect(revealed.length).toBeGreaterThan(0); - // ...but never moves the selection: Enter still picks the same model. + // ...but never moves the selection: Enter still opens the same model's strip. hub.handleInput("\n"); - expect(onPick).toHaveBeenCalledTimes(2); - expect(onPick.mock.calls[1]?.[0]).toBe(initiallySelected); + expect(footerLine(hub.render(220))).toBe(initialStrip); + hub.handleInput(ESC); // The window clamps at the bottom instead of wrapping back to the top... for (let i = 0; i < 500; i++) hub.handleInput(WHEEL_DOWN_BODY); @@ -773,14 +698,9 @@ describe("ModelHub", () => { test("search inside a provider scope keeps that provider's model (#4522)", () => { const openrouterGlm = makeModel("openrouter", "z-ai/glm-5.2"); const customGlm = makeModel("custom-provider", "glm-5.2"); - const { hub, onPick } = createHub({ - models: [openrouterGlm, customGlm], - hub: { mode: "pick" }, - }); + const { hub } = createHub({ models: [openrouterGlm, customGlm] }); installTestTheme(); - // Focus scope first to allow scope-hopping - hub.handleInput("\t"); // Scope-hop: All models → custom-provider → openrouter. hub.handleInput(DOWN); hub.handleInput(DOWN); @@ -789,9 +709,9 @@ describe("ModelHub", () => { for (const ch of "glm-5.2") hub.handleInput(ch); hub.handleInput("\n"); - expect(onPick).toHaveBeenCalledTimes(1); - expect(onPick.mock.calls[0]?.[0]?.provider).toBe("openrouter"); - expect(onPick.mock.calls[0]?.[0]?.id).toBe("z-ai/glm-5.2"); + // The role strip opened for the provider-scoped match, not the + // identically named custom-provider model. + expect(footerLine(hub.render(220))).toContain("z-ai/glm-5.2 →"); }); test("search on All models spans every provider", () => { diff --git a/packages/coding-agent/test/model-picker.test.ts b/packages/coding-agent/test/model-picker.test.ts new file mode 100644 index 000000000..20de0b766 --- /dev/null +++ b/packages/coding-agent/test/model-picker.test.ts @@ -0,0 +1,172 @@ +import { beforeAll, describe, expect, type Mock, test, vi } from "bun:test"; +import { stripVTControlCharacters } from "node:util"; +import type { Model } from "@oh-my-pi/pi-ai"; +import { buildModel } from "@oh-my-pi/pi-catalog/build"; +import type { ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry"; +import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import { ModelPickerComponent, type ModelPickerOptions } from "@oh-my-pi/pi-coding-agent/modes/components/model-picker"; +import { getThemeByName, setThemeInstance } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; +import type { TUI } from "@oh-my-pi/pi-tui"; + +function normalize(lines: readonly string[]): string { + return stripVTControlCharacters(lines.join("\n")).replace(/\s+/g, " ").trim(); +} + +function makeModel(provider: string, id: string, contextWindow = 128_000): Model { + return buildModel({ + id, + name: id, + api: "ollama-chat", + provider, + baseUrl: "https://example.com", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow, + maxTokens: 1024, + }); +} + +let testTheme = await getThemeByName("dark"); + +function installTestTheme(): void { + if (!testTheme) { + throw new Error("Failed to load dark theme for ModelPicker tests"); + } + setThemeInstance(testTheme); +} + +interface RegistryOverrides { + refresh?: (mode: string) => Promise; +} + +interface PickerHarness { + picker: ModelPickerComponent; + onPick: Mock<(model: Model, selector: string) => void>; + onCancel: Mock<() => void>; +} + +function createPicker(options: { + models: Model[] | (() => Model[]); + scoped?: boolean; + settings?: Settings; + registry?: RegistryOverrides; + picker?: ModelPickerOptions; +}): PickerHarness { + installTestTheme(); + const modelsFn = typeof options.models === "function" ? options.models : () => options.models as Model[]; + const settings = options.settings ?? Settings.isolated({}); + const registry = { + refresh: options.registry?.refresh ?? (async () => {}), + getError: () => undefined, + getAvailable: modelsFn, + getAll: modelsFn, + } as unknown as ModelRegistry; + const ui = { requestRender: vi.fn(), terminal: { rows: 40 } } as unknown as TUI; + const onPick = vi.fn(); + const onCancel = vi.fn(); + const picker = new ModelPickerComponent( + ui, + settings, + registry, + options.scoped ? modelsFn().map(model => ({ model })) : [], + { onPick, onCancel }, + options.picker ?? {}, + ); + return { picker, onPick, onCancel }; +} + +const DOWN = "\x1b[B"; +const ESC = "\x1b"; + +describe("ModelPicker", () => { + beforeAll(async () => { + testTheme = await getThemeByName("dark"); + if (!testTheme) { + throw new Error("Failed to load dark theme for ModelPicker tests"); + } + }); + + test("disables models below the current context size and picks the first enabled one", () => { + const small = makeModel("test", "a-small", 4096); + const large = makeModel("test", "b-large", 128_000); + const { picker, onPick } = createPicker({ + models: [small, large], + scoped: true, + picker: { currentContextTokens: 6000 }, + }); + + const rendered = normalize(picker.render(220)); + expect(rendered).toContain("a-small"); + expect(rendered).toContain("context>4.1k"); + expect(rendered).toContain("Session-only switch"); + + picker.handleInput("\n"); + expect(onPick).toHaveBeenCalledTimes(1); + expect(onPick.mock.calls[0]?.[0]).toBe(large); + }); + + test("uses cached models for Enter while the offline refresh is still pending", () => { + const cached = makeModel("test", "cached-fast"); + const refreshGate = Promise.withResolvers(); + const refresh = vi.fn(() => refreshGate.promise); + const { picker, onPick } = createPicker({ + models: [cached], + registry: { refresh }, + }); + + picker.handleInput("\n"); + expect(onPick).toHaveBeenCalledTimes(1); + expect(onPick.mock.calls[0]?.[0]).toBe(cached); + expect(refresh).toHaveBeenCalledTimes(1); + refreshGate.resolve(); + }); + + test("keeps the highlighted model when a background refresh reorders the list", async () => { + const modelBb = makeModel("test", "bb-model"); + const modelCc = makeModel("test", "cc-model"); + const modelAa = makeModel("test", "aa-model"); + let available = [modelBb, modelCc]; + const refreshGate = Promise.withResolvers(); + const { picker, onPick } = createPicker({ + models: () => available, + registry: { refresh: () => refreshGate.promise }, + }); + + picker.handleInput(DOWN); // highlight cc-model + available = [modelAa, modelBb, modelCc]; + refreshGate.resolve(); + // Not a tuned delay: one zero-length tick drains the component's + // refresh().then(...) continuation chain deterministically. + await Bun.sleep(0); + picker.handleInput("\n"); + expect(onPick.mock.calls[0]?.[0]?.id).toBe("cc-model"); + }); + + test("highlights and preselects the session's current model", () => { + const models = [makeModel("test", "aa-model"), makeModel("test", "bb-model"), makeModel("test", "cc-model")]; + const { picker, onPick } = createPicker({ + models, + scoped: true, + picker: { currentSelector: "test/bb-model" }, + }); + + // The detail block tags the selected (= current) model. + expect(normalize(picker.render(220))).toContain("current"); + + // Enter without navigation picks the preselected current model. + picker.handleInput("\n"); + expect(onPick.mock.calls[0]?.[0]?.id).toBe("bb-model"); + }); + + test("Esc clears an active query first, then cancels", () => { + const { picker, onCancel } = createPicker({ models: [makeModel("test", "test-model")], scoped: true }); + + picker.handleInput("q"); + picker.handleInput(ESC); + expect(onCancel).not.toHaveBeenCalled(); + + picker.handleInput(ESC); + expect(onCancel).toHaveBeenCalledTimes(1); + }); +}); From d0f90f35ae0f4aba48430b51a7203013dc0c5ff3 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 00:34:27 +0200 Subject: [PATCH 41/80] refactor(coding-agent): removed unreliable web search providers - Removed unreliable Bing and Yahoo HTML-scraping search providers. - Deleted `src/web/search/providers/bing.ts` and `src/web/search/providers/yahoo.ts` implementation files. - Updated `provider.ts`, `types.ts`, and `public.ts` to prune provider registration and configuration. - Adjusted `web-search-public.test.ts` to exclude removed engines from test coverage. --- packages/coding-agent/CHANGELOG.md | 6 + .../coding-agent/src/web/search/provider.ts | 10 - .../src/web/search/providers/bing.ts | 197 ------------------ .../src/web/search/providers/public.ts | 6 +- .../src/web/search/providers/yahoo.ts | 179 ---------------- packages/coding-agent/src/web/search/types.ts | 10 - .../test/tools/web-search-public.test.ts | 2 +- 7 files changed, 9 insertions(+), 401 deletions(-) delete mode 100644 packages/coding-agent/src/web/search/providers/bing.ts delete mode 100644 packages/coding-agent/src/web/search/providers/yahoo.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index abeb200e5..e0b1b1608 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -10,6 +10,10 @@ - Updated status event log to prioritize the most recent entries in the display window +### Removed + +- Removed the unreliable Bing and Yahoo HTML-scraping web search providers + ## [16.4.8] - 2026-07-12 ### Added @@ -17,6 +21,7 @@ ### Changed +- Replaced the Alt+P / `/switch` temporary model selector's fullscreen /models hub with a compact full-width floating overlay anchored above the editor (~40% of the terminal height): just the searchable model list — no provider sidebar or role management — with the session's active model highlighted and preselected - Improved tab recovery after timeouts by automatically clearing pending navigation and JS dialogs - Made `tab.goto` navigation failures catchable with a named error instead of triggering a whole-cell timeout - Made `tab.evaluate` run in the page's main JavaScript world so page-defined globals are available without a directive @@ -26,6 +31,7 @@ ### Fixed +- Fixed the eval tool's status-event tree truncating from the bottom: the newest `log()` progress lines were hidden behind an `… N more` marker while the oldest stayed visible; the tree now shows a tail window behind an `… N earlier` marker, and the expanded view widens to the viewport instead of a fixed 10 events - Fixed the `//!world=main` directive being silently ignored for string expressions passed to raw Puppeteer evaluation APIs - Fixed tab reuse issues where hung navigation or unhandled modals would cause initialization to stall and trigger a force-kill - Improved search reliability for Perplexity provider by forcing retrieval for all queries diff --git a/packages/coding-agent/src/web/search/provider.ts b/packages/coding-agent/src/web/search/provider.ts index 4c2ee6f28..3a4a33c2a 100644 --- a/packages/coding-agent/src/web/search/provider.ts +++ b/packages/coding-agent/src/web/search/provider.ts @@ -119,16 +119,6 @@ const PROVIDER_META: Record = { label: SEARCH_PROVIDER_LABELS.google, load: async () => new (await import("./providers/google")).GoogleProvider(), }, - bing: { - id: "bing", - label: SEARCH_PROVIDER_LABELS.bing, - load: async () => new (await import("./providers/bing")).BingProvider(), - }, - yahoo: { - id: "yahoo", - label: SEARCH_PROVIDER_LABELS.yahoo, - load: async () => new (await import("./providers/yahoo")).YahooProvider(), - }, ecosia: { id: "ecosia", label: SEARCH_PROVIDER_LABELS.ecosia, diff --git a/packages/coding-agent/src/web/search/providers/bing.ts b/packages/coding-agent/src/web/search/providers/bing.ts deleted file mode 100644 index aaad2d427..000000000 --- a/packages/coding-agent/src/web/search/providers/bing.ts +++ /dev/null @@ -1,197 +0,0 @@ -import type { AuthStorage } from "@oh-my-pi/pi-ai"; -import { parseHTML } from "linkedom"; -import type { SearchResponse, SearchSource } from "../../../web/search/types"; -import { SearchProviderError } from "../../../web/search/types"; -import { clampNumResults } from "../utils"; -import type { SearchParams } from "./base"; -import { SearchProvider } from "./base"; -import { browserFetch } from "./browser-page"; -import { classifyProviderHttpError, withHardTimeout } from "./utils"; - -/** - * Bing's HTML search frontend. A plain GET with browser navigation headers - * returns a fully server-rendered results page — no JavaScript challenge on - * the organic path — so we parse it directly without a real browser. - */ -const BING_HOME_URL = "https://www.bing.com/"; -const BING_SEARCH_URL = "https://www.bing.com/search"; -const DEFAULT_NUM_RESULTS = 10; -const MAX_NUM_RESULTS = 20; -const MS_PER_DAY = 86_400_000; - -/** - * Recency → Bing `filters=ex1:"…"` freshness codes, as emitted by Bing's own - * "Any time" dropdown. `year` has no fixed code; the dropdown emits a custom - * epoch-day range (`ez5__`, days since 1970-01-01) which - * {@link recencyToFilters} computes. Bing parses the parameter (the SERP - * filter UI reflects it) but enforcement is server-side and vantage-dependent. - */ -const RECENCY_TO_BING_EZ: Record, "year">, string> = { - day: "ez1", - week: "ez2", - month: "ez3", -}; - -/** Snippet containers observed on Bing result blocks, in preference order. */ -const BING_SNIPPET_SELECTORS: readonly string[] = [".b_caption p", "p[class*='b_lineclamp']", ".b_algoSlug"]; - -interface ParsedResult { - title: string; - url: string; - snippet?: string; -} - -/** Build the `filters` value for a recency window, mirroring Bing's dropdown URLs. */ -function recencyToFilters(recency: NonNullable): string { - if (recency === "year") { - const epochDay = Math.floor(Date.now() / MS_PER_DAY); - return `ex1:"ez5_${epochDay - 365}_${epochDay}"`; - } - return `ex1:"${RECENCY_TO_BING_EZ[recency]}"`; -} - -/** - * Resolve a Bing result href to the underlying target URL. - * - * Organic hrefs are usually wrapped as `https://www.bing.com/ck/a?…&u=a1` - * where the payload after the literal `a1` prefix is the unpadded base64url - * encoding of the target URL. Direct external hrefs also occur; Bing-internal - * links (vertical tabs, ads plumbing) and non-http(s) schemes are rejected. - */ -function unwrapResultUrl(href: string): string | undefined { - let url: URL; - try { - url = new URL(href, BING_HOME_URL); - } catch { - return undefined; - } - - if (url.hostname === "bing.com" || url.hostname.endsWith(".bing.com")) { - if (url.pathname !== "/ck/a") return undefined; - const wrapped = url.searchParams.get("u"); - if (!wrapped?.startsWith("a1")) return undefined; - try { - url = new URL(Buffer.from(wrapped.slice(2), "base64url").toString("utf-8")); - } catch { - return undefined; - } - } - - if (url.protocol !== "http:" && url.protocol !== "https:") return undefined; - return url.href; -} - -function findSnippet(item: Element): string | undefined { - for (const selector of BING_SNIPPET_SELECTORS) { - const text = (item.querySelector(selector)?.textContent ?? "").replace(/\s+/g, " ").trim(); - if (text) return text; - } - return undefined; -} - -/** - * Pull organic result blocks out of the page in document order. - * - * Each organic hit is an `
  • ` with the title link in - * `h2 > a[href]` (sitelink/attribution anchors live outside the `h2`) and the - * preview text in one of {@link BING_SNIPPET_SELECTORS}. Ads, answer cards, - * and the "no results" row use other classes and fall out naturally. - */ -function parseHtmlResults(html: string): ParsedResult[] { - const { document } = parseHTML(html); - const results: ParsedResult[] = []; - for (const item of document.querySelectorAll("li.b_algo")) { - const anchor = item.querySelector("h2 a[href]"); - const href = anchor?.getAttribute("href"); - if (!href) continue; - const url = unwrapResultUrl(href); - if (!url) continue; - const title = (anchor?.textContent ?? "").replace(/\s+/g, " ").trim(); - if (!title) continue; - results.push({ title, url, snippet: findSnippet(item) }); - } - return results; -} - -/** - * `true` when Bing answered with its CAPTCHA/consent interstitial instead of - * a results page. The challenge redirects to `/turing/captcha/…`; body - * markers are only trusted when no organic result block is present so a - * search *about* CAPTCHAs never trips the detector. - */ -function isChallengeResponse(html: string, finalUrl: string): boolean { - if (finalUrl.includes("/turing/captcha")) return true; - if (html.includes('class="b_algo"')) return false; - return /turing\/captcha|b_captcha|px-captcha|verify (?:that )?you are (?:a )?human/i.test(html); -} - -function buildSearchUrl(params: SearchParams, numResults: number): string { - const url = new URL(BING_SEARCH_URL); - url.searchParams.set("q", params.query); - url.searchParams.set("count", String(numResults)); - url.searchParams.set("mkt", "en-US"); - url.searchParams.set("setlang", "en"); - if (params.recency) url.searchParams.set("filters", recencyToFilters(params.recency)); - return url.href; -} - -async function callBingHtml(params: SearchParams, numResults: number): Promise { - const url = buildSearchUrl(params, numResults); - const page = await browserFetch(url, { - fetch: params.fetch ?? fetch, - signal: withHardTimeout(params.signal), - referer: BING_HOME_URL, - }); - - const body = page.html; - if (isChallengeResponse(body, page.url)) { - throw new SearchProviderError( - "bing", - "Bing blocked the request with a CAPTCHA challenge. Bing throttles automated searches from datacenter/shared-egress IPs; try the duckduckgo or mojeek provider, or configure a credentialed provider such as Brave, Tavily, Exa, or Kagi.", - 429, - ); - } - if (page.status < 200 || page.status >= 300) { - const classified = classifyProviderHttpError("bing", page.status, body); - if (classified) throw classified; - throw new SearchProviderError("bing", `Bing HTML error (${page.status})`, page.status); - } - - return body; -} - -/** Execute a Bing web search via the server-rendered HTML results page. */ -export async function searchBing(params: SearchParams): Promise { - const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); - const html = await callBingHtml(params, numResults); - const parsed = parseHtmlResults(html); - - const sources: SearchSource[] = []; - const seen = new Set(); - for (const result of parsed) { - if (seen.has(result.url)) continue; - seen.add(result.url); - sources.push({ title: result.title, url: result.url, snippet: result.snippet }); - if (sources.length >= numResults) break; - } - - return { provider: "bing", sources }; -} - -/** Search provider for Bing (no API key required). */ -export class BingProvider extends SearchProvider { - readonly id = "bing"; - readonly label = "Bing"; - - isAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - isExplicitlyAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - search(params: SearchParams): Promise { - return searchBing(params); - } -} diff --git a/packages/coding-agent/src/web/search/providers/public.ts b/packages/coding-agent/src/web/search/providers/public.ts index 4e0fb7ce8..34c5a5617 100644 --- a/packages/coding-agent/src/web/search/providers/public.ts +++ b/packages/coding-agent/src/web/search/providers/public.ts @@ -11,15 +11,13 @@ import { withHardTimeout } from "./utils"; * Credential-free engines the Public Web aggregate fans out to. Order is the * tiebreak for merged ranking (earlier engines win equal consensus/rank), so * engines with the best ranking quality when they answer come first: - * Google-index engines (startpage, google) lead, Bing-backed scrapers follow, - * and Mojeek's independent index breaks remaining ties (measured 2026-07). + * Google-index engines (startpage, google) lead, and Mojeek's independent + * index breaks remaining ties (measured 2026-07). */ const PUBLIC_ENGINE_IDS = [ "startpage", "google", "duckduckgo", - "bing", - "yahoo", "ecosia", "mojeek", ] as const satisfies readonly SearchProviderId[]; diff --git a/packages/coding-agent/src/web/search/providers/yahoo.ts b/packages/coding-agent/src/web/search/providers/yahoo.ts deleted file mode 100644 index bdc1e312f..000000000 --- a/packages/coding-agent/src/web/search/providers/yahoo.ts +++ /dev/null @@ -1,179 +0,0 @@ -import type { AuthStorage } from "@oh-my-pi/pi-ai"; -import { parseHTML } from "linkedom"; -import type { SearchResponse, SearchSource } from "../../../web/search/types"; -import { SearchProviderError } from "../../../web/search/types"; -import { clampNumResults } from "../utils"; -import type { SearchParams } from "./base"; -import { SearchProvider } from "./base"; -import { browserFetch } from "./browser-page"; -import { classifyProviderHttpError, withHardTimeout } from "./utils"; - -/** - * Yahoo Search's server-rendered results page. A plain GET with browser - * navigation headers returns the full SERP without any JavaScript challenge, - * so no headless-browser fallback is needed (verified live 2026-07). - */ -const YAHOO_HOME_URL = "https://search.yahoo.com/"; -const YAHOO_SEARCH_URL = "https://search.yahoo.com/search"; -const DEFAULT_NUM_RESULTS = 10; -const MAX_NUM_RESULTS = 20; - -/** - * Recency → Yahoo `btf` query param. Yahoo's time filter only offers - * day/week/month; `year` has no equivalent and is silently dropped per the - * {@link SearchParams.recency} contract. - */ -const RECENCY_TO_YAHOO_BTF: Partial, string>> = { - day: "d", - week: "w", - month: "m", -}; - -interface ParsedResult { - title: string; - url: string; - snippet?: string; -} - -/** - * Resolve a Yahoo result href back to the underlying target URL. - * - * Organic hrefs are wrapped through the click tracker - * `https://r.search.yahoo.com/_ylt=…/RU=/RK=…/RS=…`; - * the `/RU=` path segment carries the destination. Older layouts emit plain - * absolute hrefs, so both shapes are handled. Tracker links without a - * recoverable target and Yahoo-internal navigation are rejected. - */ -function unwrapResultUrl(href: string): string | undefined { - let url: URL; - try { - url = new URL(href, YAHOO_HOME_URL); - } catch { - return undefined; - } - if (url.protocol !== "http:" && url.protocol !== "https:") return undefined; - - const wrapped = /\/RU=([^/]+)/.exec(url.pathname); - if (wrapped) { - let target: string; - try { - target = decodeURIComponent(wrapped[1]); - } catch { - return undefined; - } - return target.startsWith("http://") || target.startsWith("https://") ? target : undefined; - } - // A tracker link without an RU segment has no recoverable destination. - if (url.hostname === "r.search.yahoo.com") return undefined; - // Relative hrefs resolve against the search host: internal navigation. - if (url.hostname === "search.yahoo.com") return undefined; - return url.href; -} - -/** - * Walk the SERP and pull organic result blocks in document order. - * - * Organics render as `
    ` blocks (inside `#web`'s - * `
      `): the title `

      ` sits inside the tracker `` in the current - * layout, while legacy layouts nested the `` inside `

      ` - * — both are handled. The preview text lives in a sibling - * `
      `. Module headers ("Videos", "People also ask") - * carry `

      `s outside `.algo` blocks and are excluded by construction. - */ -function parseHtmlResults(html: string): ParsedResult[] { - const { document } = parseHTML(html); - const results: ParsedResult[] = []; - for (const block of document.querySelectorAll("div.algo")) { - const heading = block.querySelector("h3"); - if (!heading) continue; - const anchor = heading.querySelector("a") ?? heading.closest("a"); - const href = anchor?.getAttribute("href"); - if (!href) continue; - const url = unwrapResultUrl(href); - if (!url) continue; - const title = (heading.textContent ?? "").replace(/\s+/g, " ").trim(); - if (!title) continue; - const snippet = (block.querySelector(".compText")?.textContent ?? "").replace(/\s+/g, " ").trim() || undefined; - results.push({ title, url, snippet }); - } - return results; -} - -/** - * `true` when Yahoo answered with its EU consent interstitial instead of - * results: either the request was redirected to consent.yahoo.com / - * guce.yahoo.com, or the body carries the consent form. The normal SERP - * mentions guce.yahoo.com only in a meta tag, so detection keys on the - * consent-host redirect and the `collectConsent` form action. - */ -function isConsentInterstitial(finalUrl: string, html: string): boolean { - if (/^https?:\/\/(?:[^/]*\.)?(?:consent|guce)\.yahoo\.com\//i.test(finalUrl)) return true; - return html.includes("consent.yahoo.com") || html.includes("collectConsent"); -} - -async function callYahooHtml(params: SearchParams, numResults: number): Promise { - const url = new URL(YAHOO_SEARCH_URL); - url.searchParams.set("p", params.query); - url.searchParams.set("n", String(numResults)); - const btf = params.recency ? RECENCY_TO_YAHOO_BTF[params.recency] : undefined; - if (btf) url.searchParams.set("btf", btf); - - const page = await browserFetch(url.href, { - fetch: params.fetch ?? fetch, - signal: withHardTimeout(params.signal), - referer: YAHOO_HOME_URL, - }); - - const body = page.html; - if (page.status < 200 || page.status >= 300) { - const classified = classifyProviderHttpError("yahoo", page.status, body); - if (classified) throw classified; - throw new SearchProviderError("yahoo", `Yahoo HTML error (${page.status})`, page.status); - } - - if (isConsentInterstitial(page.url, body)) { - throw new SearchProviderError( - "yahoo", - "Yahoo served its GDPR consent interstitial instead of search results. This typically affects EU egress IPs; use another web search provider such as DuckDuckGo, Brave, or Mojeek.", - 429, - ); - } - - return body; -} - -/** Execute a Yahoo web search via the server-rendered HTML results page. */ -export async function searchYahoo(params: SearchParams): Promise { - const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); - const html = await callYahooHtml(params, numResults); - const parsed = parseHtmlResults(html); - - const sources: SearchSource[] = []; - const seen = new Set(); - for (const result of parsed) { - if (seen.has(result.url)) continue; - seen.add(result.url); - sources.push({ title: result.title, url: result.url, snippet: result.snippet }); - if (sources.length >= numResults) break; - } - - return { provider: "yahoo", sources }; -} - -/** Search provider for Yahoo (no API key required). */ -export class YahooProvider extends SearchProvider { - readonly id = "yahoo"; - readonly label = "Yahoo"; - - isAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - isExplicitlyAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - search(params: SearchParams): Promise { - return searchYahoo(params); - } -} diff --git a/packages/coding-agent/src/web/search/types.ts b/packages/coding-agent/src/web/search/types.ts index 477525ea1..86ac098fa 100644 --- a/packages/coding-agent/src/web/search/types.ts +++ b/packages/coding-agent/src/web/search/types.ts @@ -53,16 +53,6 @@ export const SEARCH_PROVIDER_OPTIONS = [ label: "DuckDuckGo", description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs", }, - { - value: "bing", - label: "Bing", - description: "Credential-free HTML scrape of Bing results; may be bot-challenged", - }, - { - value: "yahoo", - label: "Yahoo", - description: "Credential-free HTML scrape of Yahoo (Bing-backed) results", - }, { value: "ecosia", label: "Ecosia", diff --git a/packages/coding-agent/test/tools/web-search-public.test.ts b/packages/coding-agent/test/tools/web-search-public.test.ts index a3c58b53f..681f5318a 100644 --- a/packages/coding-agent/test/tools/web-search-public.test.ts +++ b/packages/coding-agent/test/tools/web-search-public.test.ts @@ -18,7 +18,7 @@ const fakeAuthStorage = { } as unknown as AuthStorage; /** Restrict the fan-out to the two engines these tests provide fixtures for. */ -const NON_TEST_ENGINES: readonly SearchProviderId[] = ["bing", "yahoo", "ecosia", "startpage", "mojeek"]; +const NON_TEST_ENGINES: readonly SearchProviderId[] = ["ecosia", "startpage", "mojeek"]; function makeParams(query: string, fetch: FetchImpl): SearchParams { return { From 87a64b2f6a2d30a7b15092621b173634b0dcef74 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 00:54:11 +0200 Subject: [PATCH 42/80] feat(coding-agent): improved background job lifecycle and display - Stop propagating real-time updates for backgrounded Bash jobs to avoid UI flickering once a job enters the background. - Refine background task tracking in `EventController` to distinguish between persistent background tasks and transient backgrounded Bash commands. - Update UI rendering to display cleaner background job metadata in the footer instead of inline text notices. --- packages/coding-agent/CHANGELOG.md | 4 ++ .../src/modes/controllers/event-controller.ts | 49 +++++++----------- packages/coding-agent/src/tools/bash.ts | 51 +++++++++++-------- ...vent-controller-task-async-updates.test.ts | 33 +++++++++++- packages/coding-agent/test/tools.test.ts | 20 ++++++-- .../test/tools/bash-sixel-render.test.ts | 28 ++++++++++ 6 files changed, 126 insertions(+), 59 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index e0b1b1608..566150cc9 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -10,6 +10,10 @@ - Updated status event log to prioritize the most recent entries in the display window +### Fixed + +- Fixed backgrounded Bash blocks continuing to repaint with live and final job output; they now freeze with a compact job notice while completion is delivered separately + ### Removed - Removed the unreliable Bing and Yahoo HTML-scraping web search providers diff --git a/packages/coding-agent/src/modes/controllers/event-controller.ts b/packages/coding-agent/src/modes/controllers/event-controller.ts index ddea4f86a..08992fc5c 100644 --- a/packages/coding-agent/src/modes/controllers/event-controller.ts +++ b/packages/coding-agent/src/modes/controllers/event-controller.ts @@ -76,7 +76,7 @@ export class EventController { #lastVisibleBlockCount = 0; #renderedCustomMessages = new Set(); #lastIntent: string | undefined = undefined; - #backgroundToolCallIds = new Set(); + #backgroundTaskCallIds = new Set(); #readToolCallArgs = new Map>(); #readToolCallAssistantComponents = new Map(); #lastAssistantComponent: AssistantMessageComponent | undefined = undefined; @@ -281,7 +281,7 @@ export class EventController { this.#lastVisibleBlockCount = 0; this.#renderedCustomMessages.clear(); this.#lastIntent = undefined; - this.#backgroundToolCallIds.clear(); + this.#backgroundTaskCallIds.clear(); this.#readToolCallArgs.clear(); this.#readToolCallAssistantComponents.clear(); this.#lastAssistantComponent = undefined; @@ -828,9 +828,9 @@ export class EventController { // The turn ended without running these calls (abort/error/TTSR rewind), // so they will never produce a result. Seal them so they stop animating // and freeze instead of pinning the transcript live region while a retry - // streams fresh blocks below them. Background tools keep updating. + // streams fresh blocks below them. Background task calls keep updating. for (const [toolCallId, component] of this.ctx.pendingTools.entries()) { - if (!this.#backgroundToolCallIds.has(toolCallId) && component instanceof ToolExecutionComponent) { + if (!this.#backgroundTaskCallIds.has(toolCallId) && component instanceof ToolExecutionComponent) { component.seal(); } } @@ -948,7 +948,7 @@ export class EventController { // While the call is still executing — a mixed blocking+async task // call whose jobs settle before its blocking subset — treat it as a // partial frame: `tool_execution_end` still owns the terminal result. - const isTerminal = isFinalAsyncState && this.#backgroundToolCallIds.has(event.toolCallId); + const isTerminal = isFinalAsyncState && this.#backgroundTaskCallIds.has(event.toolCallId); component.updateResult( { ...event.partialResult, isError: asyncState === "failed" }, !isTerminal, @@ -956,7 +956,7 @@ export class EventController { ); if (isTerminal) { this.ctx.pendingTools.delete(event.toolCallId); - this.#backgroundToolCallIds.delete(event.toolCallId); + this.#backgroundTaskCallIds.delete(event.toolCallId); } this.ctx.ui.requestRender(); } @@ -977,13 +977,7 @@ export class EventController { component.updateResult({ ...event.result, isError: event.isError }, false, event.toolCallId); this.ctx.pendingTools.delete(event.toolCallId); } - const asyncState = (event.result.details as { async?: { state?: string } } | undefined)?.async?.state; - if (asyncState === "running") { - this.#backgroundToolCallIds.add(event.toolCallId); - } else { - this.#backgroundToolCallIds.delete(event.toolCallId); - this.#clearReadToolCall(event.toolCallId); - } + this.#clearReadToolCall(event.toolCallId); this.ctx.ui.requestRender(); } else { let component = this.ctx.pendingTools.get(event.toolCallId); @@ -996,29 +990,22 @@ export class EventController { component = group; this.ctx.pendingTools.set(event.toolCallId, group); } - const asyncState = (event.result.details as { async?: { state?: string } } | undefined)?.async?.state; - const isBackgroundRunning = asyncState === "running"; - component.updateResult({ ...event.result, isError: event.isError }, isBackgroundRunning, event.toolCallId); - if (isBackgroundRunning) { - this.#backgroundToolCallIds.add(event.toolCallId); - } else { - this.ctx.pendingTools.delete(event.toolCallId); - this.#backgroundToolCallIds.delete(event.toolCallId); - this.#clearReadToolCall(event.toolCallId); - } + component.updateResult({ ...event.result, isError: event.isError }, false, event.toolCallId); + this.ctx.pendingTools.delete(event.toolCallId); + this.#clearReadToolCall(event.toolCallId); this.ctx.ui.requestRender(); } } else { const component = this.ctx.pendingTools.get(event.toolCallId); if (component) { const asyncState = (event.result.details as { async?: { state?: string } } | undefined)?.async?.state; - const isBackgroundRunning = asyncState === "running"; - component.updateResult({ ...event.result, isError: event.isError }, isBackgroundRunning, event.toolCallId); - if (isBackgroundRunning) { - this.#backgroundToolCallIds.add(event.toolCallId); + const isBackgroundTask = event.toolName === "task" && asyncState === "running"; + component.updateResult({ ...event.result, isError: event.isError }, isBackgroundTask, event.toolCallId); + if (isBackgroundTask) { + this.#backgroundTaskCallIds.add(event.toolCallId); } else { this.ctx.pendingTools.delete(event.toolCallId); - this.#backgroundToolCallIds.delete(event.toolCallId); + this.#backgroundTaskCallIds.delete(event.toolCallId); } if (component instanceof ToolExecutionComponent && component.isDisplaceableBlock()) { if (event.toolName === "job" && component.canBeDisplacedBy("job")) { @@ -1098,7 +1085,7 @@ export class EventController { } await this.ctx.flushPendingModelSwitch(); for (const toolCallId of Array.from(this.ctx.pendingTools.keys())) { - if (!this.#backgroundToolCallIds.has(toolCallId)) { + if (!this.#backgroundTaskCallIds.has(toolCallId)) { // A foreground tool still pending at turn end never delivered a result; // seal it so it freezes (and stops animating) rather than lingering in // the transcript live region as a streaming preview until the next thaw. @@ -1112,8 +1099,8 @@ export class EventController { this.ctx.pendingTools.delete(toolCallId); } } - this.#backgroundToolCallIds = new Set( - Array.from(this.#backgroundToolCallIds).filter(toolCallId => this.ctx.pendingTools.has(toolCallId)), + this.#backgroundTaskCallIds = new Set( + Array.from(this.#backgroundTaskCallIds).filter(toolCallId => this.ctx.pendingTools.has(toolCallId)), ); this.#readToolCallArgs.clear(); this.#readToolCallAssistantComponents.clear(); diff --git a/packages/coding-agent/src/tools/bash.ts b/packages/coding-agent/src/tools/bash.ts index 14e67375b..d4dfe2458 100644 --- a/packages/coding-agent/src/tools/bash.ts +++ b/packages/coding-agent/src/tools/bash.ts @@ -193,10 +193,9 @@ type ManagedBashJobCompletion = interface ManagedBashJobHandle { jobId: string; - label: string; completion: Promise; getLatestText: () => string; - setBackgrounded: (backgrounded: boolean) => void; + stopUpdates: () => void; } function normalizeResultOutput(result: BashResult | BashInteractiveResult): string { @@ -327,6 +326,10 @@ function formatExitCodeNotice(exitCode: number): string { return `Command exited with code ${exitCode}`; } +function formatBackgroundNotice(jobId: string): string { + return `Backgrounded as job ${jobId}; result will be delivered automatically.`; +} + /** * Strip the trailing occurrence of `notice` (plus a single surrounding newline * on each side) so the TUI can echo the value via a styled footer label @@ -355,6 +358,11 @@ function stripExitCodeNotice(text: string, exitCode: number | undefined): string return stripTrailingNotice(text, formatExitCodeNotice(exitCode)); } +function stripBackgroundNotice(text: string, async: BashToolDetails["async"] | undefined): string { + if (async?.state !== "running") return text; + return stripTrailingNotice(text, formatBackgroundNotice(async.jobId)); +} + /** * Bash tool implementation. * @@ -518,7 +526,6 @@ export class BashTool implements AgentTool; onUpdate?: AgentToolUpdateCallback; - startBackgrounded: boolean; + forwardUpdates: boolean; }): ManagedBashJobHandle { const manager = this.session.asyncJobManager; if (!manager) { @@ -576,7 +579,7 @@ export class BashTool implements AgentTool 120 ? `${options.command.slice(0, 117)}...` : options.command; let latestText = ""; - let backgrounded = options.startBackgrounded; + let forwardUpdates = options.forwardUpdates; const completion = Promise.withResolvers(); const jobId = manager.register( @@ -632,11 +635,12 @@ export class BashTool implements AgentTool { + onProgress: async text => { latestText = text; + if (!forwardUpdates) return; await options.onUpdate?.({ content: [{ type: "text", text }], - details: backgrounded ? ((details ?? {}) as BashToolDetails) : {}, + details: {}, }); }, }, @@ -644,11 +648,10 @@ export class BashTool implements AgentTool latestText, - setBackgrounded: (nextBackgrounded: boolean) => { - backgrounded = nextBackgrounded; + stopUpdates: () => { + forwardUpdates = false; }, }; } @@ -813,9 +816,9 @@ export class BashTool implements AgentTool(config: ShellRendererConfig) { ) { return cachedLines; } - const strippedOutput = stripOutputNotice(rawOutput, details?.meta); + const withoutBackground = stripBackgroundNotice(rawOutput, details?.async); + const strippedOutput = stripOutputNotice(withoutBackground, details?.meta); const withoutExit = stripExitCodeNotice(strippedOutput, details?.exitCode); const withoutWall = stripWallTimeNotice(withoutExit, details?.wallTimeMs); const rawOutputArtifact = stripRawOutputArtifactNotice(withoutWall); @@ -1317,6 +1321,9 @@ export function createShellRenderer(config: ShellRendererConfig) { const requestedTimeoutSeconds = details?.requestedTimeoutSeconds; const wallTimeMs = details?.wallTimeMs; const statsParts: string[] = []; + if (details?.async?.state === "running") { + statsParts.push(`Backgrounded: ${details.async.jobId}`); + } if (wallTimeMs !== undefined) { statsParts.push(`Wall: ${formatWallTimeSeconds(wallTimeMs)}s`); } diff --git a/packages/coding-agent/test/modes/controllers/event-controller-task-async-updates.test.ts b/packages/coding-agent/test/modes/controllers/event-controller-task-async-updates.test.ts index 185831e4d..2a342e3ea 100644 --- a/packages/coding-agent/test/modes/controllers/event-controller-task-async-updates.test.ts +++ b/packages/coding-agent/test/modes/controllers/event-controller-task-async-updates.test.ts @@ -21,6 +21,7 @@ import { EventController } from "@oh-my-pi/pi-coding-agent/modes/controllers/eve import { initTheme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; import type { InteractiveModeContext } from "@oh-my-pi/pi-coding-agent/modes/types"; import type { TaskToolDetails } from "@oh-my-pi/pi-coding-agent/task/types"; +import type { BashToolDetails } from "@oh-my-pi/pi-coding-agent/tools/bash"; function taskResult(asyncState: "running" | "completed" | "failed" | undefined, text: string) { const details: TaskToolDetails = { @@ -32,7 +33,14 @@ function taskResult(asyncState: "running" | "completed" | "failed" | undefined, return { content: [{ type: "text" as const, text }], details }; } -describe("EventController task async update finalization", () => { +function bashResult(text: string) { + const details: BashToolDetails = { + async: { state: "running", jobId: "bash-1", type: "bash" }, + }; + return { content: [{ type: "text" as const, text }], details }; +} + +describe("EventController async update finalization", () => { const sealed: ToolExecutionComponent[] = []; beforeEach(async () => { @@ -132,4 +140,27 @@ describe("EventController task async update finalization", () => { expect(pendingTools.has("tc-task")).toBe(false); expect(component.isTranscriptBlockFinalized()).toBe(true); }); + + it("finalizes a backgrounded Bash block without tracking later job updates", async () => { + const { controller, pendingTools } = createFixture(); + await controller.handleEvent({ + type: "tool_execution_start", + toolCallId: "tc-bash", + toolName: "bash", + args: { command: "sleep 30" }, + }); + const component = pendingTools.get("tc-bash")!; + sealed.push(component); + + await controller.handleEvent({ + type: "tool_execution_end", + toolCallId: "tc-bash", + toolName: "bash", + result: bashResult("Backgrounded as job bash-1; result will be delivered automatically."), + isError: false, + }); + + expect(pendingTools.has("tc-bash")).toBe(false); + expect(component.isTranscriptBlockFinalized()).toBe(true); + }); }); diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index 90c001c02..1bff803e7 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -1414,6 +1414,7 @@ function b() { it("should auto-background long-running commands when enabled", async () => { const deliveries: Array<{ jobId: string; text: string }> = []; + const updates: string[] = []; const asyncJobManager = new AsyncJobManager({ onJobComplete: async (jobId, text) => { deliveries.push({ jobId, text }); @@ -1435,13 +1436,20 @@ function b() { ), ); - const result = await autoBackgroundBashTool.execute("test-call-9-auto-running", { - command: "printf 'start\\n'; sleep 0.03; printf 'done\\n'", - }); + const result = await autoBackgroundBashTool.execute( + "test-call-9-auto-running", + { + command: "printf 'start\\n'; sleep 0.03; printf 'done\\n'", + }, + undefined, + update => { + updates.push(update.content?.find(block => block.type === "text")?.text ?? ""); + }, + ); expect(result.details?.async?.state).toBe("running"); expect(result.details?.async?.type).toBe("bash"); - expect(getTextOutput(result)).toContain("Background job"); + expect(getTextOutput(result)).toContain("Backgrounded as job"); expect(getTextOutput(result)).toContain("start"); const jobId = result.details?.async?.jobId; @@ -1450,11 +1458,13 @@ function b() { } const runningJob = asyncJobManager.getJob(jobId); expect(runningJob?.status).toBe("running"); + const updatesAtBackground = updates.slice(); await runningJob?.promise; await asyncJobManager.drainDeliveries({ timeoutMs: 1 }); expect(deliveries).toHaveLength(1); expect(deliveries[0]?.jobId).toBe(jobId); expect(deliveries[0]?.text).toContain("done"); + expect(updates).toEqual(updatesAtBackground); await asyncJobManager.dispose(); }); @@ -1495,7 +1505,7 @@ function b() { expect(result.details?.timeoutSeconds).toBe(0.05); expect(result.details?.async?.state).toBe("running"); - expect(getTextOutput(result)).toContain("Background job"); + expect(getTextOutput(result)).toContain("Backgrounded as job"); const jobId = result.details?.async?.jobId; if (!jobId) { throw new Error("expected an auto-backgrounded job id"); diff --git a/packages/coding-agent/test/tools/bash-sixel-render.test.ts b/packages/coding-agent/test/tools/bash-sixel-render.test.ts index 45cb05568..e6d2d23e9 100644 --- a/packages/coding-agent/test/tools/bash-sixel-render.test.ts +++ b/packages/coding-agent/test/tools/bash-sixel-render.test.ts @@ -128,6 +128,34 @@ describe("bashToolRenderer", () => { expect(rendered).not.toContain("Wall time: 1.23 seconds"); }); + it("renders a backgrounded job as a static footer notice", async () => { + const theme = await getThemeByName("dark"); + expect(theme).toBeDefined(); + const uiTheme = theme!; + const component = bashToolRenderer.renderResult( + { + content: [ + { + type: "text", + text: "started\n\nBackgrounded as job bash-42; result will be delivered automatically.", + }, + ], + details: { + timeoutSeconds: 300, + async: { state: "running", jobId: "bash-42", type: "bash" }, + }, + isError: false, + }, + { expanded: false, isPartial: false }, + uiTheme, + { command: "sleep 30" }, + ); + const rendered = sanitizeText(component.render(120).join("\n")); + expect(rendered).toContain("started"); + expect(rendered).toContain("Backgrounded: bash-42"); + expect(rendered).not.toContain("result will be delivered automatically"); + }); + it("folds raw output artifact notices into the status footer", async () => { const theme = await getThemeByName("dark"); expect(theme).toBeDefined(); From fabded89e5d76888241ad702657819c571f695e3 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 00:55:03 +0200 Subject: [PATCH 43/80] fix(ai): classified empty provider responses as retriable to prevent failures - Marked empty-body provider errors as transient to allow session retries. - Enabled model-fallback chains for empty responses, preventing hard-failures. - Added regression test to verify empty responses are classified as retriable. --- packages/ai/CHANGELOG.md | 4 ++++ packages/ai/src/error/provider.ts | 9 +++++---- packages/ai/test/error-aierr.test.ts | 13 +++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index f72bb7642..927ed24cb 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed empty provider responses (e.g. "Cloud Code Assist API returned an empty response") being classified as non-retryable: `ProviderResponseError` with kind `empty-body` now carries the transient flag, so session retry and configured model-fallback chains engage instead of hard-failing the turn + ## [16.4.6] - 2026-07-12 ### Added diff --git a/packages/ai/src/error/provider.ts b/packages/ai/src/error/provider.ts index 36d83ee48..12c15e0d9 100644 --- a/packages/ai/src/error/provider.ts +++ b/packages/ai/src/error/provider.ts @@ -39,10 +39,11 @@ export class ProviderResponseError extends Error { // A safety filter block is a terminal provider finish, not a transient fault. if (this.kind === "content-blocked") attach(this, create(Flag.ProviderFinishError)); // An incomplete stream (connection dropped / truncated before any terminal - // event) never produced a finish reason — the request didn't complete, so it - // is safe to retry. The retry layer's replay-unsafe guard still blocks a - // retry when partial tool output was already emitted. - else if (this.kind === "incomplete-stream") attach(this, create(Flag.Transient)); + // event) or an empty body never produced any content — the request didn't + // complete, so it is safe to retry and eligible for model fallback. The + // retry layer's replay-unsafe guard still blocks a retry when partial tool + // output was already emitted. + else if (this.kind === "incomplete-stream" || this.kind === "empty-body") attach(this, create(Flag.Transient)); } } diff --git a/packages/ai/test/error-aierr.test.ts b/packages/ai/test/error-aierr.test.ts index 57617b60e..a9458255c 100644 --- a/packages/ai/test/error-aierr.test.ts +++ b/packages/ai/test/error-aierr.test.ts @@ -69,6 +69,19 @@ describe("AIError.classify — structural provider errors", () => { expect(AIError.retriable(id)).toBe(true); }); + it("classifies an empty provider response as transient + retryable", () => { + // Regression: "Cloud Code Assist API returned an empty response" matched no + // text pattern and empty-body carried no flag, so retry/model-fallback + // chains never engaged and the turn hard-failed. + const err = new AIError.ProviderResponseError("Cloud Code Assist API returned an empty response", { + provider: "google-antigravity", + kind: "empty-body", + }); + const id = AIError.classify(err); + expect(AIError.is(id, AIError.Flag.Transient)).toBe(true); + expect(AIError.retriable(id)).toBe(true); + }); + it("does not mark a terminal output provider error as transient", () => { const err = new AIError.ProviderResponseError("upstream error", { provider: "google", kind: "output" }); expect(AIError.retriable(AIError.classify(err))).toBe(false); From 58d6130b506b17c13c880247f53a1a3cebfb3b4b Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 00:59:43 +0200 Subject: [PATCH 44/80] feat(coding-agent): enabled model fallback for hard errors - Extended `AgentSession` to consult `retry.fallbackChains` on non-retryable (hard) model errors. - Implemented `#isHardErrorFallbackEligible` to validate eligibility for model switching before surfacing terminal errors. - Updated `#handleRetryableError` to orchestrate immediate model switching for hard errors, bypassing backoff-retries for the failing model. - Ensured hard errors propagate to the user if no fallback candidates are available or if no credential can be resolved for the fallback model. --- packages/coding-agent/CHANGELOG.md | 1 + .../coding-agent/src/session/agent-session.ts | 62 ++++++- .../test/agent-session-retry-fallback.test.ts | 159 +++++++++++++++++- 3 files changed, 210 insertions(+), 12 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 566150cc9..d9d9cc3c9 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -12,6 +12,7 @@ ### Fixed +- Fixed configured `retry.fallbackChains` never engaging on non-retryable provider errors (e.g. "Cloud Code Assist API returned an empty response"): a hard error on a model covered by a fallback chain now switches to the next candidate instead of failing the turn, while still never backoff-retrying the failing model itself - Fixed backgrounded Bash blocks continuing to repaint with live and final job output; they now freeze with a compact job notice while completion is delivered separately ### Removed diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 31d6b1383..a58edff03 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -4136,6 +4136,17 @@ export class AgentSession { await emitAgentEndNotification(); return; } + } else if (this.#isHardErrorFallbackEligible(msg)) { + // A non-retryable hard error on a model covered by a configured + // fallback chain: retrying the SAME model is pointless, but a + // DIFFERENT model is a fresh chance — consult the chain before + // surfacing the failure. #handleRetryableError bails out (no + // backoff-retry of the failing model) when no switch happens. + const didRetry = await this.#handleRetryableError(msg, { hardErrorFallback: true }); + if (didRetry) { + await emitAgentEndNotification(); + return; + } } // Classifier refusals are persisted-skipped above; also prune the trailing // stub from active context so the next turn's prompt does not replay it. @@ -13845,6 +13856,34 @@ export class AgentSession { return this.#modelRegistry.find("fireworks", toFireworksBaseModelId(model.id)) !== undefined; } + /** + * True when a turn failed with a hard (non-retryable) provider error but a + * configured `retry.fallbackChains` entry covers the active model: the same + * model is not worth retrying, yet a DIFFERENT model is a fresh chance, so + * the chain is consulted before the error becomes final. Skips failures a + * model switch cannot fix or must not replay: cancellations (abort-flavored + * errors are not model faults), context overflow (compaction's job), + * classifier refusals (chain consult is handled on the retryable path with + * `pinFallback`), and turns that already emitted a tool call (replaying + * could duplicate work). + */ + #isHardErrorFallbackEligible(message: AssistantMessage): boolean { + if (message.stopReason !== "error") return false; + const model = this.model; + if (!model) return false; + const retrySettings = this.settings.getGroup("retry"); + if (!retrySettings.enabled || !retrySettings.modelFallback) return false; + if (this.#isClassifierRefusal(message)) return false; + const id = this.#classifyRetryMessage(message); + if (AIError.is(id, AIError.Flag.Abort) || AIError.is(id, AIError.Flag.UserInterrupt)) return false; + if (AIError.isContextOverflow(message, model.contextWindow ?? 0)) return false; + if (this.#hasReplayUnsafeToolOutput(message)) return false; + const currentSelector = formatRetryFallbackSelector(model, this.thinkingLevel); + const role = this.#activeRetryFallback?.role ?? this.#resolveRetryFallbackRole(currentSelector); + if (!role) return false; + return this.#findRetryFallbackCandidates(role, currentSelector).length > 0; + } + /** * Switch the active model from a Fireworks Fast (`-fast`) variant to its base * (Standard) id and stick there for the rest of the session — the auto @@ -13968,12 +14007,16 @@ export class AgentSession { } /** - * Handle retryable errors with exponential backoff. + * Handle retryable errors with exponential backoff, credential rotation, and + * model-fallback chains. Also entered for NON-retryable errors when a switch + * is the recovery (`fireworksFastFallback`, `hardErrorFallback`): then a + * successful model switch retries immediately, and a failed switch surfaces + * the error without a same-model backoff retry. * @returns true if retry was initiated, false if max retries exceeded or disabled */ async #handleRetryableError( message: AssistantMessage, - options?: { allowModelFallback?: boolean; fireworksFastFallback?: boolean }, + options?: { allowModelFallback?: boolean; fireworksFastFallback?: boolean; hardErrorFallback?: boolean }, ): Promise { const retrySettings = this.settings.getGroup("retry"); // The Fireworks Fast→base degrade is an intrinsic model-selection safety net, @@ -14115,11 +14158,16 @@ export class AgentSession { this.#resolveRetry(); return false; } - // Fast→base was requested but the base switch could not happen (e.g. the - // base model has no credential). Don't fall through to backing-off and - // retrying the failing fast model for a hard router error that the generic - // classifier wouldn't retry — surface it instead. - if (options?.fireworksFastFallback && !switchedModel && !this.#isRetryableError(message)) { + // A fallback switch was the whole reason we entered (Fast→base degrade or + // a hard-error chain consult) but it could not happen (e.g. no candidate + // has a credential). Don't fall through to backing-off and retrying the + // failing model for an error the generic classifier wouldn't retry — + // surface it instead. + if ( + (options?.fireworksFastFallback || options?.hardErrorFallback) && + !switchedModel && + !this.#isRetryableError(message) + ) { this.#retryAttempt = 0; this.#resolveRetry(); return false; diff --git a/packages/coding-agent/test/agent-session-retry-fallback.test.ts b/packages/coding-agent/test/agent-session-retry-fallback.test.ts index 0c11ed000..1e5494cb4 100644 --- a/packages/coding-agent/test/agent-session-retry-fallback.test.ts +++ b/packages/coding-agent/test/agent-session-retry-fallback.test.ts @@ -444,6 +444,140 @@ describe("AgentSession retry fallback", () => { ]); }); + it("consults the fallback chain on a non-retryable hard error instead of failing the turn", async () => { + const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5"); + const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); + if (!primaryModel || !fallbackModel) { + throw new Error("Expected bundled test models to exist"); + } + + const mock = createMockModel(); + const requestedModels: string[] = []; + const fallbackAppliedEvents: Array> = []; + const agent = new Agent({ + getApiKey: model => `${model.provider}-test-key`, + initialState: { + model: primaryModel, + systemPrompt: ["Test"], + tools: [], + messages: [], + }, + streamFn: (model, context, options) => { + requestedModels.push(`${model.provider}/${model.id}`); + if (model.provider === primaryModel.provider) { + // Classifies as neither transient, usage-limit, nor auth: + // the generic retry classifier rejects it outright. + mock.push({ throw: "unrecoverable model quirk" }); + } else { + mock.push({ content: ["Recovered on fallback"] }); + } + return mock.stream(model, context, options); + }, + }); + + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.baseDelayMs": 5, + "retry.fallbackChains": { + "anthropic/*": [`${fallbackModel.provider}/${fallbackModel.id}`], + }, + }); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + + session.subscribe(event => { + if (event.type === "retry_fallback_applied") { + fallbackAppliedEvents.push(event); + } + }); + + await session.prompt("Survive a hard error"); + await session.waitForIdle(); + + // Exactly one attempt on the failing model: a hard error switches models + // immediately, it never backoff-retries the same model. + expect(requestedModels).toEqual([ + `${primaryModel.provider}/${primaryModel.id}`, + `${fallbackModel.provider}/${fallbackModel.id}`, + ]); + expect(fallbackAppliedEvents).toEqual([ + { + type: "retry_fallback_applied", + from: `${primaryModel.provider}/${primaryModel.id}`, + to: `${fallbackModel.provider}/${fallbackModel.id}`, + role: "anthropic/*", + }, + ]); + expect(session.model?.provider).toBe(fallbackModel.provider); + expect(getLastAssistantMessage(session).stopReason).toBe("stop"); + }); + + it("surfaces a non-retryable error without same-model retries when no fallback candidate has a credential", async () => { + const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5"); + const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); + if (!primaryModel || !fallbackModel) { + throw new Error("Expected bundled test models to exist"); + } + + const originalGetApiKey = modelRegistry.getApiKey.bind(modelRegistry); + vi.spyOn(modelRegistry, "getApiKey").mockImplementation((model, sessionId) => + model.provider === fallbackModel.provider ? Promise.resolve(undefined) : originalGetApiKey(model, sessionId), + ); + + const mock = createMockModel(); + const requestedModels: string[] = []; + const fallbackAppliedEvents: Array> = []; + const agent = new Agent({ + getApiKey: model => `${model.provider}-test-key`, + initialState: { + model: primaryModel, + systemPrompt: ["Test"], + tools: [], + messages: [], + }, + streamFn: (model, context, options) => { + requestedModels.push(`${model.provider}/${model.id}`); + mock.push({ throw: "unrecoverable model quirk" }); + return mock.stream(model, context, options); + }, + }); + + const settings = Settings.isolated({ + "compaction.enabled": false, + "retry.baseDelayMs": 5, + "retry.fallbackChains": { + "anthropic/*": [`${fallbackModel.provider}/${fallbackModel.id}`], + }, + }); + + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings, + modelRegistry, + }); + + session.subscribe(event => { + if (event.type === "retry_fallback_applied") { + fallbackAppliedEvents.push(event); + } + }); + + await session.prompt("Fail hard with no fallback credential"); + await session.waitForIdle(); + + // The switch could not happen and the error is non-retryable: surface it + // after a single attempt instead of backoff-retrying the failing model. + expect(requestedModels).toEqual([`${primaryModel.provider}/${primaryModel.id}`]); + expect(fallbackAppliedEvents).toEqual([]); + expect(getLastAssistantMessage(session).stopReason).toBe("error"); + }); + it("substitutes the failing model id into provider-wildcard chain entries", async () => { const primaryModel = getBundledModel("google", "gemini-2.5-flash"); const fallbackModel = getBundledModel("google-vertex", "gemini-2.5-flash"); @@ -1387,13 +1521,16 @@ describe("AgentSession retry fallback", () => { expect(lastAssistant.content).toContainEqual({ type: "text", text: "Recovered after Anthropic envelope retry" }); }); - it("does not auto-retry Anthropic stream-envelope failures before terminal stop signal", async () => { + it("falls back on mid-stream Anthropic envelope failures without same-model retries", async () => { const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5"); const fallbackModel = getBundledModel("openai", "gpt-4o-mini"); if (!primaryModel || !fallbackModel) { throw new Error("Expected bundled test models to exist"); } + // Mid-stream envelope corruption is not auto-retried on the same model + // (partial content may have been delivered), but a configured fallback + // chain is still consulted: a different model is a fresh chance. const envelopeError = "Anthropic stream envelope error: received content_block_delta before terminal stop signal"; const requestedModels: string[] = []; const fallbackAppliedEvents: Array> = []; @@ -1443,11 +1580,23 @@ describe("AgentSession retry fallback", () => { await session.prompt("Do not retry Anthropic envelope failure before terminal stop signal"); await session.waitForIdle(); - expect(requestedModels).toEqual([`${primaryModel.provider}/${primaryModel.id}`]); - expect(retryStartEvents).toHaveLength(0); - expect(retryEndEvents).toHaveLength(0); - expect(fallbackAppliedEvents).toHaveLength(0); + // One attempt per model: chain advances, never a same-model backoff retry. + expect(requestedModels).toEqual([ + `${primaryModel.provider}/${primaryModel.id}`, + `${fallbackModel.provider}/${fallbackModel.id}`, + ]); + expect(fallbackAppliedEvents).toEqual([ + { + type: "retry_fallback_applied", + from: `${primaryModel.provider}/${primaryModel.id}`, + to: `${fallbackModel.provider}/${fallbackModel.id}`, + role: "default", + }, + ]); + // The fallback fails with the same hard error and the chain is exhausted: + // the failure surfaces instead of looping. expect(fallbackSucceededEvents).toHaveLength(0); + expect(retryStartEvents).toHaveLength(1); const lastAssistant = getLastAssistantMessage(session); expect(lastAssistant.stopReason).toBe("error"); expect(lastAssistant.errorMessage).toBe(envelopeError); From 46ed33f27b8da64c909b2641742d05f764cd6ad6 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:09:42 +0200 Subject: [PATCH 45/80] feat(coding-agent): initialized session metadata during tan creation - Initialized session metadata including system prompt, task, and toolset within the controller. - Added session initialization tracking to the clone creation flow to ensure session state visibility. - Updated unit tests to verify that session initialization data is correctly appended when a tan is created. --- .../src/modes/controllers/tan-command-controller.ts | 5 +++++ .../test/modes/controllers/tan-command-controller.test.ts | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/packages/coding-agent/src/modes/controllers/tan-command-controller.ts b/packages/coding-agent/src/modes/controllers/tan-command-controller.ts index 0fcec31fc..3b19087dd 100644 --- a/packages/coding-agent/src/modes/controllers/tan-command-controller.ts +++ b/packages/coding-agent/src/modes/controllers/tan-command-controller.ts @@ -127,6 +127,11 @@ export class TanCommandController { disableExtensionDiscovery: true, }); clone = created.session; + clone.sessionManager?.appendSessionInit?.({ + systemPrompt: clone.systemPrompt ? clone.systemPrompt.join("\n\n") : systemPrompt.join("\n\n"), + task: trimmedWork, + tools: clone.getActiveToolNames ? clone.getActiveToolNames() : toolNames, + }); const abortClone = () => { void clone?.abort(); }; diff --git a/packages/coding-agent/test/modes/controllers/tan-command-controller.test.ts b/packages/coding-agent/test/modes/controllers/tan-command-controller.test.ts index 5cf608994..d9e2ce529 100644 --- a/packages/coding-agent/test/modes/controllers/tan-command-controller.test.ts +++ b/packages/coding-agent/test/modes/controllers/tan-command-controller.test.ts @@ -261,7 +261,9 @@ describe("TanCommandController", () => { it("parks the finished tan in the registry so it stays visible in the Agent Hub", async () => { const harness = createContext(); vi.spyOn(SessionManager, "forkFrom").mockResolvedValue(harness.cloneManager); + const appendSessionInit = vi.fn(); const clone = { + sessionManager: { appendSessionInit }, prompt: vi.fn(async () => {}), waitForIdle: vi.fn(async () => {}), getLastAssistantMessage: vi.fn(() => assistantText("done")), @@ -287,6 +289,11 @@ describe("TanCommandController", () => { }); expect(result).toBe("done"); + expect(appendSessionInit).toHaveBeenCalledWith({ + systemPrompt: "system prompt", + task: "park me", + tools: ["read", "bash"], + }); // Parked (not unregistered) before dispose, then the disposed session is nulled // out — the hub keeps the ref and reads its transcript from the session file. expect(setStatus).toHaveBeenCalledWith(expect.stringMatching(/^Tan-/), "parked"); From bf5eb3769ffcc128733533492b9a41d2f29cd4a4 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:27:30 +0200 Subject: [PATCH 46/80] feat(coding-agent): rebased pending context snapshot after compaction - Update in-flight context snapshots after historical messages are modified by compaction or elision. - Prevent stale run-start token counts from triggering false-positive dead-end "no progress" warnings during auto-continuation. - Add regression test to verify that prompt headroom measurements correctly account for post-compaction context sizes. --- packages/coding-agent/CHANGELOG.md | 2 + .../coding-agent/src/session/agent-session.ts | 31 ++++++++- ...ion-auto-compaction-progress-guard.test.ts | 69 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d9d9cc3c9..0ca555e84 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,6 +13,8 @@ ### Fixed - Fixed configured `retry.fallbackChains` never engaging on non-retryable provider errors (e.g. "Cloud Code Assist API returned an empty response"): a hard error on a model covered by a fallback chain now switches to the next candidate instead of failing the turn, while still never backoff-retrying the failing model itself +- Fixed the post-compaction transcript rebuild (auto-compaction and `/compact`) repainting the entire collapsed transcript — welcome banner included — below the stale pre-compaction scrollback: the rebuild collapses history behind the summary divider, shrinking the frame far below the committed row count, and the renderer's "duplication, never loss" resync re-showed everything without retracting native scrollback; both rebuild paths now request `clearScrollback` like auto-handoff already did +- Fixed mid-run auto-compaction spuriously warning "Compaction freed too little context to make progress" and pausing maintenance even when compaction genuinely shrank the context (observed: snapcompact took a 312k-token gpt-5.6 session to 86k real tokens and still dead-ended): the in-flight prompt's pending context snapshot — set once at run start and alive for the whole tool loop — was read as live residual context by the post-compaction headroom/retry-fit checks because the fresh compaction entry hides every earlier usage anchor; compaction (auto and `/compact`), the retry drop, and the dead-end shake rescue now rebase the snapshot onto the rewritten message set - Fixed backgrounded Bash blocks continuing to repaint with live and final job output; they now freeze with a compact job notice while completion is delivered separately ### Removed diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index a58edff03..389bfcc04 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -10115,6 +10115,7 @@ export class AgentSession { const newEntries = this.sessionManager.getEntries(); const sessionContext = this.buildDisplaySessionContext(); this.agent.replaceMessages(sessionContext.messages); + this.#rebasePendingContextSnapshotAfterCompaction(); // Compaction discarded the conversation history that carried the approved // plan reference. Clear the sent-flag so #buildPlanReferenceMessage re-reads // the plan from disk and re-injects it on the next turn (issue #1246). @@ -12534,7 +12535,11 @@ export class AgentSession { if (signal.aborted) return undefined; try { const result = await this.shake("elide", { signal }); - return result.toolResultsDropped + result.blocksDropped > 0 ? result : undefined; + if (result.toolResultsDropped + result.blocksDropped === 0) return undefined; + // The elide pass rewrote history; re-anchor the in-flight snapshot so + // the caller's headroom/retry-fit re-test measures the shaken context. + this.#rebasePendingContextSnapshotAfterCompaction(); + return result; } catch (error) { logger.warn("Dead-end shake rescue failed", { error: error instanceof Error ? error.message : String(error), @@ -13059,6 +13064,7 @@ export class AgentSession { const newEntries = this.sessionManager.getEntries(); const sessionContext = this.buildDisplaySessionContext(); this.agent.replaceMessages(sessionContext.messages); + this.#rebasePendingContextSnapshotAfterCompaction(); // Compaction discarded the conversation history that carried the approved // plan reference. Clear the sent-flag so #buildPlanReferenceMessage re-reads // the plan from disk and re-injects it on the next turn (issue #1246). @@ -13124,6 +13130,7 @@ export class AgentSession { (reason === "incomplete" && lastAssistant.stopReason === "length"); if (shouldDrop) { this.agent.replaceMessages(messages.slice(0, -1)); + this.#rebasePendingContextSnapshotAfterCompaction(); } } @@ -15941,6 +15948,28 @@ export class AgentSession { this.#contextUsageRevision++; } + /** + * Rebase the in-flight pending context snapshot onto the current message + * set after a compaction (or its dead-end rescue) rewrote history mid-run. + * The snapshot captures the prompt as submitted at run start and lives for + * the whole run; once a compaction entry lands, every earlier usage anchor + * is hidden from {@link getContextBreakdown}, so the stale run-start figure + * would be reported as live context until the next provider response. That + * inflated residual is what the post-compaction headroom/retry-fit checks + * measure — a run that started above the recovery band then trips the + * "freed too little context" dead-end even when compaction genuinely + * shrank the context. No-op while no prompt is in flight. + */ + #rebasePendingContextSnapshotAfterCompaction(): void { + if (!this.#pendingContextSnapshot) return; + const nonMessageTokens = computeNonMessageTokens(this); + this.#setPendingContextSnapshot({ + promptTokens: nonMessageTokens + this.messages.reduce((sum, msg) => sum + estimateTokens(msg), 0), + nonMessageTokens, + cutoffCount: this.messages.length, + }); + } + #ingestProviderUsageHeaders(response: ProviderResponseMetadata, model?: Model): void { if (model?.provider !== "anthropic") return; this.#modelRegistry.authStorage.ingestUsageHeaders("anthropic", response.headers, { diff --git a/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts b/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts index e537902df..f90ced886 100644 --- a/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts +++ b/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts @@ -416,6 +416,75 @@ describe("AgentSession auto-compaction progress guard", () => { const noProgress = notices.filter(n => n.source === NOTICE_SOURCE && n.message.includes(NO_PROGRESS_FRAGMENT)); expect(noProgress.length).toBe(0); }); + + it("rebases the in-flight prompt snapshot so mid-run compaction is not misread as a dead-end", async () => { + // Regression: the pending context snapshot is set once per prompt and + // lives for the whole run. A fresh compaction entry hides every earlier + // usage anchor from getContextBreakdown, which then fell back to the + // stale run-start figure until the next provider response — a run + // submitted above the recovery band (0.8 × 170k = 136k here) tripped the + // "freed too little context" warning even though compaction had + // genuinely shrunk the context (observed live: 312k → 86k real tokens, + // warning still emitted). + const continueSpy = vi.spyOn(session.agent, "continue").mockResolvedValue(); + // Hold the initial prompt in flight so the pending snapshot stays alive + // through the compaction, exactly like a live tool-loop run. The second + // agent.prompt call is the scheduled auto-continue — the "headroom was + // seen" signal the test awaits. + const gate = Promise.withResolvers(); + const firstPromptCall = Promise.withResolvers(); + const secondPromptCall = Promise.withResolvers(); + let promptCalls = 0; + const promptSpy = vi.spyOn(session.agent, "prompt").mockImplementation(() => { + promptCalls++; + if (promptCalls === 1) firstPromptCall.resolve(); + if (promptCalls === 2) secondPromptCall.resolve(); + return gate.promise as never; + }); + + const notices = collectNotices(); + // The dead-end warning is the "no headroom was seen" signal: the headroom + // tail runs AFTER auto_compaction_end is emitted, so the test awaits one + // of the tail's two observable outcomes instead of the end event. + const noProgressSeen = Promise.withResolvers(); + session.subscribe(event => { + if (event.type === "notice" && event.message.includes(NO_PROGRESS_FRAGMENT)) noProgressSeen.resolve(); + }); + + // ~150k-token prompt: above the recovery band, below the 170k threshold, + // so the pre-prompt maintenance pass stays quiet and the snapshot records + // the run-start size. agent.prompt is mocked, so the text never reaches + // the branch — it exists only in the in-flight snapshot. + const inFlightPrompt = session.prompt("x".repeat(600_000)); + // The snapshot is written immediately before agent.prompt; awaiting the + // first (gated) call guarantees it is in place before the threshold turn + // lands — emitting earlier would race the submission pipeline and let + // compaction run against an unset snapshot. + await firstPromptCall.promise; + + // Mid-run, the billed context crosses the threshold and compaction fires; + // the rewritten context (summary only) is tiny. + const assistantMsg = highUsageAssistant(); + session.agent.emitExternalEvent({ type: "message_end", message: assistantMsg }); + session.agent.emitExternalEvent({ type: "agent_end", messages: [assistantMsg] }); + // Wait for the headroom verdict while the prompt is still gated — + // releasing the gate earlier would clear the snapshot and mask the + // regression. Fixed behavior schedules the auto-continue (second prompt + // call); the regression emits the dead-end warning instead. + await Promise.race([secondPromptCall.promise, noProgressSeen.promise]); + + gate.resolve(); + await inFlightPrompt; + await session.waitForIdle(); + + // The stale 150k run-start snapshot must not be measured as residual + // context: no dead-end warning, and the auto-continue prompt ran + // (initial call + continuation). + expect(promptSpy).toHaveBeenCalledTimes(2); + expect(continueSpy).not.toHaveBeenCalled(); + const noProgress = notices.filter(n => n.source === NOTICE_SOURCE && n.message.includes(NO_PROGRESS_FRAGMENT)); + expect(noProgress.length).toBe(0); + }); /** * Seed several large prior turns into the session branch so `prepareCompaction` * returns a real preparation after the overflow recovery drops the failed From d6f8c061b40f33f341ef0ca38d813fc43839f749 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:29:50 +0200 Subject: [PATCH 47/80] feat(config): introduced schema for collapsed transcript mode - Added display.collapseCompacted boolean setting to the appearance tab. - Enabled default collapse behavior for pre-compaction history in the transcript view. --- packages/coding-agent/CHANGELOG.md | 9 +++++---- packages/coding-agent/src/config/settings-schema.ts | 12 ++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 0ca555e84..766f03e5a 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,12 +4,17 @@ ### Added +- Added display setting to toggle between collapsing or keeping compacted history inline - Added a compact session-only model picker (Alt+P) for quick model switching without changing roles ### Changed - Updated status event log to prioritize the most recent entries in the display window +### Removed + +- Removed the unreliable Bing and Yahoo HTML-scraping web search providers + ### Fixed - Fixed configured `retry.fallbackChains` never engaging on non-retryable provider errors (e.g. "Cloud Code Assist API returned an empty response"): a hard error on a model covered by a fallback chain now switches to the next candidate instead of failing the turn, while still never backoff-retrying the failing model itself @@ -17,10 +22,6 @@ - Fixed mid-run auto-compaction spuriously warning "Compaction freed too little context to make progress" and pausing maintenance even when compaction genuinely shrank the context (observed: snapcompact took a 312k-token gpt-5.6 session to 86k real tokens and still dead-ended): the in-flight prompt's pending context snapshot — set once at run start and alive for the whole tool loop — was read as live residual context by the post-compaction headroom/retry-fit checks because the fresh compaction entry hides every earlier usage anchor; compaction (auto and `/compact`), the retry drop, and the dead-end shake rescue now rebase the snapshot onto the rewritten message set - Fixed backgrounded Bash blocks continuing to repaint with live and final job output; they now freeze with a compact job notice while completion is delivered separately -### Removed - -- Removed the unreliable Bing and Yahoo HTML-scraping web search providers - ## [16.4.8] - 2026-07-12 ### Added diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index e702bcc20..182ca9a38 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -918,6 +918,18 @@ export const SETTINGS_SCHEMA = { }, }, + "display.collapseCompacted": { + type: "boolean", + default: true, + ui: { + tab: "appearance", + group: "Display", + label: "Collapse Compacted History", + description: + "Collapse pre-compaction history behind the summary divider on the live transcript; disable to keep the full transcript inline with dividers at each compaction point", + }, + }, + showHardwareCursor: { type: "boolean", default: true, // will be computed based on platform if undefined From 5c2bae47ac8ee4e0421edf1bcae99411a5009238 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:29:53 +0200 Subject: [PATCH 48/80] feat(mode): enabled conditional transcript compaction logic - Introduced conditional scrollback clearing during UI renders when transcript compaction is enabled. - Updated `CommandController` and `EventController` to respect the `display.collapseCompacted` setting. - Configured `SelectorController` to trigger a chat rebuild and UI reset when the compaction setting changes. - Updated `InteractiveMode` to dynamically toggle between collapsed and full inline history based on user settings. --- packages/coding-agent/CHANGELOG.md | 3 ++- .../src/modes/controllers/command-controller.ts | 10 +++++++++- .../src/modes/controllers/event-controller.ts | 14 +++++++++++++- .../src/modes/controllers/selector-controller.ts | 7 +++++++ .../coding-agent/src/modes/interactive-mode.ts | 9 ++++++--- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 766f03e5a..2f95527a6 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -17,8 +17,9 @@ ### Fixed +- Fixed inconsistent history rendering when toggling the display setting for compacted items - Fixed configured `retry.fallbackChains` never engaging on non-retryable provider errors (e.g. "Cloud Code Assist API returned an empty response"): a hard error on a model covered by a fallback chain now switches to the next candidate instead of failing the turn, while still never backoff-retrying the failing model itself -- Fixed the post-compaction transcript rebuild (auto-compaction and `/compact`) repainting the entire collapsed transcript — welcome banner included — below the stale pre-compaction scrollback: the rebuild collapses history behind the summary divider, shrinking the frame far below the committed row count, and the renderer's "duplication, never loss" resync re-showed everything without retracting native scrollback; both rebuild paths now request `clearScrollback` like auto-handoff already did +- Fixed transcript rebuilds (compaction, `/compact`, and toggling history display) repainting content below stale scrollback when collapsing history; rebuilds now correctly clear the scrollback buffer when history is collapsed - Fixed mid-run auto-compaction spuriously warning "Compaction freed too little context to make progress" and pausing maintenance even when compaction genuinely shrank the context (observed: snapcompact took a 312k-token gpt-5.6 session to 86k real tokens and still dead-ended): the in-flight prompt's pending context snapshot — set once at run start and alive for the whole tool loop — was read as live residual context by the post-compaction headroom/retry-fit checks because the fresh compaction entry hides every earlier usage anchor; compaction (auto and `/compact`), the retry drop, and the dead-end shake rescue now rebase the snapshot onto the rewritten message set - Fixed backgrounded Bash blocks continuing to repaint with live and final job output; they now freeze with a compact job notice while completion is delivered separately diff --git a/packages/coding-agent/src/modes/controllers/command-controller.ts b/packages/coding-agent/src/modes/controllers/command-controller.ts index ef6363d89..44c99cbd4 100644 --- a/packages/coding-agent/src/modes/controllers/command-controller.ts +++ b/packages/coding-agent/src/modes/controllers/command-controller.ts @@ -1200,7 +1200,15 @@ export class CommandController { this.ctx.rebuildChatFromMessages(); this.ctx.statusLine.invalidate(); - this.ctx.ui.requestRender(); + // Same as the auto-compaction rebuild: a collapsed transcript is an + // intentional replacement, so drop the stale pre-compaction scrollback + // instead of repainting the shrunken frame below it. With collapse + // disabled the full history stays inline and scrollback is kept. + if (this.ctx.settings.get("display.collapseCompacted")) { + this.ctx.ui.requestRender(true, { clearScrollback: true }); + } else { + this.ctx.ui.requestRender(); + } } catch (error) { if (error instanceof CompactionCancelledError) { outcome = "cancelled"; diff --git a/packages/coding-agent/src/modes/controllers/event-controller.ts b/packages/coding-agent/src/modes/controllers/event-controller.ts index 08992fc5c..dee063c49 100644 --- a/packages/coding-agent/src/modes/controllers/event-controller.ts +++ b/packages/coding-agent/src/modes/controllers/event-controller.ts @@ -1234,7 +1234,19 @@ export class EventController { this.ctx.lastAssistantUsage = undefined; this.ctx.rebuildChatFromMessages(); this.ctx.statusLine.invalidate(); - this.ctx.ui.requestRender(); + // When history collapses behind the summary divider, the frame + // shrinks far below the committed row count; without clearing, the + // differential renderer's "duplication, never loss" resync repaints + // the whole collapsed transcript (welcome box included) BELOW the + // stale pre-compaction scrollback. Compaction is an intentional + // transcript replacement then — same as auto-handoff below. With + // collapse disabled the rebuilt transcript keeps the full history, + // so the resync handles it and scrollback stays. + if (settings.get("display.collapseCompacted")) { + this.ctx.ui.requestRender(true, { clearScrollback: true }); + } else { + this.ctx.ui.requestRender(); + } } else if (event.errorMessage) { this.ctx.showWarning(event.errorMessage); } else if (isHandoffAction) { diff --git a/packages/coding-agent/src/modes/controllers/selector-controller.ts b/packages/coding-agent/src/modes/controllers/selector-controller.ts index e49ad1827..1b983cc92 100644 --- a/packages/coding-agent/src/modes/controllers/selector-controller.ts +++ b/packages/coding-agent/src/modes/controllers/selector-controller.ts @@ -452,6 +452,13 @@ export class SelectorController { this.ctx.rebuildChatFromMessages(); this.ctx.ui.resetDisplay(); break; + case "display.collapseCompacted": + // Rebuild swaps between the collapsed tail and the full inline + // history; full reset retires blocks already committed to native + // scrollback (mirrors cacheMissMarker). + this.ctx.rebuildChatFromMessages(); + this.ctx.ui.resetDisplay(); + break; case "tui.tight": setTuiTight(value as boolean); this.ctx.ui.invalidate(); diff --git a/packages/coding-agent/src/modes/interactive-mode.ts b/packages/coding-agent/src/modes/interactive-mode.ts index c53bea59b..8311acbc2 100644 --- a/packages/coding-agent/src/modes/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive-mode.ts @@ -1614,9 +1614,12 @@ export class InteractiveMode implements InteractiveModeContext { } } this.chatContainer.clear(); - // Live display uses the compacted transcript tail; export/resume callers - // can still request the full inline compaction history. - const context = this.viewSession.buildTranscriptSessionContext({ collapseCompactedHistory: true }); + // Live display collapses to the compacted transcript tail unless the + // user opted into the full inline history; export/resume callers choose + // their own mode. + const context = this.viewSession.buildTranscriptSessionContext({ + collapseCompactedHistory: settings.get("display.collapseCompacted"), + }); this.renderSessionContext(context); for (const child of liveComponents) { this.chatContainer.addChild(child); From 585b9e437e2ea7b0018c7e37920bfbbc6d1e2549 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:29:55 +0200 Subject: [PATCH 49/80] ux(mode): enabled user control over compacted transcript session collapse - Updated transcript session context to use display.collapseCompacted setting instead of a hardcoded boolean. - Enabled user-defined control over whether compacted history is collapsed during live rendering sessions. --- packages/coding-agent/CHANGELOG.md | 2 +- packages/coding-agent/src/modes/utils/ui-helpers.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 2f95527a6..38546bed5 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- Added display setting to toggle between collapsing or keeping compacted history inline +- Added display setting to toggle between collapsing or keeping compacted history inline, now applied to live session displays - Added a compact session-only model picker (Alt+P) for quick model switching without changing roles ### Changed diff --git a/packages/coding-agent/src/modes/utils/ui-helpers.ts b/packages/coding-agent/src/modes/utils/ui-helpers.ts index 8a5e5776f..384a5036f 100644 --- a/packages/coding-agent/src/modes/utils/ui-helpers.ts +++ b/packages/coding-agent/src/modes/utils/ui-helpers.ts @@ -607,13 +607,14 @@ export class UiHelpers { this.ctx.pendingBashComponents = []; this.ctx.pendingPythonComponents = []; - // Live display uses the compacted transcript tail; export/resume callers - // can still request the full inline compaction history. Mid-turn rebuilds + // Live display collapses to the compacted transcript tail unless the + // user opted into the full inline history; export/resume callers can + // still request either mode. Mid-turn rebuilds // (focus attach/unfocus while a tool executes) keep dangling toolCalls so // the in-flight call re-renders as pending instead of vanishing; // renderSessionContext then keeps it in `pendingTools` for live routing. const context = this.ctx.viewSession.buildTranscriptSessionContext({ - collapseCompactedHistory: true, + collapseCompactedHistory: settings.get("display.collapseCompacted"), keepDanglingToolCalls: this.ctx.viewSession.isStreaming, }); this.ctx.renderSessionContext(context, { From 711fa43123c40cd6032dd210fc2041a6605cfb01 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:35:40 +0200 Subject: [PATCH 50/80] feat(snapcompact): switched to compact scope markers for archived transcripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replaced markdown-style headings with concise `¶user:`, `¶think:`, `¶ai:`, and `¶call:` scope markers. - Updated the serializer to merge consecutive messages or blocks of the same type under a shared prefix. - Updated the documentation prompt to reflect the new compact formatting and scope rules. - Added regression tests verifying scope merging and correct formatting of tool calls and intents. --- .../snapcompact-shape-preview-doc.md | 18 +++--- packages/snapcompact/CHANGELOG.md | 4 ++ .../src/prompts/snapcompact-summary.md | 7 ++- packages/snapcompact/src/snapcompact.ts | 59 ++++++++++--------- packages/snapcompact/test/snapcompact.test.ts | 51 +++++++++++----- 5 files changed, 82 insertions(+), 57 deletions(-) diff --git a/packages/coding-agent/src/modes/components/snapcompact-shape-preview-doc.md b/packages/coding-agent/src/modes/components/snapcompact-shape-preview-doc.md index 0fd41c895..cb68b772b 100644 --- a/packages/coding-agent/src/modes/components/snapcompact-shape-preview-doc.md +++ b/packages/coding-agent/src/modes/components/snapcompact-shape-preview-doc.md @@ -1,18 +1,14 @@ -# User ¶ -Fix the settings overlay crash. Wheeling past the last row throws. +¶user:Fix the settings overlay crash. Wheeling past the last row throws. -# Tool call ¶ -//Reading the select-list hit test -read(path="src/select-list.ts:140-180") +¶call:read(path="src/select-list.ts:140-180")//Reading the select-list hit test 162: const index = Math.floor(line / rowHeight); index is never checked against bounds. -# Assistant ¶ -Found it. The hit test indexes past the filtered list; clamping to the last row fixes the crash. +¶ai:Found it. The hit test indexes past the filtered list; clamping to the last row fixes the crash. -# User ¶ -Does the fix survive filtering? +¶user:Does the fix survive filtering? -# Assistant ¶ -Yes. The clamp applies after the filter pass, so a narrowed list keeps the hit map in sync. Added a regression test that wheels past the last row with a filter active and asserts no throw. +¶think:Check whether the clamp runs before or after filtering. + +¶ai:Yes. The clamp applies after the filter pass, so a narrowed list keeps the hit map in sync. Added a regression test that wheels past the last row with a filter active and asserts no throw. diff --git a/packages/snapcompact/CHANGELOG.md b/packages/snapcompact/CHANGELOG.md index cd4dbad80..d020535e9 100644 --- a/packages/snapcompact/CHANGELOG.md +++ b/packages/snapcompact/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- Changed archived transcript rendering to compact `¶user:`, `¶think:`, `¶ai:`, and `¶call:` scopes; repeated adjacent scopes now continue as plain lines, tool-call intents trail calls as `//` comments, and the compaction prompt documents the format. + ## [16.3.7] - 2026-07-05 ### Fixed diff --git a/packages/snapcompact/src/prompts/snapcompact-summary.md b/packages/snapcompact/src/prompts/snapcompact-summary.md index 286ee0c94..6343f94e8 100644 --- a/packages/snapcompact/src/prompts/snapcompact-summary.md +++ b/packages/snapcompact/src/prompts/snapcompact-summary.md @@ -1,6 +1,9 @@ You are resuming a prior conversation. Its earlier turns were archived to reclaim context and are reproduced under HISTORY below, oldest to newest. Read HISTORY in full, then continue from the live conversation that follows it. -The archived transcript is compact: each turn opens with a heading — `# User ¶`, `# Assistant ¶`, or `# Tool call ¶` — assistant reasoning is wrapped in _italics_, and tool output sits inside `…`. +The archived transcript uses compact scopes: +- `¶user:`, `¶think:`, `¶ai:`, and `¶call:` open user, assistant reasoning, assistant reply, and tool-call scopes. +- Following lines without a `¶…:` prefix remain in the current scope. Consecutive blocks of the same kind omit the repeated prefix. +- A tool call reads `¶call:name(args)//intent`; the trailing `//intent` is optional. Its `…` block is tool output. Reading HISTORY: - Plain-text sections are the verbatim transcript — rely on them exactly. @@ -9,9 +12,7 @@ Reading HISTORY: {{else}} - A frame is one grid {{cols}} characters wide and up to {{rows}} rows tall: read left to right, top to bottom — there is no word wrap, so a word may break across rows. {{/if}}{{#if sentenceInk}} - Ink cycles through six colors, one per sentence. {{/if}}{{#if stopwordDimmed}} - Function words are dim gray; content words keep full ink. -{{/if}}{{#if dimmedToolResults}} - Text inside `` is dim gray — that gray is archived tool output, not conversation. {{/if}}{{#if lineRepeated}} - Each line is printed twice (white, then a pale-yellow band); the two copies are identical. -{{/if}}{{#if mixedShapes}} - The compressed middle frames use a smaller, denser font than the edge frames; the reading order is unchanged. {{/if}}{{/if}}{{#if includedPreviousSummary}}- HISTORY opens with a condensed digest of still-older context that predates the archived turns. {{/if}}{{#if truncatedChars}}- About {{truncatedChars}} characters of older middle history were dropped to fit the archive budget. {{/if}}- When an exact earlier detail matters and a section reads unclearly, re-derive it from the workspace (re-read files, re-run commands) rather than guessing. diff --git a/packages/snapcompact/src/snapcompact.ts b/packages/snapcompact/src/snapcompact.ts index 4793cb262..443fe93f3 100644 --- a/packages/snapcompact/src/snapcompact.ts +++ b/packages/snapcompact/src/snapcompact.ts @@ -638,8 +638,6 @@ export function createFileOps(): FileOperations { } const URL_SCHEME_RE = /[a-z][a-z0-9+.-]*:\/\//i; -const HEADING_MARKER = " ¶"; - export function isUrlSchemePath(path: string): boolean { return URL_SCHEME_RE.test(path); } @@ -776,10 +774,22 @@ export function serializeConversation(messages: Message[], options?: SerializeOp const headRatio = options?.truncateHeadRatio ?? TRUNCATE_HEAD_RATIO; const dimToolResults = options?.dimToolResults !== false; const parts: string[] = []; + let lastPrefix: string | null = null; + + const pushPart = (prefix: string, content: string) => { + const lastIndex = parts.length - 1; + if (lastIndex >= 0 && lastPrefix === prefix) { + const sep = parts[lastIndex].endsWith("\n") || content.startsWith("\n") ? "" : "\n"; + parts[lastIndex] += sep + content; + } else { + parts.push(prefix + content); + lastPrefix = prefix; + } + }; // Tool results flagged contextually useless (and their paired calls) carry no // information worth archiving — skip the whole pair. Surviving results are - // indexed by tool-call id so each merges into its originating `# Tool call`. + // indexed by tool-call id so each merges into its originating `¶call:` scope. const uselessCallIds = new Set(); const resultTextByCallId = new Map(); for (const msg of messages) { @@ -796,7 +806,7 @@ export function serializeConversation(messages: Message[], options?: SerializeOp } // Wrap a raw tool-result body in an `` block, dimming only the body so - // the frame coloring keeps structure (headings, calls) loud. + // the frame coloring keeps scope markers and calls loud. const renderResultBlock = (rawText: string): string => { const body = truncateForSummary(stripDimMarkers(rawText), toolResultMaxChars, headRatio); return `\n${dimToolResults ? `${DIM_ON}${body}${DIM_OFF}` : body}\n`; @@ -813,18 +823,19 @@ export function serializeConversation(messages: Message[], options?: SerializeOp .filter((content): content is { type: "text"; text: string } => content.type === "text") .map(content => content.text) .join(""); - if (content) parts.push(`# User${HEADING_MARKER}\n${stripDimMarkers(content)}`); + if (content) pushPart("¶user:", stripDimMarkers(content)); } else if (msg.role === "assistant") { // Stream blocks in content order: buffer thinking/text, then flush a - // `# Assistant` block (thinking as italics above the text) right before - // each tool call, so text or thinking after a call stays after it. + // separate section for each block type right before each tool call. let pendingThinking: string[] = []; let pendingText: string[] = []; const flushAssistant = () => { - const sections: string[] = []; - if (pendingThinking.length > 0) sections.push(`_${pendingThinking.join("\n")}_`); - if (pendingText.length > 0) sections.push(pendingText.join("\n")); - if (sections.length > 0) parts.push(`# Assistant${HEADING_MARKER}\n${sections.join("\n\n")}`); + if (pendingThinking.length > 0) { + pushPart("¶think:", pendingThinking.join("\n")); + } + if (pendingText.length > 0) { + pushPart("¶ai:", pendingText.join("\n")); + } pendingThinking = []; pendingText = []; }; @@ -860,24 +871,27 @@ export function serializeConversation(messages: Message[], options?: SerializeOp toolCallMaxChars, headRatio, ); - const lines = [`# Tool call${HEADING_MARKER}`]; - if (intent) lines.push(`//${intent}`); - lines.push(`${block.name}(${argsStr})`); + const lines: string[] = []; + let firstLine = `${block.name}(${argsStr})`; + if (intent) { + firstLine += `//${intent}`; + } + lines.push(firstLine); const resultText = resultTextByCallId.get(block.id); if (resultText !== undefined) { mergedCallIds.add(block.id); lines.push(renderResultBlock(resultText)); } - parts.push(lines.join("\n")); + pushPart("¶call:", lines.join("\n")); } } flushAssistant(); } else if (msg.role === "toolResult") { - // Paired results already merged into their `# Tool call` block above; + // Paired results already merged into their tool call block above; // only orphans (call archived outside this window) render standalone. if (uselessCallIds.has(msg.toolCallId) || mergedCallIds.has(msg.toolCallId)) continue; const resultText = resultTextByCallId.get(msg.toolCallId); - if (resultText !== undefined) parts.push(`# Tool call${HEADING_MARKER}\n${renderResultBlock(resultText)}`); + if (resultText !== undefined) pushPart("¶call:", `\n${renderResultBlock(resultText)}`); } } @@ -1911,15 +1925,6 @@ export async function compact( const frames = await Promise.all(newFrames); const totalChars = frames.reduce((sum, frame) => sum + frame.chars, 0) + textChars; - const mixedShapes = frames.some( - frame => - frame.cols !== geo.cols || - frame.rows !== geo.rows || - (frame.variant ?? "sent") !== high.variant || - (frame.lineRepeat ?? 1) !== high.lineRepeat || - (frame.columns ?? 1) !== (high.columns ?? 1) || - (frame.stopwordDim ?? false) !== (high.stopwordDim ?? false), - ); const { readFiles, modifiedFiles } = computeFileLists(fileOps); const files = formatFileList(readFiles, modifiedFiles, fileOps.read); @@ -1936,9 +1941,7 @@ export async function compact( rows: geo.rows, sentenceInk: high.variant === "sent", stopwordDimmed: high.stopwordDim === true, - dimmedToolResults: options?.dimToolResults !== false, lineRepeated: high.lineRepeat > 1, - mixedShapes, truncatedChars, includedPreviousSummary, files: files.length > 0 ? files : undefined, diff --git a/packages/snapcompact/test/snapcompact.test.ts b/packages/snapcompact/test/snapcompact.test.ts index a11fd4ed3..e49dcb573 100644 --- a/packages/snapcompact/test/snapcompact.test.ts +++ b/packages/snapcompact/test/snapcompact.test.ts @@ -664,12 +664,12 @@ describe("serializeConversation", () => { expect(out.length).toBeLessThan(2200); }); - it("renders roles as markdown headings", () => { + it("renders roles with compact inline headings", () => { const out = snapcompact.serializeConversation([ createUserMessage("do the thing"), createAssistantMessage([{ type: "text", text: "done" }]), ]); - expect(out).toBe("# User ¶\ndo the thing\n\n# Assistant ¶\ndone"); + expect(out).toBe("¶user:do the thing\n\n¶ai:done"); }); it("merges a tool call with its paired result into one block, intent as a // comment", () => { @@ -687,7 +687,7 @@ describe("serializeConversation", () => { ], { dimToolResults: false }, ); - expect(out).toBe('# Tool call ¶\n//Running tests\nbash(command="bun test")\n\n3 pass\n'); + expect(out).toBe('¶call:bash(command="bun test")//Running tests\n\n3 pass\n'); }); it("prefers the harness-derived intent over the raw intent arg and squashes newlines", () => { @@ -707,29 +707,29 @@ describe("serializeConversation", () => { expect(out).not.toContain(`${INTENT_FIELD}=`); }); - it("folds thinking into the assistant block as italics above the text", () => { + it("folds thinking into separate sections above the text", () => { const out = snapcompact.serializeConversation([ createAssistantMessage([ { type: "thinking", thinking: "weigh options" }, { type: "text", text: "the answer" }, ]), ]); - expect(out).toBe("# Assistant ¶\n_weigh options_\n\nthe answer"); + expect(out).toBe("¶think:weigh options\n\n¶ai:the answer"); }); - it("gives a thinking-only turn its own assistant heading before the tool calls", () => { + it("gives a thinking-only turn its own heading before the tool calls", () => { const out = snapcompact.serializeConversation([ createAssistantMessage([ { type: "thinking", thinking: "plan first" }, { type: "toolCall", id: "c1", name: "read", arguments: { path: "a.ts" } }, ]), ]); - expect(out).toBe('# Assistant ¶\n_plan first_\n\n# Tool call ¶\nread(path="a.ts")'); + expect(out).toBe('¶think:plan first\n\n¶call:read(path="a.ts")'); }); it("renders an orphan tool result (call outside the window) standalone", () => { const out = snapcompact.serializeConversation([createToolResultMessage("ok")], { dimToolResults: false }); - expect(out).toBe("# Tool call ¶\n\nok\n"); + expect(out).toBe("¶call:\n\nok\n"); }); it("preserves content order: text before and after a tool call stay split around it", () => { @@ -744,9 +744,7 @@ describe("serializeConversation", () => { ], { dimToolResults: false }, ); - expect(out).toBe( - '# Assistant ¶\nbefore\n\n# Tool call ¶\nread(path="a.ts")\n\nfile body\n\n\n# Assistant ¶\nafter', - ); + expect(out).toBe('¶ai:before\n\n¶call:read(path="a.ts")\n\nfile body\n\n\n¶ai:after'); }); it("does not split assistant prose around a useless tool call", () => { @@ -759,7 +757,7 @@ describe("serializeConversation", () => { { ...createToolResultMessage("No matches found"), toolCallId: "c-drop", useless: true } as Message, ]); // The useless call vanishes and its surrounding prose stays in one block. - expect(out).toBe("# Assistant ¶\nbefore\nafter"); + expect(out).toBe("¶ai:before\nafter"); }); it("drops blank text/thinking blocks instead of emitting an empty assistant heading", () => { @@ -774,8 +772,8 @@ describe("serializeConversation", () => { ], { dimToolResults: false }, ); - expect(out).toBe('# Tool call ¶\nread(path="a.ts")\n\nbody\n'); - expect(out).not.toContain("# Assistant ¶"); + expect(out).toBe('¶call:read(path="a.ts")\n\nbody\n'); + expect(out).not.toContain("¶ai:"); }); it("wraps tool-result bodies in dim toggles by default and strips stray toggles from content", () => { @@ -785,7 +783,7 @@ describe("serializeConversation", () => { ]); expect(out).toContain(`\n${snapcompact.DIM_ON}ok${snapcompact.DIM_OFF}\n`); // A stray toggle in user content cannot forge a dim span. - expect(out).toContain("# User ¶\nhello world"); + expect(out).toContain("¶user:hello world"); }); it("skips tool call/result pairs flagged useless", () => { @@ -802,6 +800,26 @@ describe("serializeConversation", () => { expect(out).not.toContain("zzz_nothing"); expect(out).not.toContain("No matches found"); }); + + it("merges consecutive blocks of the same role", () => { + const out = snapcompact.serializeConversation([ + createUserMessage("hello"), + createUserMessage("world"), + createAssistantMessage([{ type: "text", text: "hi" }]), + createAssistantMessage([{ type: "text", text: "there" }]), + ]); + expect(out).toBe("¶user:hello\nworld\n\n¶ai:hi\nthere"); + }); + + it("merges consecutive tool calls under a single prefix", () => { + const out = snapcompact.serializeConversation([ + createAssistantMessage([ + { type: "toolCall", id: "c1", name: "read", arguments: { path: "a.ts" } }, + { type: "toolCall", id: "c2", name: "read", arguments: { path: "b.ts" } }, + ]), + ]); + expect(out).toBe('¶call:read(path="a.ts")\nread(path="b.ts")'); + }); }); describe("compact", () => { @@ -815,6 +833,9 @@ describe("compact", () => { expect(result.tokensBefore).toBe(99000); expect(result.summary).toContain("You are resuming a prior conversation."); expect(result.summary).toContain("HISTORY"); + expect(result.summary).toContain("`¶user:`, `¶think:`, `¶ai:`, and `¶call:`"); + expect(result.summary).toContain("Following lines without a `¶…:` prefix remain in the current scope."); + expect(result.summary).toContain("`¶call:name(args)//intent`"); expect(result.summary).toContain("FILES\n===================\n# src/\nauth.ts (Read)\nlogin.ts (Write)"); const archive = snapcompact.getPreservedArchive(result.preserveData); From 4903a13511d03207ebf9dc37b3ca5c7116d3ebe0 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:40:14 +0200 Subject: [PATCH 51/80] feat(session): implemented automated recovery and status reporting - Added `warning` field to `CompactionEntry` and `CompactionSummaryMessage` to persist dead-end status in session history. - Introduced a multi-tier rescue mechanism in `AgentSession` that automatically performs `elide` and `dropImages` passes when maintenance fails to recover sufficient headroom. - Integrated visual indicators for dead-ends into the `CompactionSummaryMessageComponent`, surfacing warnings directly on the compaction divider and detail block. - Updated `AgentSession` logic to re-evaluate progress after each rescue tier and emit recovery notices, ensuring transparent reporting of automated history rewrites. --- packages/agent/CHANGELOG.md | 9 + packages/agent/src/compaction/entries.ts | 6 + packages/agent/src/compaction/messages.ts | 4 + packages/coding-agent/CHANGELOG.md | 3 +- .../components/compaction-summary-message.ts | 10 +- .../coding-agent/src/session/agent-session.ts | 200 +++++++++++------- .../src/session/session-entries.ts | 6 + 7 files changed, 156 insertions(+), 82 deletions(-) diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 32b89991a..ba7b65aec 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,6 +2,15 @@ ## [Unreleased] +### Added + +- Added automated image-dropping rescue tier to compaction dead-end recovery +- Added visual warnings to the session timeline when compaction fails to free sufficient space + +### Changed + +- Improved compaction dead-end notifications with specific recovery instructions + ## [16.4.5] - 2026-07-11 ### Added diff --git a/packages/agent/src/compaction/entries.ts b/packages/agent/src/compaction/entries.ts index f06299990..a3735805b 100644 --- a/packages/agent/src/compaction/entries.ts +++ b/packages/agent/src/compaction/entries.ts @@ -43,6 +43,12 @@ export interface CompactionEntry extends SessionEntryBase { preserveData?: Record; /** True if generated by an extension, undefined/false if pi-generated (backward compatible) */ fromExtension?: boolean; + /** + * Dead-end warning from the post-pass progress guard: the pass completed + * but freed too little for maintenance to continue. Rendered on the + * compaction divider. + */ + warning?: string; } export interface BranchSummaryEntry extends SessionEntryBase { diff --git a/packages/agent/src/compaction/messages.ts b/packages/agent/src/compaction/messages.ts index c9aa3f282..72474a2bd 100644 --- a/packages/agent/src/compaction/messages.ts +++ b/packages/agent/src/compaction/messages.ts @@ -56,6 +56,8 @@ export interface CompactionSummaryMessage { blocks?: (TextContent | ImageContent)[]; /** Snapcompact image blocks, kept for display counts / legacy consumers. */ images?: ImageContent[]; + /** Post-pass dead-end warning attached to this compaction (progress guard). */ + warning?: string; timestamp: number; } @@ -105,6 +107,7 @@ export function createCompactionSummaryMessage( providerPayload?: ProviderPayload, images?: ImageContent[], blocks?: (TextContent | ImageContent)[], + warning?: string, ): CompactionSummaryMessage { const imageBlocks = blocks?.filter((block): block is ImageContent => block.type === "image") ?? @@ -117,6 +120,7 @@ export function createCompactionSummaryMessage( providerPayload, blocks: blocks && blocks.length > 0 ? blocks : undefined, images: imageBlocks && imageBlocks.length > 0 ? imageBlocks : undefined, + warning, timestamp: new Date(timestamp).getTime(), }; } diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 38546bed5..1e0ad60e8 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -10,6 +10,7 @@ ### Changed - Updated status event log to prioritize the most recent entries in the display window +- Updated the snapcompact shape preview transcript to use the compact scope format shown to models during compaction. ### Removed @@ -20,7 +21,7 @@ - Fixed inconsistent history rendering when toggling the display setting for compacted items - Fixed configured `retry.fallbackChains` never engaging on non-retryable provider errors (e.g. "Cloud Code Assist API returned an empty response"): a hard error on a model covered by a fallback chain now switches to the next candidate instead of failing the turn, while still never backoff-retrying the failing model itself - Fixed transcript rebuilds (compaction, `/compact`, and toggling history display) repainting content below stale scrollback when collapsing history; rebuilds now correctly clear the scrollback buffer when history is collapsed -- Fixed mid-run auto-compaction spuriously warning "Compaction freed too little context to make progress" and pausing maintenance even when compaction genuinely shrank the context (observed: snapcompact took a 312k-token gpt-5.6 session to 86k real tokens and still dead-ended): the in-flight prompt's pending context snapshot — set once at run start and alive for the whole tool loop — was read as live residual context by the post-compaction headroom/retry-fit checks because the fresh compaction entry hides every earlier usage anchor; compaction (auto and `/compact`), the retry drop, and the dead-end shake rescue now rebase the snapshot onto the rewritten message set +- Improved auto-compaction to automatically drop images and elide content when context is tight, and added persistent warning badges to the compaction divider when manual intervention is required - Fixed backgrounded Bash blocks continuing to repaint with live and final job output; they now freeze with a compact job notice while completion is delivered separately ## [16.4.8] - 2026-07-12 diff --git a/packages/coding-agent/src/modes/components/compaction-summary-message.ts b/packages/coding-agent/src/modes/components/compaction-summary-message.ts index 04aeb9ece..f5cb45590 100644 --- a/packages/coding-agent/src/modes/components/compaction-summary-message.ts +++ b/packages/coding-agent/src/modes/components/compaction-summary-message.ts @@ -87,7 +87,12 @@ export class CompactionSummaryMessageComponent implements Component { constructor(private readonly message: CompactionSummaryMessage) { this.#divider = new SummaryDividerComponent({ - label: () => `${theme.icon.camera} compacted`, + // A dead-end warning stamped by the progress guard badges the bar; + // the full text lives in the ctrl+o detail block below. + label: () => + this.message.warning + ? `${theme.icon.camera} compacted ${theme.fg("warning", theme.icon.warning)}` + : `${theme.icon.camera} compacted`, detailMarkdown: () => this.#detailMarkdown(), }); } @@ -109,7 +114,8 @@ export class CompactionSummaryMessageComponent implements Component { const frameCount = this.message.images?.length ?? 0; const frameNote = frameCount > 0 ? `\n\n_${frameCount} snapcompact frame${frameCount === 1 ? "" : "s"} attached_` : ""; - return `**Compacted from ${tokenStr} tokens**\n\n${this.message.summary}${frameNote}`; + const warningNote = this.message.warning ? `\n\n${theme.icon.warning} **Warning:** ${this.message.warning}` : ""; + return `**Compacted from ${tokenStr} tokens**${warningNote}\n\n${this.message.summary}${frameNote}`; } } diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 389bfcc04..f800310a6 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -12510,53 +12510,85 @@ export class AgentSession { } /** - * Last-resort reducer when {@link #runAutoCompaction} would otherwise dead-end. - * The summarizer cut at the only available turn boundary, but the kept tail is - * still over the recovery band because a single recent turn (a large - * tool-result, a heavy fenced/XML block) is itself bigger than the band and - * `findCutPoint` cannot cut inside one message. `shake("elide")` reaches INSIDE - * that tail — it offloads heavy tool-result / block content to one - * `artifact://` blob and leaves a recoverable placeholder — so residual context - * genuinely drops instead of the guard pausing maintenance and looping the - * warning. Without it the guard would pause/warn here; with it the caller - * re-tests its progress predicate after the elide pass and only falls through - * to the warning when residual stays over. + * Last-resort tiered reducer when {@link #runAutoCompaction} would otherwise + * dead-end. The summarizer cut at the only available turn boundary, but the + * kept tail is still over the recovery band because a single recent turn (a + * large tool-result, a heavy fenced/XML block, attached images) is itself + * bigger than the band and `findCutPoint` cannot cut inside one message. * - * Image-only tails are out of scope: `collectShakeRegions` skips image-only - * tool results and user-message images aren't counted by the local estimate - * that gates the dead-end, so those still surface the warning (remedy: - * `/shake images`). + * Tier 1 — `shake("elide")` reaches INSIDE that tail: heavy tool-result / + * block content is offloaded to one `artifact://` blob behind a recoverable + * placeholder. Skipped when this pass already ran a shake (`skipElide`). + * Tier 2 — `dropImages()`: the manual `/shake images` remedy, automated. + * Image blocks are stripped from the branch; unlike elided text they are NOT + * artifact-recoverable, so this tier only runs once elide has failed the + * progress re-test. * - * Returns the elide {@link ShakeResult} when something was offloaded (so the - * caller can re-test and report), or `undefined` when nothing was eligible or - * the pass aborted/failed. + * Each tier that rewrote history re-anchors the in-flight context snapshot, + * then the caller's progress predicate is re-tested; the first tier that + * restores progress emits one info notice describing everything freed and + * stops. Returns whether progress was restored — `false` falls through to + * the dead-end warning. */ - async #tryShakeRescueForDeadEnd(signal: AbortSignal): Promise { - if (signal.aborted) return undefined; + async #rescueCompactionDeadEnd( + signal: AbortSignal, + options: { skipElide: boolean; hasProgress: () => boolean }, + ): Promise { + if (signal.aborted) return false; + let elided = 0; + let elidedTokens = 0; + let elideSink = "placeholders"; + if (!options.skipElide) { + try { + const result = await this.shake("elide", { signal }); + elided = result.toolResultsDropped + result.blocksDropped; + elidedTokens = result.tokensFreed; + if (result.artifactId) elideSink = "an artifact"; + if (elided > 0) { + // The elide pass rewrote history; re-anchor the in-flight snapshot + // so the caller's headroom/retry-fit re-test measures the shaken + // context. + this.#rebasePendingContextSnapshotAfterCompaction(); + } + } catch (error) { + logger.warn("Dead-end shake rescue failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + if (elided > 0 && options.hasProgress()) { + this.emitNotice( + "info", + `Compaction dead-end recovery: ${this.#describeElideRescue(elided, elidedTokens, elideSink)} so maintenance could make progress.`, + "compaction", + ); + return true; + } + } + if (signal.aborted) return false; + let imagesDropped = 0; try { - const result = await this.shake("elide", { signal }); - if (result.toolResultsDropped + result.blocksDropped === 0) return undefined; - // The elide pass rewrote history; re-anchor the in-flight snapshot so - // the caller's headroom/retry-fit re-test measures the shaken context. - this.#rebasePendingContextSnapshotAfterCompaction(); - return result; + imagesDropped = (await this.dropImages()).removed; + if (imagesDropped > 0) this.#rebasePendingContextSnapshotAfterCompaction(); } catch (error) { - logger.warn("Dead-end shake rescue failed", { + logger.warn("Dead-end image-drop rescue failed", { error: error instanceof Error ? error.message : String(error), }); - return undefined; } + if (imagesDropped > 0 && options.hasProgress()) { + const elidedPart = elided > 0 ? `${this.#describeElideRescue(elided, elidedTokens, elideSink)} and ` : ""; + this.emitNotice( + "info", + `Compaction dead-end recovery: ${elidedPart}dropped ${imagesDropped} attached image${imagesDropped === 1 ? "" : "s"} so maintenance could make progress.`, + "compaction", + ); + return true; + } + return false; } - /** Notice describing a successful dead-end elide rescue. */ - #emitShakeRescueNotice(result: ShakeResult): void { - const elided = result.toolResultsDropped + result.blocksDropped; - const sink = result.artifactId ? "an artifact" : "placeholders"; - this.emitNotice( - "info", - `Compaction dead-end recovery: elided ${elided} heavy block${elided === 1 ? "" : "s"} (~${result.tokensFreed.toLocaleString()} tokens) to ${sink} so maintenance could make progress.`, - "compaction", - ); + /** Notice fragment for a dead-end elide tier: what was freed and where it went. */ + #describeElideRescue(elided: number, tokensFreed: number, sink: string): string { + return `elided ${elided} heavy block${elided === 1 ? "" : "s"} (~${tokensFreed.toLocaleString()} tokens) to ${sink}`; } /** @@ -13098,23 +13130,28 @@ export class AgentSession { details, preserveData, }; - await this.#emitSessionEvent({ type: "auto_compaction_end", action, result, aborted: false, willRetry }); - - // Post-maintenance progress guard. Snapcompact can project over budget and - // fall back to a context-full summary; the summarizer keeps `keepRecentTokens` - // of recent history verbatim and findCutPoint can only cut at turn - // boundaries (never tool results), so a single oversized recent turn (e.g. a - // huge tool result) leaves the rewritten context still above threshold. - // Scheduling the continuation regardless means the next agent_end re-enters - // #checkCompaction over the same oversized tail and re-fires forever. The - // retry and the threshold auto-continue use different progress tests (a - // recoverable overflow only has to fit; the auto-continue thrash needs the - // stricter recovery band), so each branch evaluates its own below. + // Post-maintenance progress guard — evaluated BEFORE emitting + // auto_compaction_end so the TUI rebuild triggered by that event + // already reflects any rescue rewrite (elide / image-drop) and the + // dead-end warning stamped on the compaction entry. Snapcompact can + // project over budget and fall back to a context-full summary; the + // summarizer keeps `keepRecentTokens` of recent history verbatim and + // findCutPoint can only cut at turn boundaries (never tool results), + // so a single oversized recent turn (e.g. a huge tool result) leaves + // the rewritten context still above threshold. Scheduling the + // continuation regardless means the next agent_end re-enters + // #checkCompaction over the same oversized tail and re-fires forever. + // The retry and the threshold auto-continue use different progress + // tests (a recoverable overflow only has to fit; the auto-continue + // thrash needs the stricter recovery band), so each branch evaluates + // its own below. let continuationScheduled = false; // A non-idle pass that wanted to continue (retry or auto-continue) but freed // too little for that path to proceed is a dead-end: warn once so the user // understands why maintenance paused instead of silently looping. let noProgressDeadEnd = false; + let retryFits = false; + let hasHeadroom = false; if (willRetry) { const messages = this.agent.state.messages; @@ -13139,18 +13176,14 @@ export class AgentSession { // won't include) is excluded. Reusing the auto-continue recovery band // here turned recoverable overflows into manual dead-ends (#3412 review), // so use the looser fit budget. - let retryFits = this.#compactionCreatedRetryFit(); - if (!retryFits && !fallbackFromShake) { - const rescue = await this.#tryShakeRescueForDeadEnd(autoCompactionSignal); - if (rescue && this.#compactionCreatedRetryFit()) { - retryFits = true; - this.#emitShakeRescueNotice(rescue); - } + retryFits = this.#compactionCreatedRetryFit(); + if (!retryFits) { + retryFits = await this.#rescueCompactionDeadEnd(autoCompactionSignal, { + skipElide: fallbackFromShake, + hasProgress: () => this.#compactionCreatedRetryFit(), + }); } - if (retryFits) { - this.#scheduleAgentContinue({ delayMs: 100, generation }); - continuationScheduled = true; - } else { + if (!retryFits) { noProgressDeadEnd = true; } } else if (reason !== "idle") { @@ -13161,23 +13194,36 @@ export class AgentSession { // when auto-continue is disabled, a no-headroom threshold pass must still // block later automatic continuations (todo reminders/session_stop hooks) // from re-entering the same oversized context. - let hasHeadroom = this.#compactionCreatedHeadroom(); - if (!hasHeadroom && !fallbackFromShake) { - const rescue = await this.#tryShakeRescueForDeadEnd(autoCompactionSignal); - if (rescue && this.#compactionCreatedHeadroom()) { - hasHeadroom = true; - this.#emitShakeRescueNotice(rescue); - } + hasHeadroom = this.#compactionCreatedHeadroom(); + if (!hasHeadroom) { + hasHeadroom = await this.#rescueCompactionDeadEnd(autoCompactionSignal, { + skipElide: fallbackFromShake, + hasProgress: () => this.#compactionCreatedHeadroom(), + }); } - if (hasHeadroom) { - if (shouldAutoContinue) { - this.#scheduleAutoContinuePrompt(generation); - continuationScheduled = true; - } - } else { + if (!hasHeadroom) { noProgressDeadEnd = true; } } + + const deadEndWarning = noProgressDeadEnd ? compactionDeadEndWarning("clear large tool output") : undefined; + if (deadEndWarning && savedCompactionEntry) { + // Stamp the divider: the compaction bar badges the dead-end and + // carries the full warning in its ctrl+o detail, so the pause + // stays explained even after the notice row scrolls away. + savedCompactionEntry.warning = deadEndWarning; + await this.sessionManager.rewriteEntries(); + } + + await this.#emitSessionEvent({ type: "auto_compaction_end", action, result, aborted: false, willRetry }); + + if (retryFits) { + this.#scheduleAgentContinue({ delayMs: 100, generation }); + continuationScheduled = true; + } else if (hasHeadroom && shouldAutoContinue) { + this.#scheduleAutoContinuePrompt(generation); + continuationScheduled = true; + } if (!continuationScheduled && !suppressContinuation && this.agent.hasQueuedMessages()) { // Auto-compaction can complete while follow-up/steering/custom messages are waiting. // Kick the loop so queued messages are actually delivered. This remains separate @@ -13190,12 +13236,8 @@ export class AgentSession { continuationScheduled = true; } - if (noProgressDeadEnd) { - this.emitNotice( - "warning", - compactionDeadEndWarning("clear large tool output, run `/shake images` to drop attached images,"), - "compaction", - ); + if (deadEndWarning) { + this.emitNotice("warning", deadEndWarning, "compaction"); } if (continuationScheduled) return COMPACTION_CHECK_CONTINUATION; return noProgressDeadEnd ? COMPACTION_CHECK_BLOCK_AUTOMATIC_CONTINUATION : COMPACTION_CHECK_NONE; diff --git a/packages/coding-agent/src/session/session-entries.ts b/packages/coding-agent/src/session/session-entries.ts index 59ed3e506..7dc763a95 100644 --- a/packages/coding-agent/src/session/session-entries.ts +++ b/packages/coding-agent/src/session/session-entries.ts @@ -92,6 +92,12 @@ export interface CompactionEntry extends SessionEntryBase { preserveData?: Record; /** True if generated by an extension, undefined/false if pi-generated (backward compatible) */ fromExtension?: boolean; + /** + * Dead-end warning from the post-pass progress guard: the pass completed + * but freed too little for maintenance to continue. Rendered on the + * compaction divider. + */ + warning?: string; } export interface BranchSummaryEntry extends SessionEntryBase { From aa52fa42314c53c7e8c8b51888c547098b2ea37b Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:40:15 +0200 Subject: [PATCH 52/80] feat(session): enabled visibility for elided tool calls in transcripts - Added `StrippedToolCallsMarker` to identify assistant messages with dangling tool calls that were removed during context building. - Updated `buildSessionContext` to preserve turn entries in transcripts even when all content is stripped, marking them with the count of elided calls. - Updated `UiHelpers` to render a dim, italicized placeholder in the TUI when an assistant message has elided tool calls, preventing silent activity gaps. --- packages/coding-agent/CHANGELOG.md | 1 + .../src/modes/utils/ui-helpers.ts | 22 +++++++++++- .../src/session/session-context.ts | 35 ++++++++++++++----- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 1e0ad60e8..23e6cc216 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -9,6 +9,7 @@ ### Changed +- Added visual markers in the transcript for elided tool calls that have no corresponding result - Updated status event log to prioritize the most recent entries in the display window - Updated the snapcompact shape preview transcript to use the compact scope format shown to models during compaction. diff --git a/packages/coding-agent/src/modes/utils/ui-helpers.ts b/packages/coding-agent/src/modes/utils/ui-helpers.ts index 384a5036f..70af14750 100644 --- a/packages/coding-agent/src/modes/utils/ui-helpers.ts +++ b/packages/coding-agent/src/modes/utils/ui-helpers.ts @@ -45,7 +45,7 @@ import { SKILL_PROMPT_MESSAGE_TYPE, type SkillPromptDetails, } from "../../session/messages"; -import type { SessionContext } from "../../session/session-context"; +import type { SessionContext, StrippedToolCallsMarker } from "../../session/session-context"; import { replaceTabs } from "../../tools/render-utils"; import { buildSkillCommandPrompt, invokeSkillCommandFromText, isKnownSkillCommand } from "../skill-command"; import { createAssistantMessageComponent } from "./interactive-context-helpers"; @@ -465,6 +465,26 @@ export class UiHelpers { this.ctx.pendingTools.set(content.id, component); } } + // Dangling toolCalls (no result on the resolved path — failed or + // retried turns, results on sibling branches) were stripped by the + // context build; surface a placeholder so the turn's activity is + // visibly elided instead of silently vanishing (the "bare thinking + // lines" transcript trap). + const strippedToolCalls = (message as AgentMessage & StrippedToolCallsMarker).strippedToolCalls ?? 0; + if (strippedToolCalls > 0) { + this.ctx.chatContainer.addChild( + new Text( + theme.fg( + "dim", + theme.italic( + `${strippedToolCalls} tool call${strippedToolCalls === 1 ? "" : "s"} elided — no result on this branch`, + ), + ), + 1, + 0, + ), + ); + } pendingUsage = this.ctx.settings.get("display.showTokenUsage") && assistantUsageIsBilled(message.usage) ? message.usage diff --git a/packages/coding-agent/src/session/session-context.ts b/packages/coding-agent/src/session/session-context.ts index cb830ba3d..eee08d6dc 100644 --- a/packages/coding-agent/src/session/session-context.ts +++ b/packages/coding-agent/src/session/session-context.ts @@ -131,6 +131,16 @@ export interface BuildSessionContextOptions { keepDanglingToolCalls?: boolean; } +/** + * Display-only marker set on transcript assistant messages whose dangling + * `toolCall` blocks were stripped (no paired result on the resolved path — + * failed/retried turns, results on sibling branches). The TUI renders a + * placeholder row from it so the turn's activity never silently vanishes. + */ +export interface StrippedToolCallsMarker { + strippedToolCalls?: number; +} + /** * Build the session context from entries using tree traversal. * If leafId is provided, walks from that entry to root. @@ -353,6 +363,7 @@ export function buildSessionContext( undefined, undefined, snapcompactHistoryBlocksForContext(snapcompactArchive, options), + entry.warning, ), ); } else { @@ -385,6 +396,7 @@ export function buildSessionContext( providerPayload, undefined, snapcompactHistoryBlocksForContext(snapcompactArchive, options), + compaction.warning, ); // Agent context (non-transcript): summary first so the LLM sees the // compacted context before recent messages. @@ -468,10 +480,11 @@ export function buildSessionContext( for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; if (message.role !== "assistant") continue; - const hasDangling = message.content.some( - block => block.type === "toolCall" && !pairedToolResultIds.has(block.id), - ); - if (!hasDangling) continue; + let strippedToolCalls = 0; + for (const block of message.content) { + if (block.type === "toolCall" && !pairedToolResultIds.has(block.id)) strippedToolCalls++; + } + if (strippedToolCalls === 0) continue; const normalized = message.content .filter( block => @@ -483,13 +496,17 @@ export function buildSessionContext( ? { ...block, thinkingSignature: undefined } : block, ); - if (normalized.length === 0) { + if (normalized.length === 0 && !options?.transcript) { messages.splice(i, 1); - if (options?.transcript) { - cacheMissExplainedAt.splice(i, 1); - } } else { - messages[i] = { ...message, content: normalized }; + const rewritten = { ...message, content: normalized }; + if (options?.transcript) { + // Display transcript: keep the turn (even content-less) and mark + // how many calls were dropped so the TUI renders a placeholder + // row instead of silently erasing the turn's activity. + (rewritten as AgentMessage & StrippedToolCallsMarker).strippedToolCalls = strippedToolCalls; + } + messages[i] = rewritten; } } } From 22f2c1947bc14eb1e6a31843d4607a480b99cb37 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 01:40:18 +0200 Subject: [PATCH 53/80] test(session): validated auto-compaction guard and recovery behavior - Added test coverage for auto-compaction entry warning attribution during dead-end scenarios. - Verified automatic continuation and warning suppression when image-drop recovery successfully clears context pressure. - Mocked session context usage and image-drop functionality to simulate high-usage state recovery. --- .../src/session/session-context.test.ts | 15 +++-- ...ion-auto-compaction-progress-guard.test.ts | 64 +++++++++++++++++++ .../test/compaction-lifecycle.test.ts | 3 + 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/packages/coding-agent/src/session/session-context.test.ts b/packages/coding-agent/src/session/session-context.test.ts index 4744c69cb..914434999 100644 --- a/packages/coding-agent/src/session/session-context.test.ts +++ b/packages/coding-agent/src/session/session-context.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "bun:test"; import type { AgentMessage } from "@oh-my-pi/pi-agent-core"; import * as snapcompact from "@oh-my-pi/snapcompact"; import type { CompactionSummaryMessage } from "./messages"; -import { buildSessionContext } from "./session-context"; +import { buildSessionContext, type StrippedToolCallsMarker } from "./session-context"; import type { SessionEntry } from "./session-entries"; const timestamp = "2026-07-09T00:00:00.000Z"; @@ -129,12 +129,16 @@ function danglingCallIds(messages: AgentMessage[]): string[] { } describe("buildSessionContext dangling toolCalls", () => { - it("strips a dangling toolCall-only assistant turn from the transcript by default", () => { + it("strips a dangling toolCall from the transcript but keeps the turn with a stripped marker", () => { const context = buildSessionContext(danglingToolCallEntries, undefined, undefined, { transcript: true }); expect(danglingCallIds(context.messages)).toEqual([]); - // The turn had nothing but the dangling call, so the whole message drops. - expect(context.messages.some(message => message.role === "assistant")).toBe(false); + // The turn survives (even content-less) carrying the marker so the TUI + // renders a placeholder row instead of silently erasing the activity. + const assistant = context.messages.find(message => message.role === "assistant"); + expect(assistant).toBeDefined(); + expect(assistant?.content).toEqual([]); + expect((assistant as AgentMessage & StrippedToolCallsMarker).strippedToolCalls).toBe(1); }); it("keeps a dangling toolCall in transcript mode with keepDanglingToolCalls", () => { @@ -146,11 +150,12 @@ describe("buildSessionContext dangling toolCalls", () => { expect(danglingCallIds(context.messages)).toEqual(["call-1"]); }); - it("always strips dangling toolCalls from the LLM context", () => { + it("always strips dangling toolCalls from the LLM context and drops the emptied turn", () => { const context = buildSessionContext(danglingToolCallEntries, undefined, undefined, { keepDanglingToolCalls: true, }); expect(danglingCallIds(context.messages)).toEqual([]); + expect(context.messages.some(message => message.role === "assistant")).toBe(false); }); }); diff --git a/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts b/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts index f90ced886..f39cec861 100644 --- a/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts +++ b/packages/coding-agent/test/agent-session-auto-compaction-progress-guard.test.ts @@ -11,6 +11,7 @@ import { loadExtensions } from "@oh-my-pi/pi-coding-agent/extensibility/extensio import { ExtensionRunner } from "@oh-my-pi/pi-coding-agent/extensibility/extensions/runner"; import { AgentSession } from "@oh-my-pi/pi-coding-agent/session/agent-session"; import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage"; +import type { CompactionEntry } from "@oh-my-pi/pi-coding-agent/session/session-entries"; import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager"; import { getProjectAgentDir, TempDir } from "@oh-my-pi/pi-utils"; @@ -1230,5 +1231,68 @@ describe("AgentSession auto-compaction progress guard", () => { expect(noProgress[0].level).toBe("warning"); const recovery = notices.filter(n => n.source === NOTICE_SOURCE && n.message.includes("dead-end recovery")); expect(recovery.length).toBe(0); + // The dead-end is also stamped on the compaction entry so the transcript + // divider badges the pause and carries the warning across rebuilds/resume. + const compactionEntry = sessionManager + .getEntries() + .filter((e): e is CompactionEntry => e.type === "compaction") + .at(-1); + expect(compactionEntry?.warning).toContain(NO_PROGRESS_FRAGMENT); + }); + + it("auto-continues (no warning) when the image-drop tier frees an image-only tail", async () => { + // Elide cannot touch image content (collectShakeRegions skips image-only + // tool results and user-message images), so the rescue's second tier drops + // attached images — the automated `/shake images` remedy — and re-tests + // the recovery band before the guard is allowed to pause. + const promptSpy = vi.spyOn(session.agent, "prompt").mockResolvedValue(undefined as never); + vi.spyOn(session.agent, "continue").mockResolvedValue(); + let imagesDropped = false; + vi.spyOn(session, "getContextUsage").mockImplementation(() => + imagesDropped + ? { tokens: 1000, contextWindow: 200000, percent: 0.5 } + : { tokens: 190000, contextWindow: 200000, percent: 95 }, + ); + // Nothing elide-eligible in the oversized tail. + vi.spyOn(session, "shake").mockResolvedValue({ + mode: "elide", + toolResultsDropped: 0, + blocksDropped: 0, + tokensFreed: 0, + }); + const dropSpy = vi.spyOn(session, "dropImages").mockImplementation(async () => { + imagesDropped = true; + return { removed: 2 }; + }); + + const notices = collectNotices(); + + const { promise: compactionDone, resolve: onCompactionDone } = Promise.withResolvers(); + session.subscribe(event => { + if (event.type === "auto_compaction_end") onCompactionDone(); + }); + + const assistantMsg = highUsageAssistant(); + session.agent.emitExternalEvent({ type: "message_end", message: assistantMsg }); + session.agent.emitExternalEvent({ type: "agent_end", messages: [assistantMsg] }); + + await compactionDone; + await session.waitForIdle(); + + expect(dropSpy).toHaveBeenCalledTimes(1); + expect(promptSpy).toHaveBeenCalledTimes(1); + const noProgress = notices.filter(n => n.source === NOTICE_SOURCE && n.message.includes(NO_PROGRESS_FRAGMENT)); + expect(noProgress.length).toBe(0); + const recovery = notices.filter( + n => n.source === NOTICE_SOURCE && n.message.includes("dropped 2 attached images"), + ); + expect(recovery.length).toBe(1); + expect(recovery[0].level).toBe("info"); + // A rescued pass must not stamp the dead-end warning on the entry. + const compactionEntry = sessionManager + .getEntries() + .filter((e): e is CompactionEntry => e.type === "compaction") + .at(-1); + expect(compactionEntry?.warning).toBeUndefined(); }); }); diff --git a/packages/coding-agent/test/compaction-lifecycle.test.ts b/packages/coding-agent/test/compaction-lifecycle.test.ts index b6275266d..8658a5e8d 100644 --- a/packages/coding-agent/test/compaction-lifecycle.test.ts +++ b/packages/coding-agent/test/compaction-lifecycle.test.ts @@ -48,6 +48,9 @@ function buildCtx(compact: InteractiveModeContext["session"]["compact"]) { updateEditorTopBorder: vi.fn(), showError, flushCompactionQueue: vi.fn(async () => undefined), + // executeCompaction consults display.collapseCompacted on the ok path to + // decide whether the rebuild replaces the terminal transcript. + settings: { get: vi.fn(() => true) }, } as unknown as InteractiveModeContext; return { From 14aa1e20657e779f86a2a5c11ab7ba743ff55b79 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 03:02:35 +0200 Subject: [PATCH 54/80] feat(harbor-manager): implemented experiment tracking and orchestration - Renamed package to harbor-manager and implemented a SQLite-backed persistence layer for run metadata and trial lifecycle tracking. - Introduced a web-based dashboard with React for experiment visualization, metrics monitoring, and interactive run management. - Added a REST and SSE API to support remote monitoring, session log streaming, and process orchestration. - Implemented a new `source` installation mode for agent runs and added CLI support for forwarding custom arguments to the agent. --- bun.lock | 77 +- packages/harbor-manager/README.md | 107 +++ .../agent/omp_local.py | 197 +++-- packages/harbor-manager/package.json | 45 + .../harbor-manager/src/experiments.test.ts | 113 +++ packages/harbor-manager/src/experiments.ts | 167 ++++ packages/harbor-manager/src/manager.test.ts | 218 +++++ packages/harbor-manager/src/runner.test.ts | 98 +++ .../src/runner.ts | 427 ++++++--- packages/harbor-manager/src/server.ts | 425 +++++++++ packages/harbor-manager/src/store.ts | 441 ++++++++++ packages/harbor-manager/src/web/app.tsx | 817 ++++++++++++++++++ packages/harbor-manager/src/web/index.html | 16 + packages/harbor-manager/tsconfig.json | 10 + packages/terminal-bench/README.md | 131 --- packages/terminal-bench/package.json | 30 - packages/terminal-bench/tsconfig.json | 6 - 17 files changed, 2934 insertions(+), 391 deletions(-) create mode 100644 packages/harbor-manager/README.md rename packages/{terminal-bench => harbor-manager}/agent/omp_local.py (75%) create mode 100644 packages/harbor-manager/package.json create mode 100644 packages/harbor-manager/src/experiments.test.ts create mode 100644 packages/harbor-manager/src/experiments.ts create mode 100644 packages/harbor-manager/src/manager.test.ts create mode 100644 packages/harbor-manager/src/runner.test.ts rename packages/{terminal-bench => harbor-manager}/src/runner.ts (72%) create mode 100755 packages/harbor-manager/src/server.ts create mode 100644 packages/harbor-manager/src/store.ts create mode 100644 packages/harbor-manager/src/web/app.tsx create mode 100644 packages/harbor-manager/src/web/index.html create mode 100644 packages/harbor-manager/tsconfig.json delete mode 100644 packages/terminal-bench/README.md delete mode 100644 packages/terminal-bench/package.json delete mode 100644 packages/terminal-bench/tsconfig.json diff --git a/bun.lock b/bun.lock index 3778bbec5..80270a3e9 100644 --- a/bun.lock +++ b/bun.lock @@ -136,6 +136,29 @@ "@types/react-dom": "catalog:", }, }, + "packages/harbor-manager": { + "name": "@oh-my-pi/harbor-manager", + "version": "0.0.1", + "bin": { + "harbor-manager": "src/runner.ts", + }, + "dependencies": { + "clsx": "^2.1.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.2.0", + "motion": "^12.15.0", + "react": "^19.1.0", + "react-dom": "^19.1.0", + "tailwind-merge": "^3.3.0", + }, + "devDependencies": { + "@types/bun": "catalog:", + "@types/d3-scale": "^4.0.9", + "@types/d3-shape": "^3.1.7", + "@types/react": "^19.1.0", + "@types/react-dom": "^19.1.0", + }, + }, "packages/hashline": { "name": "@oh-my-pi/hashline", "version": "16.4.8", @@ -236,16 +259,6 @@ "@oh-my-pi/pi-coding-agent": "^16", }, }, - "packages/terminal-bench": { - "name": "@oh-my-pi/terminal-bench", - "version": "0.0.1", - "bin": { - "tb2": "src/runner.ts", - }, - "devDependencies": { - "@types/bun": "catalog:", - }, - }, "packages/tui": { "name": "@oh-my-pi/pi-tui", "version": "16.4.8", @@ -740,6 +753,8 @@ "@oh-my-pi/collab-web": ["@oh-my-pi/collab-web@workspace:packages/collab-web"], + "@oh-my-pi/harbor-manager": ["@oh-my-pi/harbor-manager@workspace:packages/harbor-manager"], + "@oh-my-pi/hashline": ["@oh-my-pi/hashline@workspace:packages/hashline"], "@oh-my-pi/omp-stats": ["@oh-my-pi/omp-stats@workspace:packages/stats"], @@ -766,8 +781,6 @@ "@oh-my-pi/swarm-extension": ["@oh-my-pi/swarm-extension@workspace:packages/swarm-extension"], - "@oh-my-pi/terminal-bench": ["@oh-my-pi/terminal-bench@workspace:packages/terminal-bench"], - "@oh-my-pi/typescript-edit-benchmark": ["@oh-my-pi/typescript-edit-benchmark@workspace:packages/typescript-edit-benchmark"], "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], @@ -900,6 +913,14 @@ "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/d3-path": ["@types/d3-path@3.1.1", "", {}, "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg=="], + + "@types/d3-scale": ["@types/d3-scale@4.0.9", "", { "dependencies": { "@types/d3-time": "*" } }, "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw=="], + + "@types/d3-shape": ["@types/d3-shape@3.1.8", "", { "dependencies": { "@types/d3-path": "*" } }, "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w=="], + + "@types/d3-time": ["@types/d3-time@3.0.4", "", {}, "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g=="], + "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], "@types/react": ["@types/react@19.2.17", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw=="], @@ -1000,6 +1021,8 @@ "cliui": ["cliui@9.0.1", "", { "dependencies": { "string-width": "^7.2.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w=="], + "clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="], + "code-block-writer": ["code-block-writer@13.0.3", "", {}, "sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg=="], "color": ["color@5.0.3", "", { "dependencies": { "color-convert": "^3.1.3", "color-string": "^2.1.3" } }, "sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA=="], @@ -1026,6 +1049,24 @@ "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], + "d3-array": ["d3-array@3.2.4", "", { "dependencies": { "internmap": "1 - 2" } }, "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg=="], + + "d3-color": ["d3-color@3.1.0", "", {}, "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="], + + "d3-format": ["d3-format@3.1.2", "", {}, "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg=="], + + "d3-interpolate": ["d3-interpolate@3.0.1", "", { "dependencies": { "d3-color": "1 - 3" } }, "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g=="], + + "d3-path": ["d3-path@3.1.0", "", {}, "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ=="], + + "d3-scale": ["d3-scale@4.0.2", "", { "dependencies": { "d3-array": "2.10.0 - 3", "d3-format": "1 - 3", "d3-interpolate": "1.2.0 - 3", "d3-time": "2.1.1 - 3", "d3-time-format": "2 - 4" } }, "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ=="], + + "d3-shape": ["d3-shape@3.2.0", "", { "dependencies": { "d3-path": "^3.1.0" } }, "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA=="], + + "d3-time": ["d3-time@3.1.0", "", { "dependencies": { "d3-array": "2 - 3" } }, "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q=="], + + "d3-time-format": ["d3-time-format@4.1.0", "", { "dependencies": { "d3-time": "1 - 3" } }, "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg=="], + "date-fns": ["date-fns@4.4.0", "", {}, "sha512-+1UMbeh68lH1SegH83CGWwpb6OHHbpSgr3+s5Eww5M4CAgswBpoWS0AjTOfEJ33HiYKz1hdj/KTFprzXHmq/6w=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -1106,6 +1147,8 @@ "fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="], + "framer-motion": ["framer-motion@12.42.2", "", { "dependencies": { "motion-dom": "^12.42.2", "motion-utils": "^12.39.0", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-5XY9luDiu0oHfHBjpDthFMh0ES+122w6p/papSJBweMkO8Sn+PW2QaEgRblQBpWFnuvZS5qvarpt/hO2pjGmnw=="], + "fs-minipass": ["fs-minipass@2.1.0", "", { "dependencies": { "minipass": "^3.0.0" } }, "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg=="], "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], @@ -1150,6 +1193,8 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + "internmap": ["internmap@2.0.3", "", {}, "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg=="], + "is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], "is-obj": ["is-obj@2.0.0", "", {}, "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="], @@ -1254,6 +1299,12 @@ "moment": ["moment@2.30.1", "", {}, "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how=="], + "motion": ["motion@12.42.2", "", { "dependencies": { "framer-motion": "^12.42.2", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-Atvv11yUKIid41cVrRBDVX5m8tF8kNpExRSlbpt6APClhDjtwQssgFHhQzejxw7/7YYbjHSPKBVbHo05BuJT5Q=="], + + "motion-dom": ["motion-dom@12.42.2", "", { "dependencies": { "motion-utils": "^12.39.0" } }, "sha512-5gIMWLp/PycBtJRJWRgjxke5n8dlvkSn2DrYW+tr3XcqAZY1xZh6BJyooJXCM8wdfM7wfMjkBJNLge1CKPUIRA=="], + + "motion-utils": ["motion-utils@12.39.0", "", {}, "sha512-8nadJAJjTtqRkmRF36FoJTrywK9nnFmnPwnSMyxaOCU7GDjN9RTMJIxx9De8ErM+vpPhMccr/6fo5WciyQLnMQ=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "mupdf": ["mupdf@1.28.0", "", {}, "sha512-ACUnbpECaQ5JLq04pwd89lS+0IGMest5qL5tb08g9TAR7bDtfqflHEkb2Xm3o4rvC/szguLiV+WEbW9kstj8Sg=="], @@ -1400,6 +1451,8 @@ "strnum": ["strnum@2.4.1", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg=="], + "tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="], + "tailwindcss": ["tailwindcss@4.3.2", "", {}, "sha512-WtctNNSH8A9jlMIqxzuYumOHU5uGZyRv0Q5svQl+oEPy5w84YpBxdb7MdqyiSPQge5jTJ6zFQLq0PFygdccSBA=="], "tapable": ["tapable@2.3.3", "", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="], diff --git a/packages/harbor-manager/README.md b/packages/harbor-manager/README.md new file mode 100644 index 000000000..107a06389 --- /dev/null +++ b/packages/harbor-manager/README.md @@ -0,0 +1,107 @@ +# @oh-my-pi/harbor-manager + +Manage [Harbor](https://github.com/laude-institute/harbor) benchmark runs against +the **local `omp` build**: a CLI runner with a live terminal dashboard, a +SQLite-backed run store, a REST/SSE API, and a web dashboard with live +run → trial → transcript drill-down. + +Works with any Harbor dataset (`terminal-bench@2.0` by default, +`swe-bench/swe-bench-verified`, …). + +```bash +# CLI run (owns the terminal, writes a markdown report) +bun src/runner.ts --model anthropic/claude-sonnet-4-6 --tasks 20 --concurrency 4 + +# Web manager: dashboard + API on :4700 +bun run serve # bun src/server.ts [--port 4700] [--jobs-dir ] +``` + +## How runs execute + +1. **Local omp, not npm.** By default the runner bind-mounts the repo + read-only into each task container (`--install source`) and runs omp + straight from `packages/coding-agent/src/cli.ts` — TS edits apply to the + next trial with no rebuild. A cached linux `node_modules` tree (built once + per lockfile change inside `oven/bun`, stored in `/_bench/_deps/`) + shadows the host's darwin one, and a linux `bun` binary is mounted at + `/opt/omp/bin` — so trial setup needs zero outbound network. Alternatives: + `--install local` (pack a tarball per run) or `--binary` (prebuilt + `dist/omp-linux-*` self-contained binaries). +2. **Auth never enters containers.** A generated `models.yml` routes provider + `baseUrl`s at the host pm2 auth-gateway; the gateway resolves credentials + host-side. +3. **Harbor owns trials.** The runner/serve layer polls each trial's + `result.json` for progress, spend, and outcomes. + +## Server + +- `GET /` — dashboard: run list (SSE-live), trial grid, transcript viewer + that tails live sessions. +- `GET /api/runs` — run rows (rollups: pass/fail/error/spend/tokens). +- `POST /api/runs` — launch. Body: + + ```json + { + "model": "anthropic/claude-opus-4-8", + "dataset": "swe-bench/swe-bench-verified", + "tasks": 20, + "concurrency": 4, + "timeoutMultiplier": 2, + "include": ["swe-bench/astropy__astropy-14995"], + "slide": { "model": "google/gemini-3.5-flash", "onAction": true, "plan": true } + } + ``` + + `slide.turns` and `slide.onAction` are mutually exclusive triggers. +- `GET /api/runs/:name` — `{ run, trials }` (syncs from disk on read). +- `DELETE /api/runs/:name` — cancel a manager-launched run. +- `GET /api/runs/:name/trials/:trial/transcript?tail=N[&raw=1]` — compact + (or raw JSONL) view of the trial's live session log. +- `GET /api/events` — SSE stream of run-list snapshots (sent on change). + +State lives in `/_manager/harbor-manager.sqlite`; the filesystem +stays the source of truth and historical CLI runs are auto-discovered. + +## Runner options (excerpt) + +| Option | Default | Notes | +|---|---|---| +| `-m, --model ` | `anthropic/claude-sonnet-4-6` | Repeatable | +| `-l, --tasks ` | `20` | Max tasks | +| `-n, --concurrency ` | `4` | Concurrent trials | +| `-k, --attempts ` | `1` | Attempts per task (pass@k) | +| `-d, --dataset ` | `terminal-bench@2.0` | Any Harbor dataset id | +| `-i/-x, --include/--exclude ` | — | Task filters (repeatable) | +| `--timeout-multiplier ` | — | Scales task agent/verifier timeouts | +| `--agent-arg ` | — | Extra arg forwarded verbatim to the in-container omp CLI (repeatable) | +| `--env ` | — | Forward env into the omp container (repeatable); `KEY` alone forwards the host value | +| `--binary ` | — | Prebuilt omp binary (repeat for arm64+x64) | +| `--install ` | `source` | `source` = repo bind-mount, `local` = tarball pack, `published` = npm `@oh-my-pi/pi-coding-agent` | +| `--gateway-url ` | `http://host.docker.internal:4000` | | +| `--no-gateway` | off | Pass host provider keys into containers instead | +| `-o, --jobs-dir ` | `/runs/harbor` | Shared with the server | +| `--dry-run` | off | Print the harbor command + models.yml and exit | + +## Outputs + +- `//` — Harbor trial dirs (`result.json` per trial). +- `/_bench//report.md` — markdown summary table. +- `/_bench//harbor.log` — full Harbor output. +- `/_manager/logs/.log` — runner output for API-launched runs. + +## Caveats + +- **Network policy.** On Harbor's local Docker backend only **public** + registries work; task containers reach models via the host gateway. +- **`--install source` reflects local TS changes** with no rebuild, but Rust + natives load from the in-tree `packages/natives/native/pi_natives.linux-*.node` + prebuilds — rebuild those when Rust changes (the loader skips the version + sentinel for workspace loads, so a stale `.node` runs silently). +- **Source mode is single-arch.** The deps tree matches the docker daemon's + native arch; trials on emulated images (e.g. x64 tasks on an arm64 host) + fail setup with an arch-mismatch error — use `--binary` for those. +- **The repo is visible (read-only) inside task containers** in source mode; + fine for curated benchmarks, but don't point it at untrusted tasks. +- **`--install local` reflects local TS changes** (inlined into `dist/cli.js`), + but **not** uncommitted Rust natives — rebuild `packages/natives` per target + first (the version sentinel must match). diff --git a/packages/terminal-bench/agent/omp_local.py b/packages/harbor-manager/agent/omp_local.py similarity index 75% rename from packages/terminal-bench/agent/omp_local.py rename to packages/harbor-manager/agent/omp_local.py index ea73160cc..66196540c 100644 --- a/packages/terminal-bench/agent/omp_local.py +++ b/packages/harbor-manager/agent/omp_local.py @@ -1,12 +1,18 @@ """Harbor agent that runs the LOCAL oh-my-pi (`omp`) build inside task containers. Unlike Harbor's built-in `pi` agent (which `npm i -g @mariozechner/pi-coding-agent`), -this installs the working tree at `/work/pi`: +this runs the working tree at `/work/pi`. Install modes (`OMP_BENCH_INSTALL`): - * the runner packs `packages/coding-agent` with `bun pm pack` (bundles every - workspace TS package into `dist/cli.js`) and hands us the tarball path, - * we upload it, install Bun, `bun install` the bundle's external deps + the - platform native addon, and run `bun .../dist/cli.js`. + * `source` (default): the runner bind-mounts the repo read-only plus a + prebuilt linux `node_modules` tree and a linux `bun` binary; omp runs + straight from `packages/coding-agent/src/cli.ts`. Zero-network setup, and + host TS edits apply to the next trial with no rebuild (Rust natives load + from the in-tree `packages/natives/native/*.node` prebuilds). + * `local`: the runner packs `packages/coding-agent` with `bun pm pack` + (bundles every workspace TS package into `dist/cli.js`) and hands us the + tarball path; we upload it, install Bun, `bun install` the bundle's + external deps + the platform native addon, and run `bun .../dist/cli.js`. + * binary (`--binary`): a self-contained compiled omp binary is uploaded. Auth never enters the container: a generated `~/.omp/agent/models.yml` routes the configured providers' `baseUrl` at the host's pm2 auth-gateway (default @@ -14,7 +20,7 @@ configured providers' `baseUrl` at the host's pm2 auth-gateway (default resolves credentials host-side. No provider API keys are passed in. All knobs come from environment variables the runner sets on the `harbor` process -(see `OMP_TB_*` below); the agent reads them from `os.environ` directly. +(see `OMP_BENCH_*` below); the agent reads them from `os.environ` directly. Selected via `harbor run --agent-import-path omp_local:OmpLocal` with the directory of this file on `PYTHONPATH`. @@ -89,7 +95,6 @@ _TARBALL_DST = "/tmp/omp-local.tgz" _MODELS_DST = "/tmp/omp-models.yml" _CONFIG_DST = "/tmp/omp-config.yml" _OUTPUT_FILENAME = "omp.txt" -_ADVISOR_FILENAME = "advisor.jsonl" # Provider → host env vars used in --no-gateway (direct-auth) mode only. _PROVIDER_KEYS: dict[str, list[str]] = { @@ -166,37 +171,42 @@ class OmpLocal(BaseInstalledAgent): def __init__(self, *args, **kwargs) -> None: # noqa: D401 - thin wrapper super().__init__(*args, **kwargs) - self._install_mode = _env("OMP_TB_INSTALL", "local") - self._tarball = _env("OMP_TB_TARBALL") - self._pkg_version = _env("OMP_TB_VERSION", "latest") - self._models_yaml_path = _env("OMP_TB_MODELS_YAML") - self._gateway_url = _env("OMP_TB_GATEWAY_URL", "http://host.docker.internal:4000") - self._gateway_token = _env("OMP_TB_GATEWAY_TOKEN", "no-auth-dummy") + self._install_mode = _env("OMP_BENCH_INSTALL", "source") + self._tarball = _env("OMP_BENCH_TARBALL") + self._pkg_version = _env("OMP_BENCH_VERSION", "latest") + self._models_yaml_path = _env("OMP_BENCH_MODELS_YAML") + self._gateway_url = _env("OMP_BENCH_GATEWAY_URL", "http://host.docker.internal:4000") + self._gateway_token = _env("OMP_BENCH_GATEWAY_TOKEN", "no-auth-dummy") self._gateway_providers = [ p.strip() - for p in _env("OMP_TB_GATEWAY_PROVIDERS", "anthropic,openai-codex").split(",") + for p in _env("OMP_BENCH_GATEWAY_PROVIDERS", "anthropic,openai-codex").split(",") if p.strip() ] - self._thinking = _env("OMP_TB_THINKING") - self._auto_approve = _truthy(_env("OMP_TB_AUTO_APPROVE", "1")) - self._extra_args = _env("OMP_TB_EXTRA_ARGS") - self._bun_version = _env("OMP_TB_BUN_VERSION", "1.3.14") - self._gateway_on = _env("OMP_TB_GATEWAY", "1") != "0" - # Optional second model reviewing the primary (separate spend, summed in). - self._advisor_model = _env("OMP_TB_ADVISOR_MODEL") - self._advisor_sync = _env("OMP_TB_ADVISOR_SYNC", "1") + self._thinking = _env("OMP_BENCH_THINKING") + self._auto_approve = _truthy(_env("OMP_BENCH_AUTO_APPROVE", "1")) + # Extra CLI args forwarded verbatim to the in-container omp invocation, + # JSON-array-encoded by the runner (OMP_BENCH_AGENT_ARGS) so multi-word + # values survive without a second layer of shell quoting. + self._agent_args = self._parse_agent_args() + self._bun_version = _env("OMP_BENCH_BUN_VERSION", "1.3.14") + self._gateway_on = _env("OMP_BENCH_GATEWAY", "1") != "0" + # web_search auth can't route through the gateway (dedicated provider creds); # off by default so search-using tasks don't false-negative on 401s. - self._web_search = _truthy(_env("OMP_TB_WEB_SEARCH", "0")) + self._web_search = _truthy(_env("OMP_BENCH_WEB_SEARCH", "0")) # Extra env (PI_* dialect knobs, explicit --env) the runner forwards into - # the in-container omp run, JSON-encoded in OMP_TB_FORWARD_ENV. + # the in-container omp run, JSON-encoded in OMP_BENCH_FORWARD_ENV. self._forward_env = self._parse_forward_env() + # Source-mount paths (defaults must match the runner's compose overlay). + self._source_dir = _env("OMP_BENCH_SOURCE_DIR", "/opt/omp/src") + self._source_bun = _env("OMP_BENCH_SOURCE_BUN", "/opt/omp/bin/bun") + self._source_arch = _env("OMP_BENCH_SOURCE_ARCH") # Resolved during install(); reused by version + run commands. self._home = "/root" self._bun = "/root/.bun/bin/bun" self._cli = "/root/.omp-bench/app/dist/cli.js" - self._binary_arm64 = _env("OMP_TB_BINARY_ARM64") - self._binary_x64 = _env("OMP_TB_BINARY_X64") + self._binary_arm64 = _env("OMP_BENCH_BINARY_ARM64") + self._binary_x64 = _env("OMP_BENCH_BINARY_X64") self._binary = bool(self._binary_arm64 or self._binary_x64) @staticmethod @@ -226,9 +236,10 @@ class OmpLocal(BaseInstalledAgent): omp spawns Bun worker subprocesses at runtime, so `bun` must resolve on PATH during `run()` too — not just for the entrypoint. """ + bun_dir = os.path.dirname(self._bun) return ( f'export BUN_INSTALL={shlex.quote(self._home + "/.bun")}; ' - f'export PATH="{self._home}/.bun/bin:$PATH"; ' + f'export PATH="{bun_dir}:$PATH"; ' f"{command}" ) @@ -242,6 +253,9 @@ class OmpLocal(BaseInstalledAgent): # Self-contained binary mode: upload + chmod only. No apt/curl/bun/npm, so # trial setup needs zero outbound network (no_network tasks set up cleanly). await self._install_binary(environment) + elif self._install_mode == "source": + # Everything is bind-mounted by the runner; nothing to download. + self._cli = await self._install_source(environment) else: # 1) System deps (root). curl+unzip for the Bun installer; ca-certs for TLS. await self.exec_as_root( @@ -279,9 +293,39 @@ class OmpLocal(BaseInstalledAgent): await self._write_models_yaml(environment) await self._write_config(environment) + async def _install_source(self, environment: BaseEnvironment) -> str: + """Verify the read-only repo + linux deps mounts and run omp from TS source. + + The runner mounts the repo at `self._source_dir`, shadows every host + `node_modules` with a linux tree, and mounts a linux `bun` binary — so + setup needs zero outbound network and no rebuild for TS changes. + """ + arch = (await self.exec_as_agent(environment, command="uname -m")).stdout.strip() + norm = {"aarch64": "arm64", "arm64": "arm64", "x86_64": "x64", "amd64": "x64"}.get(arch) + if self._source_arch and norm != self._source_arch: + raise RuntimeError( + f"source mode: container arch {arch!r} != mounted deps tree arch " + f"({self._source_arch}); use --binary for emulated-arch tasks" + ) + self._bun = self._source_bun + cli = f"{self._source_dir}/packages/coding-agent/src/cli.ts" + q = shlex.quote + await self.exec_as_agent( + environment, + command=( + "set -e; " + f"test -x {q(self._source_bun)} || {{ echo 'omp source mode: bun mount missing' >&2; exit 5; }}; " + f"test -f {q(cli)} || {{ echo 'omp source mode: repo mount missing' >&2; exit 5; }}; " + f"test -d {q(self._source_dir + '/node_modules/@oh-my-pi')} || " + "{ echo 'omp source mode: linux deps mount missing' >&2; exit 5; }; " + f"{q(self._source_bun)} --version" + ), + ) + return cli + async def _install_local(self, environment: BaseEnvironment) -> str: if not self._tarball: - raise RuntimeError("OMP_TB_INSTALL=local requires OMP_TB_TARBALL (host tarball path)") + raise RuntimeError("OMP_BENCH_INSTALL=local requires OMP_BENCH_TARBALL (host tarball path)") await environment.upload_file(self._tarball, _TARBALL_DST) app = f"{self._home}/.omp-bench/app" await self.exec_as_agent( @@ -363,7 +407,7 @@ class OmpLocal(BaseInstalledAgent): ) def _generate_models_yaml(self) -> str: - lines = ["# Generated by terminal-bench runner — routes auth via host gateway.", "providers:"] + lines = ["# Generated by harbor-manager runner — routes auth via host gateway.", "providers:"] for provider in self._gateway_providers: lines += [ f" {provider}:", @@ -375,25 +419,15 @@ class OmpLocal(BaseInstalledAgent): return "\n".join(lines) async def _write_config(self, environment: BaseEnvironment) -> None: - """Write $HOME/.omp/agent/config.yml: web_search toggle + optional advisor. + """Write $HOME/.omp/agent/config.yml: the web_search toggle. - The advisor is a separate model with its own spend; its turns are written - to /__advisor.jsonl (requires a persisted session, see run()). web_search can't authenticate through the gateway, so it's off by default. """ lines = [ - "# Generated by terminal-bench runner.", + "# Generated by harbor-manager runner.", "web_search:", f" enabled: {'true' if self._web_search else 'false'}", ] - if self._advisor_model: - lines += [ - "modelRoles:", - f" advisor: {self._advisor_model}", - "advisor:", - " enabled: true", - f' syncBacklog: "{self._advisor_sync}"', - ] content = "\n".join(lines) heredoc = f"cat > {_CONFIG_DST} <<'OMP_CONFIG_EOF'\n{content}\nOMP_CONFIG_EOF" await self.exec_as_agent(environment, command=heredoc) @@ -407,8 +441,8 @@ class OmpLocal(BaseInstalledAgent): @staticmethod def _parse_forward_env() -> dict[str, str]: - """Extra run-time env from the runner (OMP_TB_FORWARD_ENV = JSON object).""" - raw = _env("OMP_TB_FORWARD_ENV") + """Extra run-time env from the runner (OMP_BENCH_FORWARD_ENV = JSON object).""" + raw = _env("OMP_BENCH_FORWARD_ENV") if not raw: return {} try: @@ -419,17 +453,27 @@ class OmpLocal(BaseInstalledAgent): return {} return {str(key): str(value) for key, value in parsed.items()} + @staticmethod + def _parse_agent_args() -> list[str]: + """Extra CLI args from the runner (OMP_BENCH_AGENT_ARGS = JSON array).""" + raw = _env("OMP_BENCH_AGENT_ARGS") + if not raw: + return [] + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(parsed, list): + return [] + return [str(item) for item in parsed] + def _collect_provider_keys(self, provider: str) -> dict[str, str]: - """Host env vars for the primary + advisor providers (direct-auth mode).""" - providers = {provider} - if self._advisor_model and "/" in self._advisor_model: - providers.add(self._advisor_model.split("/", 1)[0]) + """Host env vars for the primary model's provider (direct-auth mode only).""" env: dict[str, str] = {} - for prov in providers: - for key in _PROVIDER_KEYS.get(prov, []): - value = os.environ.get(key) - if value: - env[key] = value + for key in _PROVIDER_KEYS.get(provider, []): + value = os.environ.get(key) + if value: + env[key] = value return env # ---------------------------------------------------------------------- run @@ -455,17 +499,13 @@ class OmpLocal(BaseInstalledAgent): "--mode json", f"--provider {shlex.quote(provider)}", f"--model {shlex.quote(model)}", + "--no-session", ] - # The advisor records its (separately-billed) turns to /__advisor.jsonl, - # which only exists with a persisted session — so keep sessions on for advisor runs. - if not self._advisor_model: - parts.append("--no-session") if self._auto_approve: parts.append("--auto-approve") if self._thinking: parts.append(f"--thinking {shlex.quote(self._thinking)}") - if self._extra_args: - parts.append(self._extra_args) + parts.extend(shlex.quote(arg) for arg in self._agent_args) # POSIX positional separator: some task prompts start with "-" (e.g. a # markdown bullet, as in pytorch-model-recovery). Without this, omp parses # the prompt as an unknown flag and exits 2. `--` forces positional mode. @@ -474,13 +514,6 @@ class OmpLocal(BaseInstalledAgent): # No pipes/stdbuf (absent in minimal images): redirect raw JSONL to the # mounted agent log dir; populate_context_post_run parses it on the host. run = " ".join(parts) + f" > /logs/agent/{_OUTPUT_FILENAME} 2>&1" - if self._advisor_model: - # Preserve omp's exit code, then collect advisor spend into the mounted dir. - run += ( - "; rc=$?; " - f'find "$HOME/.omp/agent/sessions" -name __advisor.jsonl -exec cat {{}} + ' - f"> /logs/agent/{_ADVISOR_FILENAME} 2>/dev/null || true; exit $rc" - ) # Exec env for the omp run. Direct-auth (no-gateway) mode contributes the # selected providers' keys (via exec env, never argv); forwarded PI_* / # --env knobs apply last so an explicit --env always wins. @@ -494,21 +527,15 @@ class OmpLocal(BaseInstalledAgent): def populate_context_post_run(self, context: AgentContext) -> None: main = _Usage() self._sum_main(self.logs_dir / _OUTPUT_FILENAME, main) - advisor = _Usage() - if self._advisor_model: - self._sum_advisor(self.logs_dir / _ADVISOR_FILENAME, advisor) - if main.empty() and advisor.empty(): + if main.empty(): return - total_cost = main.cost + advisor.cost - context.n_input_tokens = main.in_tok + main.cache_read + advisor.in_tok + advisor.cache_read - context.n_output_tokens = main.out_tok + advisor.out_tok - context.n_cache_tokens = main.cache_read + advisor.cache_read - context.cost_usd = total_cost if total_cost > 0 else None + context.n_input_tokens = main.in_tok + main.cache_read + context.n_output_tokens = main.out_tok + context.n_cache_tokens = main.cache_read + context.cost_usd = main.cost if main.cost > 0 else None context.metadata = { **(context.metadata or {}), - "cache_write_tokens": main.cache_write + advisor.cache_write, - "main_cost_usd": main.cost, - "advisor_cost_usd": advisor.cost, + "cache_write_tokens": main.cache_write, } def _sum_main(self, path: Path, acc: "_Usage") -> None: @@ -522,19 +549,3 @@ class OmpLocal(BaseInstalledAgent): message = event.get("message") if isinstance(message, dict) and message.get("role") == "assistant": acc.add(message.get("usage")) - - def _sum_advisor(self, path: Path, acc: "_Usage") -> None: - """Sum assistant-turn usage from concatenated __advisor.jsonl session entries.""" - if not path.exists(): - return - for line in path.read_text(errors="replace").splitlines(): - entry = _loads(line) - if not entry: - continue - # Session-tree entries are flat: {role: "assistant", usage: {...}}. - if entry.get("role") == "assistant": - acc.add(entry.get("usage")) - else: - message = entry.get("message") - if isinstance(message, dict) and message.get("role") == "assistant": - acc.add(message.get("usage")) diff --git a/packages/harbor-manager/package.json b/packages/harbor-manager/package.json new file mode 100644 index 000000000..42146e3f4 --- /dev/null +++ b/packages/harbor-manager/package.json @@ -0,0 +1,45 @@ +{ + "type": "module", + "private": true, + "name": "@oh-my-pi/harbor-manager", + "version": "0.0.1", + "description": "Manage Harbor benchmark runs against the local omp build: CLI runner, SQLite-backed run store, REST/SSE API, and a live web dashboard", + "homepage": "https://omp.sh", + "author": "Can Boluk", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/can1357/oh-my-pi.git", + "directory": "packages/harbor-manager" + }, + "bin": { + "harbor-manager": "src/runner.ts" + }, + "scripts": { + "check": "biome check . && bun run check:types", + "check:types": "tsgo -p tsconfig.json --noEmit", + "lint": "biome lint .", + "start": "bun run src/runner.ts", + "serve": "bun run src/server.ts", + "test": "bun test" + }, + "dependencies": { + "clsx": "^2.1.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.2.0", + "motion": "^12.15.0", + "react": "^19.1.0", + "react-dom": "^19.1.0", + "tailwind-merge": "^3.3.0" + }, + "devDependencies": { + "@types/bun": "catalog:", + "@types/d3-scale": "^4.0.9", + "@types/d3-shape": "^3.1.7", + "@types/react": "^19.1.0", + "@types/react-dom": "^19.1.0" + }, + "engines": { + "bun": ">=1.3.14" + } +} diff --git a/packages/harbor-manager/src/experiments.test.ts b/packages/harbor-manager/src/experiments.test.ts new file mode 100644 index 000000000..d659168ca --- /dev/null +++ b/packages/harbor-manager/src/experiments.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "bun:test"; +import { armOf, experimentOf, summarizeArm } from "./experiments"; +import type { RunRow, TrialRow } from "./store"; + +/** + * Contracts under test: + * - job names group by their first `-` token; arm labels strip that prefix. + * - summarizeArm computes observed metrics from decided trials only and + * projects running arms linearly (ETA, pass%, total cost). + */ + +function runRow(overrides: Partial): RunRow { + return { + jobName: "exp-arm", + dataset: "d", + agent: "omp", + models: "anthropic/claude-opus-4-8", + slide: null, + role: "", + note: "", + status: "running", + pid: null, + exitCode: null, + createdAt: Date.now(), + finishedAt: null, + nTotal: 0, + done: 0, + pass: 0, + fail: 0, + error: 0, + running: 0, + costUsd: 0, + tokIn: 0, + tokOut: 0, + tokCache: 0, + ...overrides, + }; +} + +function trialRow(overrides: Partial): TrialRow { + return { + jobName: "exp-arm", + name: "task__x", + task: "task", + status: "pass", + reward: 1, + costUsd: 1, + durationMs: 60_000, + detail: "", + updatedAt: Date.now(), + ...overrides, + }; +} + +describe("experiment grouping", () => { + it("groups by prefix and strips it from arm labels", () => { + expect(experimentOf("sb2-n4p-fix")).toBe("sb2"); + expect(armOf("sb2-n4p-fix")).toBe("n4p-fix"); + expect(experimentOf("standalone")).toBe("standalone"); + expect(armOf("standalone")).toBe("standalone"); + }); +}); + +describe("summarizeArm", () => { + it("projects running arms linearly and leaves finished arms unprojected", () => { + const tenMinutesAgo = Date.now() - 10 * 60_000; + const running = summarizeArm( + runRow({ + jobName: "sb2-n8", + status: "running", + createdAt: tenMinutesAgo, + nTotal: 20, + done: 10, + pass: 8, + costUsd: 5, + }), + [ + trialRow({ status: "pass", durationMs: 120_000 }), + trialRow({ name: "b__x", task: "b", status: "fail", reward: 0, durationMs: 240_000 }), + ], + ); + expect(running.arm).toBe("n8"); + expect(running.projected).not.toBeNull(); + // 10 done in 10 min → 1/min → 10 remaining ≈ 10 min out. + const etaMin = ((running.projected?.etaMs ?? 0) - Date.now()) / 60_000; + expect(etaMin).toBeGreaterThan(8); + expect(etaMin).toBeLessThan(12); + expect(running.projected?.passPct).toBeCloseTo(80, 5); + expect(running.projected?.costPerTask).toBeCloseTo(0.5, 5); + expect(running.projected?.totalCostUsd).toBeCloseTo(10, 5); + // observed metrics come from decided trials only + expect(running.passPct).toBeCloseTo((100 * 8) / 2, 5); // decided=2 in fixture trials + expect(running.meanTrialMs).toBeCloseTo(180_000, 5); + + const finished = summarizeArm( + runRow({ jobName: "sb2-opus", status: "complete", nTotal: 20, done: 20, pass: 15, costUsd: 30 }), + [trialRow({})], + ); + expect(finished.projected).toBeNull(); + expect(finished.costPerTask).toBeCloseTo(1.5, 5); + }); + + it("describes the slide config in the arm line", () => { + const arm = summarizeArm( + runRow({ + jobName: "sb2-nact", + slide: JSON.stringify({ model: "google/gemini-3.5-flash", onAction: true, plan: true }), + }), + [], + ); + expect(arm.config).toBe("anthropic/claude-opus-4-8 → google/gemini-3.5-flash on first edit/write +plan"); + }); +}); diff --git a/packages/harbor-manager/src/experiments.ts b/packages/harbor-manager/src/experiments.ts new file mode 100644 index 000000000..277128d54 --- /dev/null +++ b/packages/harbor-manager/src/experiments.ts @@ -0,0 +1,167 @@ +/** + * Experiment layer: groups runs that share a job-name prefix (`sb2-n8`, + * `sb2-gemini` → experiment `sb2`) so comparable arms can be charted together, + * with linear projections for arms still in flight. + */ +import type { RunRow, RunStore, TrialRow } from "./store"; + +/** Linear extrapolation of a running arm to its full task count. */ +export interface ArmProjection { + /** Expected finish timestamp (ms epoch), from observed completion rate. */ + etaMs: number | null; + passPct: number; + costPerTask: number; + totalCostUsd: number; + meanTrialMs: number; +} + +export interface ArmSummary { + run: RunRow; + /** Arm label: job name minus the experiment prefix. */ + arm: string; + /** Human config line: models plus slide description when known. */ + config: string; + /** Observed pass% over decided trials. */ + passPct: number | null; + costPerTask: number | null; + meanTrialMs: number | null; + /** Present only while the arm is running with at least one decided trial. */ + projected: ArmProjection | null; +} + +export interface ExperimentSummary { + id: string; + goal: string; + arms: number; + runningArms: number; + datasets: string[]; + nTotal: number; + done: number; + pass: number; + fail: number; + error: number; + costUsd: number; + createdAt: number; + updatedAt: number; +} + +export interface ExperimentDetail { + id: string; + goal: string; + arms: ArmSummary[]; + /** Union of task ids across arms, sorted. */ + tasks: string[]; + /** arm label → task → cell. */ + matrix: Record>; +} + +/** Experiment id = first `-`-delimited token of the job name. */ +export function experimentOf(jobName: string): string { + const dash = jobName.indexOf("-"); + return dash > 0 ? jobName.slice(0, dash) : jobName; +} + +/** Arm label = job name minus the experiment prefix (falls back to the full name). */ +export function armOf(jobName: string): string { + const exp = experimentOf(jobName); + return jobName.length > exp.length ? jobName.slice(exp.length + 1) : jobName; +} + +function slideLabel(slideJson: string | null): string { + if (!slideJson) return ""; + try { + const slide = JSON.parse(slideJson) as { model?: string; turns?: number; onAction?: boolean; plan?: boolean }; + if (!slide.model) return ""; + const trigger = slide.onAction ? "on first edit/write" : `after ${slide.turns} turns`; + return ` → ${slide.model} ${trigger}${slide.plan ? " +plan" : ""}`; + } catch { + return ""; + } +} + +export function summarizeArm(run: RunRow, trials: TrialRow[]): ArmSummary { + const decided = trials.filter(t => t.status === "pass" || t.status === "fail" || t.status === "error"); + const durations = decided.filter(t => t.durationMs > 0).map(t => t.durationMs); + const meanTrialMs = durations.length > 0 ? durations.reduce((a, b) => a + b, 0) / durations.length : null; + const passPct = decided.length > 0 ? (100 * run.pass) / decided.length : null; + const costPerTask = run.done > 0 ? run.costUsd / run.done : null; + + let projected: ArmProjection | null = null; + if (run.status === "running" && run.done > 0 && run.nTotal > run.done) { + const elapsed = Date.now() - run.createdAt; + const rate = run.done / Math.max(elapsed, 1); + const remaining = run.nTotal - run.done; + projected = { + etaMs: rate > 0 ? Date.now() + remaining / rate : null, + passPct: (100 * run.pass) / run.done, + costPerTask: run.costUsd / run.done, + totalCostUsd: (run.costUsd / run.done) * run.nTotal, + meanTrialMs: meanTrialMs ?? 0, + }; + } + return { + run, + arm: armOf(run.jobName), + config: `${run.models}${slideLabel(run.slide)}`, + passPct, + costPerTask, + meanTrialMs, + projected, + }; +} + +export function buildExperiments(store: RunStore): ExperimentSummary[] { + const groups = new Map(); + for (const run of store.listRuns()) { + const id = experimentOf(run.jobName); + let bucket = groups.get(id); + if (!bucket) { + bucket = []; + groups.set(id, bucket); + } + bucket.push(run); + } + const out: ExperimentSummary[] = []; + for (const [id, runs] of groups) { + out.push({ + id, + goal: store.getExperimentGoal(id), + arms: runs.length, + runningArms: runs.filter(r => r.status === "running").length, + datasets: [...new Set(runs.map(r => r.dataset).filter(Boolean))], + nTotal: runs.reduce((a, r) => a + r.nTotal, 0), + done: runs.reduce((a, r) => a + r.done, 0), + pass: runs.reduce((a, r) => a + r.pass, 0), + fail: runs.reduce((a, r) => a + r.fail, 0), + error: runs.reduce((a, r) => a + r.error, 0), + costUsd: runs.reduce((a, r) => a + r.costUsd, 0), + createdAt: Math.min(...runs.map(r => r.createdAt)), + updatedAt: Math.max(...runs.map(r => r.finishedAt ?? Date.now())), + }); + } + out.sort((a, b) => b.updatedAt - a.updatedAt); + return out; +} + +export function experimentDetail(store: RunStore, id: string): ExperimentDetail | null { + const runs = store.listRuns().filter(r => experimentOf(r.jobName) === id); + if (runs.length === 0) return null; + const arms: ArmSummary[] = []; + const matrix: ExperimentDetail["matrix"] = {}; + const tasks = new Set(); + for (const run of runs) { + const trials = store.listTrials(run.jobName); + arms.push(summarizeArm(run, trials)); + const cells: Record = {}; + for (const t of trials) { + tasks.add(t.task); + cells[t.task] = { status: t.status, reward: t.reward }; + } + matrix[armOf(run.jobName)] = cells; + } + // Baselines first, then variants, then untagged — the table reads as + // "reference rows, then treatments". + const roleRank = (role: string) => (role === "baseline" ? 0 : role === "variant" ? 1 : 2); + arms.sort((a, b) => roleRank(a.run.role) - roleRank(b.run.role) || a.arm.localeCompare(b.arm)); + return { id, goal: store.getExperimentGoal(id), arms, tasks: [...tasks].sort(), matrix }; +} diff --git a/packages/harbor-manager/src/manager.test.ts b/packages/harbor-manager/src/manager.test.ts new file mode 100644 index 000000000..26974c862 --- /dev/null +++ b/packages/harbor-manager/src/manager.test.ts @@ -0,0 +1,218 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { experimentDetail } from "./experiments"; +import { ManagerServer } from "./server"; +import { RunStore } from "./store"; + +/** + * Contracts under test: + * - discover() backfills historical job dirs into run rows. + * - syncRun() mirrors trial outcomes (pass / error / running) and rollups. + * - REST API surfaces runs, trials, compact transcripts, and rejects bad launches. + */ + +const cleanups: Array<() => void> = []; +afterEach(() => { + while (cleanups.length) cleanups.pop()?.(); +}); + +function makeJobsDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "harbor-manager-test-")); + cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +function writeFixtureJob(jobsDir: string, jobName: string): void { + const jobDir = path.join(jobsDir, jobName); + fs.mkdirSync(jobDir, { recursive: true }); + fs.writeFileSync( + path.join(jobDir, "result.json"), + JSON.stringify({ + n_total_trials: 3, + stats: { n_running_trials: 1, n_pending_trials: 0 }, + }), + ); + fs.writeFileSync( + path.join(jobDir, "config.json"), + JSON.stringify({ + dataset: "test-dataset@1.0", + agents: [{ name: "omp", model_name: "anthropic/claude-opus-4-8" }], + }), + ); + const mkTrial = (name: string, body: Record | null) => { + const dir = path.join(jobDir, name, "agent"); + fs.mkdirSync(dir, { recursive: true }); + if (body) fs.writeFileSync(path.join(jobDir, name, "result.json"), JSON.stringify(body)); + }; + mkTrial("alpha__abc", { + started_at: "2026-07-12T10:00:00", + finished_at: "2026-07-12T10:05:00", + verifier_result: { rewards: { reward: 1 } }, + agent_result: { cost_usd: 0.5, n_input_tokens: 100, n_output_tokens: 10, n_cache_tokens: 80 }, + }); + mkTrial("beta__def", { + started_at: "2026-07-12T10:00:00", + finished_at: "2026-07-12T10:02:00", + exception_info: { exception_type: "AgentTimeoutError" }, + agent_result: { cost_usd: 0.2 }, + }); + mkTrial("gamma__ghi", null); // running: no result.json yet + // transcript for alpha + const transcript = [ + JSON.stringify({ + type: "message_end", + message: { + role: "assistant", + model: "claude-opus-4-8", + content: [ + { type: "text", text: "Reading the file first." }, + { type: "toolCall", id: "t1", name: "read", arguments: { path: "x" } }, + ], + }, + }), + JSON.stringify({ + type: "message_end", + message: { + role: "toolResult", + toolName: "read", + isError: false, + content: [{ type: "text", text: "file contents" }], + }, + }), + ].join("\n"); + fs.writeFileSync(path.join(jobDir, "alpha__abc", "agent", "omp.txt"), transcript); +} + +describe("RunStore", () => { + it("discovers historical job dirs and mirrors trial state", () => { + const jobsDir = makeJobsDir(); + writeFixtureJob(jobsDir, "job-a"); + const store = new RunStore(jobsDir); + cleanups.push(() => store.close()); + + expect(store.discover()).toBe(1); + const run = store.getRun("job-a"); + // No job-level finished_at + fresh dir + a running trial → still running. + expect(run?.status).toBe("running"); + expect(run?.dataset).toBe("test-dataset@1.0"); + expect(run?.models).toBe("anthropic/claude-opus-4-8"); + expect(run?.nTotal).toBe(3); + expect(run?.pass).toBe(1); + expect(run?.error).toBe(1); + expect(run?.running).toBe(1); + expect(run?.costUsd).toBeCloseTo(0.7, 5); + + const trials = store.listTrials("job-a"); + expect(trials.map(t => [t.task, t.status])).toEqual([ + ["alpha", "pass"], + ["beta", "error"], + ["gamma", "running"], + ]); + expect(trials[1].detail).toBe("AgentTimeoutError"); + + // re-discover is idempotent + expect(store.discover()).toBe(0); + }); + + it("marks discovered runs complete when harbor recorded a terminal state", () => { + const jobsDir = makeJobsDir(); + writeFixtureJob(jobsDir, "job-done"); + const jobDir = path.join(jobsDir, "job-done"); + fs.writeFileSync( + path.join(jobDir, "result.json"), + JSON.stringify({ + n_total_trials: 2, + finished_at: "2026-07-12T11:00:00", + stats: { n_running_trials: 0, n_pending_trials: 0 }, + }), + ); + fs.rmSync(path.join(jobDir, "gamma__ghi"), { recursive: true, force: true }); + const store = new RunStore(jobsDir); + cleanups.push(() => store.close()); + store.discover(); + expect(store.getRun("job-done")?.status).toBe("complete"); + expect(store.getRun("job-done")?.finishedAt).not.toBeNull(); + }); + + it("stores experiment goals and run roles, and orders baselines first", () => { + const jobsDir = makeJobsDir(); + writeFixtureJob(jobsDir, "exp-treat"); + writeFixtureJob(jobsDir, "exp-base"); + const store = new RunStore(jobsDir); + cleanups.push(() => store.close()); + store.discover(); + store.setExperimentGoal("exp", "does the treatment beat the baseline?"); + expect(store.setRunMeta("exp-base", { role: "baseline", note: "plain model" })).toBe(true); + expect(store.setRunMeta("exp-treat", { role: "variant", note: "slide N=8" })).toBe(true); + expect(store.setRunMeta("exp-missing", { role: "variant" })).toBe(false); + + const detail = experimentDetail(store, "exp"); + expect(detail?.goal).toBe("does the treatment beat the baseline?"); + expect(detail?.arms.map(a => [a.arm, a.run.role, a.run.note])).toEqual([ + ["base", "baseline", "plain model"], + ["treat", "variant", "slide N=8"], + ]); + }); + + it("finalizes running rows whose owning process died", () => { + const jobsDir = makeJobsDir(); + writeFixtureJob(jobsDir, "job-b"); + const store = new RunStore(jobsDir); + cleanups.push(() => store.close()); + store.registerLaunch({ + jobName: "job-b", + dataset: "test-dataset@1.0", + agent: "omp", + models: ["m"], + pid: 999999999, // certainly dead + }); + const rows = store.syncActive(); + expect(rows).toHaveLength(1); + expect(store.getRun("job-b")?.status).toBe("failed"); + }); +}); + +describe("ManagerServer API", () => { + it("serves runs, trials, transcripts, and validates launches", async () => { + const jobsDir = makeJobsDir(); + writeFixtureJob(jobsDir, "job-api"); + const manager = new ManagerServer(jobsDir); + const server = manager.start(0); + cleanups.push(() => { + void manager.stop(); + }); + const base = `http://localhost:${server.port}`; + + const runs = (await (await fetch(`${base}/api/runs`)).json()) as Array<{ jobName: string; pass: number }>; + expect(runs.map(r => r.jobName)).toContain("job-api"); + + const detailRes = await fetch(`${base}/api/runs/job-api`); + expect(detailRes.status).toBe(200); + const detail = (await detailRes.json()) as { run: { pass: number }; trials: Array<{ status: string }> }; + expect(detail.run.pass).toBe(1); + expect(detail.trials).toHaveLength(3); + + const tr = await fetch(`${base}/api/runs/job-api/trials/alpha__abc/transcript?tail=10`); + expect(tr.status).toBe(200); + const transcript = (await tr.json()) as { entries: Array<{ kind: string; tools?: string[] }> }; + expect(transcript.entries.map(e => e.kind)).toEqual(["assistant", "toolResult"]); + expect(transcript.entries[0].tools).toEqual(["read"]); + + const missing = await fetch(`${base}/api/runs/nope`); + expect(missing.status).toBe(404); + + const badLaunch = await fetch(`${base}/api/runs`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }); + expect(badLaunch.status).toBe(400); + + const cancelUnknown = (await (await fetch(`${base}/api/runs/nope`, { method: "DELETE" })).json()) as { + cancelled: boolean; + }; + expect(cancelUnknown.cancelled).toBe(false); + }); +}); diff --git a/packages/harbor-manager/src/runner.test.ts b/packages/harbor-manager/src/runner.test.ts new file mode 100644 index 000000000..8464823c6 --- /dev/null +++ b/packages/harbor-manager/src/runner.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "bun:test"; +import { buildHarborEnv, collectForwardEnv, parseArgs } from "./runner"; + +describe("generic agent-arg / env passthrough", () => { + it("forwards repeated --agent-arg as a JSON array the in-container agent can parse", () => { + const cfg = parseArgs([ + "--model", + "anthropic/claude-opus-4-8", + "--agent-arg", + "--reasoning-slide-model", + "--agent-arg", + "google/gemini-3.5-flash", + "--agent-arg", + "--reasoning-slide-turns", + "--agent-arg", + "3", + ]); + expect(cfg.agentArgs).toEqual([ + "--reasoning-slide-model", + "google/gemini-3.5-flash", + "--reasoning-slide-turns", + "3", + ]); + + const env = buildHarborEnv(cfg, "/tmp/models.yml", null, "test"); + expect(JSON.parse(env.OMP_BENCH_AGENT_ARGS ?? "[]")).toEqual(cfg.agentArgs); + }); + + it("omits OMP_BENCH_AGENT_ARGS when no --agent-arg was passed", () => { + const cfg = parseArgs(["--model", "anthropic/claude-opus-4-8"]); + const env = buildHarborEnv(cfg, "/tmp/models.yml", null, "test"); + expect(env.OMP_BENCH_AGENT_ARGS).toBeUndefined(); + }); + + it("routes an explicit --providers entry alongside the model's own provider", () => { + // The runner has no built-in concept of a "second model"; gateway routing + // for any extra model introduced via --agent-arg must be declared + // explicitly via --providers. + const cfg = parseArgs(["--model", "anthropic/claude-opus-4-8", "--providers", "google"]); + const env = buildHarborEnv(cfg, "/tmp/models.yml", null, "test"); + expect(new Set(env.OMP_BENCH_GATEWAY_PROVIDERS?.split(","))).toEqual(new Set(["anthropic", "google"])); + }); + + it("collects explicit --env pairs, with an explicit value winning over a bare host-forwarded key", () => { + const cfg = parseArgs([ + "--model", + "anthropic/claude-opus-4-8", + "--env", + "SOME_FLAG=1", + "--env", + "OTHER=two words", + ]); + const forwarded = collectForwardEnv(cfg); + expect(forwarded.SOME_FLAG).toBe("1"); + expect(forwarded.OTHER).toBe("two words"); + }); +}); + +describe("install modes", () => { + it("defaults to source mode and publishes the mount contract to the agent", () => { + const cfg = parseArgs(["--model", "anthropic/claude-opus-4-8"]); + expect(cfg.install).toBe("source"); + const env = buildHarborEnv(cfg, "/tmp/models.yml", null, "test", { + arch: "arm64", + depsDir: "/tmp/deps", + nodeModules: ["node_modules"], + }); + expect(env.OMP_BENCH_INSTALL).toBe("source"); + expect(env.OMP_BENCH_SOURCE_DIR).toBe("/opt/omp/src"); + expect(env.OMP_BENCH_SOURCE_BUN).toBe("/opt/omp/bin/bun"); + expect(env.OMP_BENCH_SOURCE_ARCH).toBe("arm64"); + }); + + it("omits source mount env when no mount was prepared (binary/local runs)", () => { + const cfg = parseArgs(["--model", "anthropic/claude-opus-4-8", "--install", "local"]); + const env = buildHarborEnv(cfg, "/tmp/models.yml", "/tmp/omp.tgz", "test"); + expect(env.OMP_BENCH_INSTALL).toBe("local"); + expect(env.OMP_BENCH_SOURCE_DIR).toBeUndefined(); + expect(env.OMP_BENCH_SOURCE_ARCH).toBeUndefined(); + }); + + it("--tarball implies a local (tarball) install", () => { + const cfg = parseArgs(["--model", "anthropic/claude-opus-4-8", "--tarball", "/tmp/omp.tgz"]); + expect(cfg.install).toBe("local"); + expect(cfg.build).toBe(false); + }); +}); + +describe("parseArgs validation", () => { + it("rejects an unknown flag", () => { + expect(() => parseArgs(["--model", "anthropic/claude-opus-4-8", "--not-a-real-flag"])).toThrow(/unknown flag/); + }); + + it("defaults to a generic, dataset-agnostic jobs directory", () => { + const cfg = parseArgs(["--model", "anthropic/claude-opus-4-8"]); + expect(cfg.jobsDir.endsWith("/runs/harbor")).toBe(true); + }); +}); diff --git a/packages/terminal-bench/src/runner.ts b/packages/harbor-manager/src/runner.ts similarity index 72% rename from packages/terminal-bench/src/runner.ts rename to packages/harbor-manager/src/runner.ts index 2aa0ee02c..a716fd39e 100755 --- a/packages/terminal-bench/src/runner.ts +++ b/packages/harbor-manager/src/runner.ts @@ -1,11 +1,11 @@ #!/usr/bin/env bun /** - * terminal-bench-2 runner for the local `omp` build. + * Harbor benchmark runner for the local `omp` build. * - * Orchestrates Harbor (`harbor run`) against the harbor-framework/terminal-bench-2 - * dataset using a custom agent (`agent/omp_local.py`) that installs the working - * tree at /work/pi and routes all model auth through the host pm2 auth-gateway - * (no provider keys ever enter the task containers). + * Orchestrates Harbor (`harbor run`) against any Harbor dataset (default + * terminal-bench-2) using a custom agent (`agent/omp_local.py`) that installs + * the working tree at /work/pi and routes all model auth through the host pm2 + * auth-gateway (no provider keys ever enter the task containers). * * It owns the terminal: Harbor's own output is redirected to a log file and this * process renders a live dashboard (progress / success% / spend / tokens / ETA) @@ -27,6 +27,10 @@ const AGENT_DIR = path.join(PKG_DIR, "agent"); const CODING_AGENT_DIR = path.join(REPO_ROOT, "packages", "coding-agent"); const AGENT_IMPORT_PATH = "omp_local:OmpLocal"; +/** Container-side mount points for `--install source` (must match omp_local.py defaults). */ +const SOURCE_SRC_MOUNT = "/opt/omp/src"; +const SOURCE_BIN_MOUNT = "/opt/omp/bin"; + export interface Config { models: string[]; dataset: string; @@ -36,10 +40,11 @@ export interface Config { include: string[]; exclude: string[]; thinking: string | null; - advisorModel: string | null; - advisorSync: string; + /** Extra args forwarded verbatim to the in-container omp CLI invocation (repeatable). */ + agentArgs: string[]; + agent: string; - install: "local" | "published"; + install: "source" | "local" | "published"; version: string | null; tarball: string | null; binaryArm64: string | null; @@ -73,16 +78,16 @@ function defaultConfig(): Config { include: [], exclude: [], thinking: null, - advisorModel: null, - advisorSync: "1", + agentArgs: [], + agent: "omp", - install: "local", + install: "source", version: null, tarball: null, binaryArm64: null, binaryX64: null, build: true, - jobsDir: path.join(REPO_ROOT, "runs", "tb2"), + jobsDir: path.join(REPO_ROOT, "runs", "harbor"), jobName: null, gatewayUrl: "http://host.docker.internal:4000", gatewayToken: "no-auth", @@ -101,7 +106,7 @@ function defaultConfig(): Config { }; } -const HELP = `terminal-bench-2 runner (local omp) +const HELP = `harbor-manager runner (local omp) Usage: bun src/runner.ts [options] [-- ] @@ -111,13 +116,15 @@ Commands: Model / agent: -m, --model Model (repeatable). Default anthropic/claude-sonnet-4-6 --agent omp (default) | oracle | nop | any harbor agent - --install omp source. local = pack /work/pi (default) + --install omp install mode (default: source). + source = mount /work/pi read-only + prebuilt linux deps tree; TS changes + apply per-trial with no rebuild. local = pack a tarball. published = npm. --version omp version for published install (default: latest) --thinking off|minimal|low|medium|high|xhigh|max - --advisor-model

      Second model reviewing the primary (spend summed in) - --advisor-sync Advisor catch-up backlog (default 1 = accurate spend; off = faster) - --tarball Reuse a prebuilt omp tarball (implies --no-build) - --no-build Skip packing; reuse newest tarball in bench dir + + --tarball Reuse a prebuilt omp tarball (implies --install local, --no-build) + --no-build Skip packing; reuse newest tarball in bench dir (--install local) + --agent-arg Extra arg forwarded verbatim to the in-container omp CLI (repeatable) --env Forward env into omp container (repeatable). KEY alone forwards host value; host PI_* auto-forwarded. @@ -138,8 +145,8 @@ Gateway (auth, no keys in container): --allow-host harbor --allow-agent-host (repeatable) Output / control: - -o, --jobs-dir Default /runs/tb2 - --job-name Default tb2-- + -o, --jobs-dir Default /runs/harbor + --job-name Default - --dry-run Print the harbor command + models.yml and exit --cleanup Clean up stale and exited Harbor Docker resources safely before starting --cleanup-force Force-stop and remove ALL previous Harbor Docker containers and networks @@ -180,7 +187,9 @@ export function parseArgs(argv: string[]): Config { break; case "--install": { const v = take(arg); - if (v !== "local" && v !== "published") throw new Error("--install must be local|published"); + if (v !== "source" && v !== "local" && v !== "published") { + throw new Error("--install must be source|local|published"); + } cfg.install = v; break; } @@ -190,14 +199,9 @@ export function parseArgs(argv: string[]): Config { case "--thinking": cfg.thinking = take(arg); break; - case "--advisor-model": - cfg.advisorModel = take(arg); - break; - case "--advisor-sync": - cfg.advisorSync = take(arg); - break; case "--tarball": cfg.tarball = path.resolve(take(arg)); + cfg.install = "local"; cfg.build = false; break; case "--binary": { @@ -212,6 +216,9 @@ export function parseArgs(argv: string[]): Config { case "--no-build": cfg.build = false; break; + case "--agent-arg": + cfg.agentArgs.push(take(arg)); + break; case "-l": case "--tasks": case "--n-tasks": @@ -239,6 +246,7 @@ export function parseArgs(argv: string[]): Config { case "--dataset": cfg.dataset = take(arg); break; + case "--gateway-url": cfg.gatewayUrl = take(arg); break; @@ -357,16 +365,19 @@ function pad(s: string, w: number): string { return s.length >= w ? s.slice(0, w) : s + " ".repeat(w - s.length); } +function agentArgsLabel(cfg: Config): string | null { + return cfg.agentArgs.length > 0 ? cfg.agentArgs.join(" ") : null; +} + // ───────────────────────────────────────────────────────────── result parsing -type TrialStatus = "pass" | "fail" | "error" | "running"; +export type TrialStatus = "pass" | "fail" | "error" | "running"; -interface Trial { +export interface Trial { name: string; status: TrialStatus; reward: number | null; costUsd: number; - advisorCostUsd: number; tokIn: number; tokOut: number; tokCache: number; @@ -379,7 +390,6 @@ interface AgentCtxLike { n_cache_tokens?: unknown; n_output_tokens?: unknown; cost_usd?: unknown; - metadata?: unknown; } function num(v: unknown): number { @@ -457,7 +467,6 @@ function parseTrial(dir: string, name: string): Trial | null { status: "running", reward: null, costUsd, - advisorCostUsd: 0, tokIn, tokOut, tokCache, @@ -481,7 +490,6 @@ function parseTrial(dir: string, name: string): Trial | null { } } let costUsd = 0, - advisorCostUsd = 0, tokIn = 0, tokOut = 0, tokCache = 0; @@ -490,9 +498,6 @@ function parseTrial(dir: string, name: string): Trial | null { tokIn += num(ctx.n_input_tokens); tokOut += num(ctx.n_output_tokens); tokCache += num(ctx.n_cache_tokens); - if (ctx.metadata && typeof ctx.metadata === "object") { - advisorCostUsd += num((ctx.metadata as Record).advisor_cost_usd); - } } // rewards: top-level verifier_result, else step_results last verifier @@ -531,10 +536,10 @@ function parseTrial(dir: string, name: string): Trial | null { } else { status = "fail"; } - return { name, status, reward, costUsd, advisorCostUsd, tokIn, tokOut, tokCache, durationMs, detail }; + return { name, status, reward, costUsd, tokIn, tokOut, tokCache, durationMs, detail }; } -function readTrials(jobDir: string): Trial[] { +export function readTrials(jobDir: string): Trial[] { let entries: fs.Dirent[] = []; try { entries = fs.readdirSync(jobDir, { withFileTypes: true }); @@ -551,13 +556,15 @@ function readTrials(jobDir: string): Trial[] { } /** Authoritative job-level totals from /result.json (written incrementally). */ -interface JobInfo { +export interface JobInfo { nTotal: number; running: number | null; pending: number | null; + /** Harbor sets this only when the job reached a terminal state. */ + finishedAt: number | null; } -function readJobResult(jobDir: string): JobInfo | null { +export function readJobResult(jobDir: string): JobInfo | null { const raw = readJson(path.join(jobDir, "result.json")); if (!raw || typeof raw !== "object") return null; const r = raw as Record; @@ -569,12 +576,14 @@ function readJobResult(jobDir: string): JobInfo | null { if (typeof s.n_running_trials === "number") running = s.n_running_trials; if (typeof s.n_pending_trials === "number") pending = s.n_pending_trials; } - return nTotal > 0 ? { nTotal, running, pending } : null; + const finishedRaw = typeof r.finished_at === "string" ? Date.parse(r.finished_at) : NaN; + const finishedAt = Number.isFinite(finishedRaw) ? finishedRaw : null; + return nTotal > 0 ? { nTotal, running, pending, finishedAt } : null; } // ──────────────────────────────────────────────────────────────────── totals -interface Totals { +export interface Totals { total: number; done: number; pass: number; @@ -583,13 +592,13 @@ interface Totals { running: number; pending: number; costUsd: number; - advisorCostUsd: number; tokIn: number; tokOut: number; tokCache: number; + durationMs: number; } -function aggregate(trials: Trial[], job: JobInfo | null, fallbackExpected: number): Totals { +export function aggregate(trials: Trial[], job: JobInfo | null, fallbackExpected: number): Totals { const t: Totals = { total: fallbackExpected, done: 0, @@ -599,14 +608,13 @@ function aggregate(trials: Trial[], job: JobInfo | null, fallbackExpected: numbe running: 0, pending: 0, costUsd: 0, - advisorCostUsd: 0, tokIn: 0, tokOut: 0, tokCache: 0, + durationMs: 0, }; for (const tr of trials) { t.costUsd += tr.costUsd; - t.advisorCostUsd += tr.advisorCostUsd; t.tokIn += tr.tokIn; t.tokOut += tr.tokOut; t.tokCache += tr.tokCache; @@ -614,6 +622,8 @@ function aggregate(trials: Trial[], job: JobInfo | null, fallbackExpected: numbe t.running++; continue; } + t.durationMs += tr.durationMs; + t.done++; if (tr.status === "pass") t.pass++; else if (tr.status === "error") t.error++; @@ -671,8 +681,9 @@ function render(st: RenderState): void { const successPct = tot.done > 0 ? (tot.pass / tot.done) * 100 : 0; const rows: string[] = []; - const advisorTag = st.cfg.advisorModel ? `${dim(" + advisor ")}${st.cfg.advisorModel}` : ""; - const header = `${bold("terminal-bench-2")} ${dim("·")} ${cyan(st.cfg.agent)} ${dim("·")} ${st.cfg.models.join(",")}${advisorTag} ${dim(`· conc=${st.cfg.concurrency} k=${st.cfg.attempts}`)}`; + const argsLabel = agentArgsLabel(st.cfg); + const argsTag = argsLabel ? `${dim(" · args ")}${argsLabel}` : ""; + const header = `${bold(st.cfg.dataset)} ${dim("·")} ${cyan(st.cfg.agent)} ${dim("·")} ${st.cfg.models.join(",")}${argsTag} ${dim(`· conc=${st.cfg.concurrency} k=${st.cfg.attempts}`)}`; rows.push(header); const width = 28; rows.push( @@ -681,9 +692,8 @@ function render(st: RenderState): void { rows.push( `${green(`pass ${tot.pass}`)} ${dim(`(${successPct.toFixed(0)}%)`)} ${red(`fail ${tot.fail}`)} ${yellow(`err ${tot.error}`)} ${cyan(`run ${tot.running}`)} ${gray(`pend ${tot.pending}`)}`, ); - const advisorSpend = tot.advisorCostUsd > 0 ? dim(` (advisor ${fmtUsd(tot.advisorCostUsd)})`) : ""; rows.push( - `${bold("spend")} ${fmtUsd(tot.costUsd)}${advisorSpend} ${dim("in")} ${fmtNum(tot.tokIn)} ${dim("out")} ${fmtNum(tot.tokOut)} ${dim("cache")} ${fmtNum(tot.tokCache)}`, + `${bold("spend")} ${fmtUsd(tot.costUsd)} ${dim("in")} ${fmtNum(tot.tokIn)} ${dim("out")} ${fmtNum(tot.tokOut)} ${dim("cache")} ${fmtNum(tot.tokCache)}`, ); rows.push(dim("─".repeat(54))); @@ -709,7 +719,7 @@ function render(st: RenderState): void { process.stdout.write(out); } else { process.stdout.write( - `[tb2] ${tot.done}/${tot.total} pass=${tot.pass}(${successPct.toFixed(0)}%) fail=${tot.fail} err=${tot.error} run=${tot.running} spend=${fmtUsd(tot.costUsd)} elapsed=${fmtDur(elapsed)}\n`, + `[harbor] ${tot.done}/${tot.total} pass=${tot.pass}(${successPct.toFixed(0)}%) fail=${tot.fail} err=${tot.error} run=${tot.running} spend=${fmtUsd(tot.costUsd)} elapsed=${fmtDur(elapsed)}\n`, ); } } @@ -722,11 +732,10 @@ function writeReport(st: RenderState, benchDir: string, exitCode: number): strin const successPct = tot.done > 0 ? (tot.pass / tot.done) * 100 : 0; const lines: string[] = []; const isOmp = st.cfg.agent === "omp"; - const modelLine = - isOmp && st.cfg.advisorModel - ? `${st.cfg.models.join(", ")} + advisor ${st.cfg.advisorModel}` - : st.cfg.models.join(", "); - lines.push(`# terminal-bench-2 — ${st.cfg.agent} — ${modelLine}`); + const argsLabel = agentArgsLabel(st.cfg); + const baseModelLine = st.cfg.models.join(", "); + const modelLine = argsLabel ? `${baseModelLine} (${argsLabel})` : baseModelLine; + lines.push(`# ${st.cfg.dataset} — ${st.cfg.agent} — ${modelLine}`); lines.push(""); lines.push(`- dataset: \`${st.cfg.dataset}\``); lines.push(`- tasks: ${st.cfg.tasks} · attempts: ${st.cfg.attempts} · concurrency: ${st.cfg.concurrency}`); @@ -735,13 +744,12 @@ function writeReport(st: RenderState, benchDir: string, exitCode: number): strin `- install: ${st.cfg.install} · auth: ${st.cfg.gateway ? "host gateway (no keys in container)" : "direct provider keys"}`, ); lines.push(`- tools: web_search=${st.cfg.webSearch ? "on" : "off"}`); - if (st.cfg.advisorModel) lines.push(`- advisor: ${st.cfg.advisorModel}`); + if (argsLabel) lines.push(`- agent args: ${argsLabel}`); } lines.push(`- elapsed: ${fmtDur(Date.now() - st.startMs)} · harbor exit: ${exitCode}`); lines.push(""); - const advisorSpend = tot.advisorCostUsd > 0 ? ` (advisor ${fmtUsd(tot.advisorCostUsd)})` : ""; lines.push( - `**${tot.pass}/${tot.done} passed (${successPct.toFixed(1)}%)** · fail ${tot.fail} · error ${tot.error} · spend ${fmtUsd(tot.costUsd)}${advisorSpend}`, + `**${tot.pass}/${tot.done} passed (${successPct.toFixed(1)}%)** · fail ${tot.fail} · error ${tot.error} · spend ${fmtUsd(tot.costUsd)}`, ); lines.push(`tokens: in ${fmtNum(tot.tokIn)} · out ${fmtNum(tot.tokOut)} · cache ${fmtNum(tot.tokCache)}`); lines.push(""); @@ -816,16 +824,176 @@ function newestTarball(benchDir: string): string | null { } } +// ─────────────────────────────────────────────────────── source mount (--install source) + +/** Linux deps tree + mount plan for running omp straight from the mounted repo. */ +export interface SourceMount { + arch: "arm64" | "x64"; + /** Host dir holding the linux `bin/bun` + skeleton `node_modules` trees. */ + depsDir: string; + /** Repo-relative node_modules dirs to shadow-mount over the darwin ones. */ + nodeModules: string[]; +} + +/** Bun version pinned by the repo's `packageManager` field. */ +function repoBunVersion(): string { + const raw = readJson(path.join(REPO_ROOT, "package.json")); + if (raw && typeof raw === "object") { + const pm = (raw as Record).packageManager; + if (typeof pm === "string" && pm.startsWith("bun@")) return pm.slice("bun@".length); + } + return "1.3.14"; +} + +/** Native arch of the docker daemon (what non-emulated task containers run as). */ +function dockerServerArch(): "arm64" | "x64" { + const r = spawnSync("docker", ["version", "--format", "{{.Server.Arch}}"], { encoding: "utf8" }); + const a = (r.stdout ?? "").trim(); + if (a === "arm64" || a === "aarch64") return "arm64"; + if (a === "amd64" || a === "x86_64") return "x64"; + throw new Error(`cannot detect docker server arch (got ${a || "nothing"}); is docker running?`); +} + +/** Workspace member dirs (repo-relative), expanded from root package.json `workspaces.packages`. */ +function workspacePackageDirs(): string[] { + const raw = readJson(path.join(REPO_ROOT, "package.json")) as { + workspaces?: { packages?: string[] }; + } | null; + const dirs = new Set(); + for (const pattern of raw?.workspaces?.packages ?? []) { + for (const match of new Bun.Glob(`${pattern}/package.json`).scanSync({ cwd: REPO_ROOT })) { + dirs.add(path.dirname(match)); + } + } + return [...dirs].sort(); +} + +/** Manifest files (repo-relative) that fully determine a `bun install` result. */ +function sourceManifestFiles(pkgDirs: string[]): string[] { + const files = ["package.json", "bun.lock"]; + if (fs.existsSync(path.join(REPO_ROOT, "bunfig.toml"))) files.push("bunfig.toml"); + const patchesDir = path.join(REPO_ROOT, "patches"); + if (fs.existsSync(patchesDir)) { + for (const f of fs.readdirSync(patchesDir).sort()) files.push(path.join("patches", f)); + } + for (const dir of pkgDirs) files.push(path.join(dir, "package.json")); + return files; +} + +function sourceDepsStamp(manifests: string[], bunVersion: string): string { + const h = new Bun.CryptoHasher("sha256"); + h.update(`bun@${bunVersion}\0source-deps-v1\0`); + for (const rel of manifests) { + h.update(rel); + h.update("\0"); + h.update(fs.readFileSync(path.join(REPO_ROOT, rel))); + h.update("\0"); + } + return h.digest("hex"); +} + +/** + * Ensure the cached linux deps tree for source mode: a manifest-only skeleton of the + * workspace with `bun install --production` run inside `oven/bun:` (matching the + * daemon's native arch), plus the image's linux `bun` under `bin/`. Rebuilt only when + * a manifest/lockfile or the pinned bun version changes; TS edits never invalidate it. + */ +export function prepareSourceDeps(cfg: Config): SourceMount { + const arch = dockerServerArch(); + const bunVersion = repoBunVersion(); + const depsDir = path.join(cfg.jobsDir, "_bench", "_deps", `linux-${arch}`); + const pkgDirs = workspacePackageDirs(); + const manifests = sourceManifestFiles(pkgDirs); + const stamp = sourceDepsStamp(manifests, bunVersion); + const stampFile = path.join(depsDir, ".stamp"); + let current: string | null = null; + try { + current = fs.readFileSync(stampFile, "utf8").trim(); + } catch { + /* no stamp yet */ + } + if (current !== stamp) { + process.stdout.write(dim(`building linux-${arch} deps tree for source mount (one-time per lockfile change)…\n`)); + fs.rmSync(depsDir, { recursive: true, force: true }); + fs.mkdirSync(depsDir, { recursive: true }); + for (const rel of manifests) { + const dst = path.join(depsDir, rel); + fs.mkdirSync(path.dirname(dst), { recursive: true }); + fs.copyFileSync(path.join(REPO_ROOT, rel), dst); + } + // --ignore-scripts: the skeleton has manifests only, so lifecycle scripts + // (root `prepare` → gen:tool-views) would fail; patchedDependencies still apply. + const script = + 'mkdir -p /deps/bin && cp "$(command -v bun)" /deps/bin/bun && cd /deps && bun install --production --omit=optional --ignore-scripts'; + const r = spawnSync( + "docker", + [ + "run", + "--rm", + "--platform", + `linux/${arch === "x64" ? "amd64" : "arm64"}`, + "-e", + "HOME=/tmp", + "-v", + `${depsDir}:/deps`, + `oven/bun:${bunVersion}`, + "sh", + "-c", + script, + ], + { stdio: ["ignore", "inherit", "inherit"] }, + ); + if (r.status !== 0) { + fs.rmSync(stampFile, { force: true }); + throw new Error(`source deps install failed (docker exit ${r.status})`); + } + fs.writeFileSync(stampFile, `${stamp}\n`); + } + if (!fs.existsSync(path.join(depsDir, "node_modules"))) { + throw new Error(`source deps tree has no node_modules (${depsDir}); delete it and retry`); + } + // Shadow-mount every node_modules visible in the host tree (they hold darwin + // binaries) with the skeleton's linux one; both sides of each mount must exist. + const nodeModules = ["node_modules"]; + for (const dir of pkgDirs) { + const rel = path.join(dir, "node_modules"); + const inHost = fs.existsSync(path.join(REPO_ROOT, rel)); + const inDeps = fs.existsSync(path.join(depsDir, rel)); + if (!inHost && !inDeps) continue; + if (!inDeps) fs.mkdirSync(path.join(depsDir, rel), { recursive: true }); + if (!inHost) fs.mkdirSync(path.join(REPO_ROOT, rel), { recursive: true }); + nodeModules.push(rel); + } + return { arch, depsDir, nodeModules }; +} + +/** + * Compose overlay applied to every trial's `main` service: host networking and/or the + * read-only source + linux-deps mounts. Returns null when nothing needs overlaying. + */ +function writeComposeOverlay(benchDir: string, cfg: Config, source: SourceMount | null): string | null { + const lines: string[] = []; + if (cfg.hostNetwork) lines.push(' network_mode: "host"'); + if (source) { + lines.push(" volumes:"); + lines.push(` - ${REPO_ROOT}:${SOURCE_SRC_MOUNT}:ro`); + for (const rel of source.nodeModules) { + lines.push(` - ${path.join(source.depsDir, rel)}:${SOURCE_SRC_MOUNT}/${rel}:ro`); + } + lines.push(` - ${path.join(source.depsDir, "bin")}:${SOURCE_BIN_MOUNT}:ro`); + } + if (lines.length === 0) return null; + const file = path.join(benchDir, "omp-compose-overlay.yaml"); + fs.writeFileSync(file, `${["services:", " main:", ...lines].join("\n")}\n`); + return file; +} + function deriveProviders(cfg: Config): string[] { const set = new Set(cfg.providers); for (const m of cfg.models) { const slash = m.indexOf("/"); if (slash > 0) set.add(m.slice(0, slash)); } - if (cfg.advisorModel) { - const slash = cfg.advisorModel.indexOf("/"); - if (slash > 0) set.add(cfg.advisorModel.slice(0, slash)); - } if (set.size === 0) { set.add("anthropic"); set.add("openai-codex"); @@ -835,7 +1003,7 @@ function deriveProviders(cfg: Config): string[] { function writeModelsYaml(benchDir: string, cfg: Config): string { const providers = deriveProviders(cfg); - const lines = ["# Generated by terminal-bench runner — auth via host pm2 gateway.", "providers:"]; + const lines = ["# Generated by harbor-manager — auth via host pm2 gateway.", "providers:"]; for (const p of providers) { lines.push(` ${p}:`); lines.push(` baseUrl: ${cfg.gatewayUrl}`); @@ -859,7 +1027,7 @@ function buildHarborArgs( jobName: string, modelsYaml: string, tarball: string | null, - hostNetworkOverlayPath: string | null, + composeOverlayPath: string | null, ): string[] { const a: string[] = ["run", "-d", cfg.dataset, "-o", cfg.jobsDir, "--job-name", jobName]; a.push("-n", String(cfg.concurrency), "-k", String(cfg.attempts), "-l", String(cfg.tasks)); @@ -869,12 +1037,12 @@ function buildHarborArgs( for (const h of cfg.allowHosts) a.push("--allow-agent-host", h); if (cfg.timeoutMultiplier !== null) a.push("--timeout-multiplier", String(cfg.timeoutMultiplier)); if (cfg.yes) a.push("-y"); - if (hostNetworkOverlayPath) { - a.push("--extra-docker-compose", hostNetworkOverlayPath); + if (composeOverlayPath) { + a.push("--extra-docker-compose", composeOverlayPath); } if (cfg.agent === "omp") { - // Config + secrets travel via env (OMP_TB_*); the agent reads os.environ. + // Config + secrets travel via env (OMP_BENCH_*); the agent reads os.environ. a.push("--agent-import-path", AGENT_IMPORT_PATH); void modelsYaml; void tarball; @@ -918,36 +1086,39 @@ export function buildHarborEnv( modelsYaml: string, tarball: string | null, version: string, + source: SourceMount | null = null, ): Record { const env: Record = { ...(process.env as Record) }; - // Drop any stale OMP_TB_FORWARD_ENV inherited from the caller's shell before + // Drop any stale OMP_BENCH_FORWARD_ENV inherited from the caller's shell before // the agent-type early return, so it never leaks (incl. into the dry-run dump). - delete env.OMP_TB_FORWARD_ENV; + delete env.OMP_BENCH_FORWARD_ENV; if (cfg.agent !== "omp") return env; const prepend = (k: string, v: string): void => { env[k] = env[k] ? `${v}:${env[k]}` : v; }; prepend("PYTHONPATH", AGENT_DIR); - env.OMP_TB_INSTALL = cfg.install; - env.OMP_TB_VERSION = cfg.version ?? version; - if (tarball) env.OMP_TB_TARBALL = tarball; - if (cfg.binaryArm64) env.OMP_TB_BINARY_ARM64 = cfg.binaryArm64; - if (cfg.binaryX64) env.OMP_TB_BINARY_X64 = cfg.binaryX64; - if (cfg.thinking) env.OMP_TB_THINKING = cfg.thinking; - if (cfg.advisorModel) { - env.OMP_TB_ADVISOR_MODEL = cfg.advisorModel; - env.OMP_TB_ADVISOR_SYNC = cfg.advisorSync; + env.OMP_BENCH_INSTALL = cfg.install; + env.OMP_BENCH_VERSION = cfg.version ?? version; + if (tarball) env.OMP_BENCH_TARBALL = tarball; + if (source) { + env.OMP_BENCH_SOURCE_DIR = SOURCE_SRC_MOUNT; + env.OMP_BENCH_SOURCE_BUN = `${SOURCE_BIN_MOUNT}/bun`; + env.OMP_BENCH_SOURCE_ARCH = source.arch; } - if (cfg.webSearch) env.OMP_TB_WEB_SEARCH = "1"; - env.OMP_TB_GATEWAY = cfg.gateway ? "1" : "0"; + if (cfg.binaryArm64) env.OMP_BENCH_BINARY_ARM64 = cfg.binaryArm64; + if (cfg.binaryX64) env.OMP_BENCH_BINARY_X64 = cfg.binaryX64; + if (cfg.thinking) env.OMP_BENCH_THINKING = cfg.thinking; + if (cfg.agentArgs.length > 0) env.OMP_BENCH_AGENT_ARGS = JSON.stringify(cfg.agentArgs); + if (cfg.webSearch) env.OMP_BENCH_WEB_SEARCH = "1"; + env.OMP_BENCH_GATEWAY = cfg.gateway ? "1" : "0"; if (cfg.gateway) { - env.OMP_TB_MODELS_YAML = modelsYaml; - env.OMP_TB_GATEWAY_URL = cfg.gatewayUrl; - env.OMP_TB_GATEWAY_TOKEN = cfg.gatewayToken; - env.OMP_TB_GATEWAY_PROVIDERS = deriveProviders(cfg).join(","); + env.OMP_BENCH_MODELS_YAML = modelsYaml; + env.OMP_BENCH_GATEWAY_URL = cfg.gatewayUrl; + env.OMP_BENCH_GATEWAY_TOKEN = cfg.gatewayToken; + env.OMP_BENCH_GATEWAY_PROVIDERS = deriveProviders(cfg).join(","); } const forward = collectForwardEnv(cfg); - if (Object.keys(forward).length > 0) env.OMP_TB_FORWARD_ENV = JSON.stringify(forward); + if (Object.keys(forward).length > 0) env.OMP_BENCH_FORWARD_ENV = JSON.stringify(forward); return env; } @@ -1056,15 +1227,18 @@ function runDockerCleanup(force: boolean): void { // ──────────────────────────────────────────────────────────────────────── main -async function main(): Promise { - const argv = process.argv.slice(2); - if (argv[0] === "cleanup") { - if (!which("docker")) throw new Error("docker not found on PATH (required for cleanup)."); - runDockerCleanup(true); - return; - } - const cfg = parseArgs(argv); +interface BenchmarkRun { + exitCode: number; + jobName: string; + jobDir: string; + benchDir: string; + tarball: string | null; + elapsedMs: number; + totals: Totals | null; + reportPath: string | null; +} +async function runBenchmark(cfg: Config): Promise { if (!which("harbor")) { throw new Error("harbor not found on PATH. Install with: uv tool install harbor"); } @@ -1073,8 +1247,8 @@ async function main(): Promise { } const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); - const modelSlug = cfg.models[0].replace(/[^a-zA-Z0-9]+/g, "-"); - const jobName = cfg.jobName ?? `tb2-${modelSlug}-${stamp}`; + const modelSlug = (cfg.models[0] ?? "model").replace(/[^a-zA-Z0-9]+/g, "-"); + const jobName = cfg.jobName ?? `${modelSlug}-${stamp}`; const jobDir = path.join(cfg.jobsDir, jobName); const benchDir = path.join(cfg.jobsDir, "_bench", jobName); fs.mkdirSync(benchDir, { recursive: true }); @@ -1094,6 +1268,12 @@ async function main(): Promise { } } + // source mount (default): repo bind-mounted read-only + cached linux deps tree + let source: SourceMount | null = null; + if (cfg.agent === "omp" && cfg.install === "source" && !cfg.binaryArm64 && !cfg.binaryX64) { + source = prepareSourceDeps(cfg); + } + // models.yml (gateway) let modelsYaml = ""; if (cfg.agent === "omp" && cfg.gateway) { @@ -1106,18 +1286,10 @@ async function main(): Promise { ); } } - let hostNetworkOverlayPath: string | null = null; - if (cfg.hostNetwork) { - hostNetworkOverlayPath = path.join(benchDir, "host-network-overlay.yaml"); - const content = `services: - main: - network_mode: "host" -`; - fs.writeFileSync(hostNetworkOverlayPath, content); - } + const composeOverlayPath = writeComposeOverlay(benchDir, cfg, source); - const harborArgs = buildHarborArgs(cfg, jobName, modelsYaml, tarball, hostNetworkOverlayPath); - const harborEnv = buildHarborEnv(cfg, modelsYaml, tarball, version); + const harborArgs = buildHarborArgs(cfg, jobName, modelsYaml, tarball, composeOverlayPath); + const harborEnv = buildHarborEnv(cfg, modelsYaml, tarball, version, source); const logPath = path.join(benchDir, "harbor.log"); if (cfg.dryRun) { process.stdout.write(bold("\nharbor command:\n")); @@ -1127,16 +1299,20 @@ async function main(): Promise { process.stdout.write(`${fs.readFileSync(modelsYaml, "utf8")}\n`); } process.stdout.write(bold("omp env:\n")); - for (const k in harborEnv) { - if (k === "OMP_TB_FORWARD_ENV") continue; - if (k.startsWith("OMP_TB_") || k === "PYTHONPATH") process.stdout.write(` ${k}=${harborEnv[k]}\n`); + for (const key in harborEnv) { + if (key === "OMP_BENCH_FORWARD_ENV") continue; + if (key.startsWith("OMP_BENCH_") || key === "PYTHONPATH") process.stdout.write(` ${key}=${harborEnv[key]}\n`); } - if (harborEnv.OMP_TB_FORWARD_ENV) { - const keys = Object.keys(JSON.parse(harborEnv.OMP_TB_FORWARD_ENV) as Record); - process.stdout.write(` OMP_TB_FORWARD_ENV=${keys.join(",")} (values hidden)\n`); + if (harborEnv.OMP_BENCH_FORWARD_ENV) { + const parsedForwardEnv: unknown = JSON.parse(harborEnv.OMP_BENCH_FORWARD_ENV); + if (parsedForwardEnv !== null && typeof parsedForwardEnv === "object" && !Array.isArray(parsedForwardEnv)) { + const keys: string[] = []; + for (const key in parsedForwardEnv) keys.push(key); + process.stdout.write(` OMP_BENCH_FORWARD_ENV=${keys.join(",")} (values hidden)\n`); + } } process.stdout.write(`\njob dir: ${jobDir}\nbench dir: ${benchDir}\n`); - return; + return { exitCode: 0, jobName, jobDir, benchDir, tarball, elapsedMs: 0, totals: null, reportPath: null }; } // Pre-run cleanup of leftover Harbor resources, if requested. @@ -1194,23 +1370,36 @@ async function main(): Promise { // final summary (printed to the normal screen) const trials = readTrials(jobDir); - const tot = aggregate(trials, readJobResult(jobDir), expected); - const successPct = tot.done > 0 ? (tot.pass / tot.done) * 100 : 0; + const totals = aggregate(trials, readJobResult(jobDir), expected); + const successPct = totals.done > 0 ? (totals.pass / totals.done) * 100 : 0; + const elapsedMs = Date.now() - st.startMs; const reportPath = writeReport(st, benchDir, exitCode); process.stdout.write("\n"); process.stdout.write( - `${bold("terminal-bench-2 complete")} — ${green(`${tot.pass}/${tot.done} passed (${successPct.toFixed(1)}%)`)}\n`, + `${bold(`${st.cfg.dataset} complete`)} — ${green(`${totals.pass}/${totals.done} passed (${successPct.toFixed(1)}%)`)}\n`, ); process.stdout.write( - `fail ${tot.fail} · error ${tot.error} · spend ${fmtUsd(tot.costUsd)} · elapsed ${fmtDur(Date.now() - st.startMs)}\n`, + `fail ${totals.fail} · error ${totals.error} · spend ${fmtUsd(totals.costUsd)} · elapsed ${fmtDur(elapsedMs)}\n`, ); process.stdout.write( - `tokens: in ${fmtNum(tot.tokIn)} · out ${fmtNum(tot.tokOut)} · cache ${fmtNum(tot.tokCache)}\n`, + `tokens: in ${fmtNum(totals.tokIn)} · out ${fmtNum(totals.tokOut)} · cache ${fmtNum(totals.tokCache)}\n`, ); process.stdout.write(`${dim("report:")} ${reportPath}\n`); process.stdout.write(`${dim("logs: ")} ${logPath}\n`); process.stdout.write(`${dim("trials:")} ${jobDir}\n`); if (exitCode !== 0) process.stdout.write(yellow(`harbor exited ${exitCode}; see harbor.log\n`)); + return { exitCode, jobName, jobDir, benchDir, tarball, elapsedMs, totals, reportPath }; +} + +async function main(): Promise { + const argv = process.argv.slice(2); + if (argv[0] === "cleanup") { + if (!which("docker")) throw new Error("docker not found on PATH (required for cleanup)."); + runDockerCleanup(true); + return; + } + const cfg = parseArgs(argv); + const exitCode = (await runBenchmark(cfg)).exitCode; process.exit(exitCode); } diff --git a/packages/harbor-manager/src/server.ts b/packages/harbor-manager/src/server.ts new file mode 100755 index 000000000..7068d555c --- /dev/null +++ b/packages/harbor-manager/src/server.ts @@ -0,0 +1,425 @@ +#!/usr/bin/env bun +/** + * harbor-manager server: REST + SSE API over the run store, static web + * dashboard, and a launcher that spawns the CLI runner as a managed child. + * + * bun src/server.ts [--port 4700] [--jobs-dir ] + * + * API: + * GET /api/runs → RunRow[] + * POST /api/runs → launch a run (JSON body, see LaunchRequest) + * GET /api/runs/:name → { run, trials } + * DELETE /api/runs/:name → cancel a manager-launched run + * GET /api/runs/:name/trials/:trial/transcript?tail=N[&raw=1] + * GET /api/events → SSE: run-list snapshots on change + */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import type { Server, Subprocess } from "bun"; +import { buildExperiments, experimentDetail, experimentOf } from "./experiments"; +import { type RunRole, RunStore } from "./store"; + +/** PUT /api/experiments/:id body — goal and per-run role/note metadata. */ +export interface ExperimentMetaUpdate { + goal?: string; + runs?: Record; +} + +const INDEX_HTML_PATH = new URL("./web/index.html", import.meta.url).pathname; + +const REPO_ROOT = path.resolve(import.meta.dir, "..", "..", ".."); +const PKG_DIR = path.resolve(import.meta.dir, ".."); +const DEFAULT_JOBS_DIR = path.join(REPO_ROOT, "runs", "harbor"); + +/** POST /api/runs body. Mirrors the runner CLI surface we actually use. */ +export interface LaunchRequest { + model: string; + dataset?: string; + /** Task count for a dataset sample, or omit when `include` is given. */ + tasks?: number; + /** Explicit task names (passed as repeated --include). */ + include?: string[]; + concurrency?: number; + timeoutMultiplier?: number; + attempts?: number; + agent?: string; + jobName?: string; + webSearch?: boolean; + slide?: { model: string; turns?: number; onAction?: boolean; plan?: boolean }; + /** Role of this run inside its experiment (baseline vs treatment). */ + role?: RunRole; + /** One-line description of what this arm tests. */ + note?: string; + /** Experiment goal; upserted for the run's experiment (job-name prefix). */ + goal?: string; + /** Use prebuilt dist/omp-linux-* binaries instead of the default source mount. */ + prebuiltBinaries?: boolean; + /** Extra raw runner args, appended verbatim. */ + extraArgs?: string[]; +} + +interface ManagedChild { + proc: Subprocess; + jobName: string; + cancelled: boolean; +} + +const enum SseState { + Open = 0, + Closed = 1, +} + +interface SseClient { + controller: ReadableStreamDefaultController; + state: SseState; +} + +function parseServerArgs(argv: string[]): { port: number; jobsDir: string } { + let port = 4700; + let jobsDir = DEFAULT_JOBS_DIR; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === "--port" && argv[i + 1]) port = Number(argv[++i]); + else if (argv[i] === "--jobs-dir" && argv[i + 1]) jobsDir = path.resolve(argv[++i]); + } + if (!Number.isSafeInteger(port) || port < 1 || port > 65535) throw new Error("--port must be 1..65535"); + return { port, jobsDir }; +} + +export class ManagerServer { + #store: RunStore; + #children = new Map(); + #sse = new Set(); + #lastSnapshot = ""; + #syncTimer: Timer | undefined; + #server: Server | null = null; + #appBundleCode: string | null = null; + readonly jobsDir: string; + + constructor(jobsDir: string, dbPath?: string) { + this.jobsDir = jobsDir; + this.#store = new RunStore(jobsDir, dbPath); + } + + get store(): RunStore { + return this.#store; + } + + start(port: number): Server { + this.#store.discover(); + this.#store.syncAll(); + this.#syncTimer = setInterval(() => this.#tick(), 2000); + this.#server = Bun.serve({ + port, + idleTimeout: 0, + fetch: request => this.#route(request), + }); + return this.#server; + } + + async stop(): Promise { + clearInterval(this.#syncTimer); + for (const client of this.#sse) { + client.state = SseState.Closed; + try { + client.controller.close(); + } catch {} + } + this.#sse.clear(); + this.#server?.stop(true); + this.#store.close(); + } + + #tick(): void { + this.#store.syncActive(); + const snapshot = JSON.stringify(this.#store.listRuns()); + if (snapshot !== this.#lastSnapshot) { + this.#lastSnapshot = snapshot; + this.#broadcast(`data: ${snapshot}\n\n`); + } + } + + /** Bundle the React dashboard once per process; served at /app.js. */ + async #appBundle(): Promise { + if (this.#appBundleCode !== null) return this.#appBundleCode; + const result = await Bun.build({ + entrypoints: [path.join(import.meta.dir, "web", "app.tsx")], + target: "browser", + minify: true, + define: { "process.env.NODE_ENV": '"production"' }, + }); + if (!result.success) { + throw new Error(`dashboard bundle failed:\n${result.logs.map(l => l.message).join("\n")}`); + } + this.#appBundleCode = await result.outputs[0].text(); + return this.#appBundleCode; + } + + #broadcast(frame: string): void { + const bytes = new TextEncoder().encode(frame); + for (const client of this.#sse) { + if (client.state === SseState.Closed) continue; + try { + client.controller.enqueue(bytes); + } catch { + client.state = SseState.Closed; + this.#sse.delete(client); + } + } + } + + async #route(request: Request): Promise { + const url = new URL(request.url); + const p = url.pathname; + try { + if (p === "/" || p === "/index.html") { + return new Response(Bun.file(INDEX_HTML_PATH)); + } + if (p === "/app.js") { + return new Response(await this.#appBundle(), { + headers: { "content-type": "text/javascript; charset=utf-8" }, + }); + } + if (p === "/api/events") return this.#sseResponse(); + if (p === "/api/experiments" && request.method === "GET") { + return Response.json(buildExperiments(this.#store)); + } + const expMatch = p.match(/^\/api\/experiments\/([^/]+)$/); + if (expMatch) { + const id = decodeURIComponent(expMatch[1]); + if (request.method === "PUT") { + const body = (await request.json()) as ExperimentMetaUpdate; + return Response.json(this.updateExperimentMeta(id, body)); + } + const detail = experimentDetail(this.#store, id); + if (!detail) return Response.json({ error: "experiment not found" }, { status: 404 }); + return Response.json(detail); + } + if (p === "/api/runs" && request.method === "GET") { + return Response.json(this.#store.listRuns()); + } + if (p === "/api/runs" && request.method === "POST") { + const body = (await request.json()) as LaunchRequest; + return Response.json(this.launch(body), { status: 201 }); + } + const runMatch = p.match(/^\/api\/runs\/([^/]+)$/); + if (runMatch) { + const jobName = decodeURIComponent(runMatch[1]); + if (request.method === "DELETE") return Response.json(this.cancel(jobName)); + const run = this.#store.syncRun(jobName); + if (!run) return Response.json({ error: "run not found" }, { status: 404 }); + return Response.json({ run, trials: this.#store.listTrials(jobName) }); + } + const trialMatch = p.match(/^\/api\/runs\/([^/]+)\/trials\/([^/]+)\/transcript$/); + if (trialMatch) { + const jobName = decodeURIComponent(trialMatch[1]); + const trial = decodeURIComponent(trialMatch[2]); + const tail = Number(url.searchParams.get("tail") ?? "120"); + const raw = url.searchParams.get("raw") === "1"; + return this.#transcript(jobName, trial, tail, raw); + } + return Response.json({ error: "not found" }, { status: 404 }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return Response.json({ error: message }, { status: 400 }); + } + } + + #sseResponse(): Response { + let client: SseClient; + const sse = this.#sse; + const initial = `data: ${JSON.stringify(this.#store.listRuns())}\n\n`; + const stream = new ReadableStream({ + start(controller) { + client = { controller, state: SseState.Open }; + sse.add(client); + controller.enqueue(new TextEncoder().encode(initial)); + }, + cancel() { + client.state = SseState.Closed; + sse.delete(client); + }, + }); + return new Response(stream, { + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache", + connection: "keep-alive", + }, + }); + } + + /** Spawn the CLI runner for `request` and register the run. */ + launch(request: LaunchRequest): { jobName: string; pid: number } { + if (!request.model) throw new Error("model is required"); + const dataset = request.dataset ?? "terminal-bench@2.0"; + const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); + const modelSlug = request.model.replace(/[^a-zA-Z0-9]+/g, "-"); + const jobName = request.jobName ?? `${modelSlug}-${stamp}`; + if (this.#children.has(jobName) || this.#store.getRun(jobName)?.status === "running") { + throw new Error(`run ${jobName} is already running`); + } + + const argv = ["bun", "src/runner.ts", "--model", request.model, "-d", dataset, "--job-name", jobName]; + if (request.agent) argv.push("--agent", request.agent); + if (request.tasks !== undefined) argv.push("--tasks", String(request.tasks)); + if (request.concurrency !== undefined) argv.push("--concurrency", String(request.concurrency)); + if (request.attempts !== undefined) argv.push("--attempts", String(request.attempts)); + if (request.timeoutMultiplier !== undefined) argv.push("--timeout-multiplier", String(request.timeoutMultiplier)); + if (request.webSearch) argv.push("--web-search"); + for (const task of request.include ?? []) argv.push("--include", task); + if (request.slide) { + argv.push("--agent-arg", "--reasoning-slide-model", "--agent-arg", request.slide.model); + if (request.slide.onAction) argv.push("--agent-arg", "--reasoning-slide-on-action"); + else if (request.slide.turns !== undefined) { + argv.push("--agent-arg", "--reasoning-slide-turns", "--agent-arg", String(request.slide.turns)); + } else throw new Error("slide requires turns or onAction"); + if (request.slide.plan) argv.push("--agent-arg", "--reasoning-slide-plan"); + // The runner only auto-routes gateway auth for the primary model's + // provider; declare the slide model's provider explicitly so its + // requests reach the gateway too. + const slideProvider = request.slide.model.split("/", 1)[0]; + if (slideProvider) argv.push("--providers", slideProvider); + } + // Default to source mode (repo bind-mount, no rebuild); prebuilt binaries only on request. + if (request.prebuiltBinaries) { + for (const name of ["omp-linux-arm64", "omp-linux-x64"]) { + const binary = path.join(REPO_ROOT, "packages", "coding-agent", "dist", name); + if (fs.existsSync(binary)) argv.push("--binary", binary); + } + } + argv.push(...(request.extraArgs ?? [])); + + const logDir = path.join(this.jobsDir, "_manager", "logs"); + fs.mkdirSync(logDir, { recursive: true }); + const logFile = fs.openSync(path.join(logDir, `${jobName}.log`), "w"); + const proc = Bun.spawn(argv, { + cwd: PKG_DIR, + stdout: logFile, + stderr: logFile, + env: { ...process.env }, + }); + const child: ManagedChild = { proc, jobName, cancelled: false }; + this.#children.set(jobName, child); + proc.exited.then(exitCode => { + this.#store.markExit(jobName, exitCode, child.cancelled); + // Final sync AFTER the terminal state: the ticker only revisits + // running rows, so the last-2s trial results would otherwise be lost. + this.#store.syncRun(jobName); + this.#children.delete(jobName); + this.#tick(); + }); + this.#store.registerLaunch({ + jobName, + dataset, + agent: request.agent ?? "omp", + models: [request.model], + slide: request.slide, + pid: proc.pid, + role: request.role, + note: request.note, + }); + if (request.goal) this.#store.setExperimentGoal(experimentOf(jobName), request.goal); + this.#tick(); + return { jobName, pid: proc.pid }; + } + + /** Apply goal + per-run role/note metadata; used by the UI and for backfill. */ + updateExperimentMeta(id: string, update: ExperimentMetaUpdate): { id: string; updatedRuns: string[] } { + if (update.goal !== undefined) this.#store.setExperimentGoal(id, update.goal); + const updatedRuns: string[] = []; + for (const [jobName, meta] of Object.entries(update.runs ?? {})) { + if (experimentOf(jobName) !== id) continue; + if (this.#store.setRunMeta(jobName, meta)) updatedRuns.push(jobName); + } + this.#tick(); + return { id, updatedRuns }; + } + + /** Cancel a manager-launched run (kills the runner; harbor children follow). */ + cancel(jobName: string): { jobName: string; cancelled: boolean } { + const child = this.#children.get(jobName); + if (child) { + child.cancelled = true; + child.proc.kill(9); + return { jobName, cancelled: true }; + } + const run = this.#store.getRun(jobName); + if (run?.pid != null) { + try { + process.kill(run.pid, "SIGKILL"); + } catch {} + this.#store.markExit(jobName, null, true); + return { jobName, cancelled: true }; + } + return { jobName, cancelled: false }; + } + + /** Compact transcript view of a trial's omp.txt session JSONL. */ + #transcript(jobName: string, trial: string, tail: number, raw: boolean): Response { + const file = path.join(this.jobsDir, jobName, trial, "agent", "omp.txt"); + if (!fs.existsSync(file)) return Response.json({ error: "transcript not found" }, { status: 404 }); + const lines = fs.readFileSync(file, "utf8").split("\n").filter(Boolean); + const n = Number.isSafeInteger(tail) && tail > 0 ? Math.min(tail, 2000) : 120; + if (raw) { + return new Response(lines.slice(-n).join("\n"), { + headers: { "content-type": "application/x-ndjson" }, + }); + } + const entries: Array> = []; + for (const line of lines) { + let event: Record; + try { + event = JSON.parse(line) as Record; + } catch { + continue; + } + const type = event.type; + if (type === "message_end") { + const message = event.message as Record | undefined; + if (!message) continue; + const role = message.role; + if (role === "assistant") { + const content = Array.isArray(message.content) + ? (message.content as Array>) + : []; + const text = content + .filter(block => block.type === "text") + .map(block => String(block.text ?? "")) + .join("\n"); + const tools = content.filter(block => block.type === "toolCall").map(block => String(block.name ?? "?")); + entries.push({ kind: "assistant", model: message.model ?? "", text, tools }); + } else if (role === "toolResult") { + const content = Array.isArray(message.content) + ? (message.content as Array>) + : []; + const text = content + .filter(block => block.type === "text") + .map(block => String(block.text ?? "")) + .join("\n"); + entries.push({ + kind: "toolResult", + tool: message.toolName ?? "?", + isError: message.isError === true, + text: text.length > 1600 ? `${text.slice(0, 1600)}…` : text, + }); + } + } else if (type === "notice") { + entries.push({ kind: "notice", text: event.message ?? "" }); + } + } + return Response.json({ jobName, trial, entries: entries.slice(-n), totalEvents: lines.length }); + } +} + +if (import.meta.main) { + const { port, jobsDir } = parseServerArgs(process.argv.slice(2)); + const manager = new ManagerServer(jobsDir); + const server = manager.start(port); + process.stdout.write(`harbor-manager listening on http://localhost:${server.port} (jobs: ${jobsDir})\n`); + const shutdown = async () => { + await manager.stop(); + process.exit(0); + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); +} diff --git a/packages/harbor-manager/src/store.ts b/packages/harbor-manager/src/store.ts new file mode 100644 index 000000000..152a56d63 --- /dev/null +++ b/packages/harbor-manager/src/store.ts @@ -0,0 +1,441 @@ +/** + * SQLite-backed store for Harbor runs managed by this package. + * + * The filesystem stays the source of truth (Harbor writes `result.json` + * per job and per trial); the store mirrors it into queryable rows and adds + * manager-owned metadata Harbor has no notion of: launch pid, requested + * config, lifecycle status. `syncRun` re-reads a job dir and upserts. + */ + +import { Database } from "bun:sqlite"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { aggregate, readJobResult, readTrials } from "./runner"; + +export type RunStatus = "running" | "complete" | "failed" | "cancelled"; + +/** How a run relates to its experiment's question. */ +export type RunRole = "baseline" | "variant" | ""; + +export interface RunRow { + jobName: string; + dataset: string; + agent: string; + models: string; + slide: string | null; + /** Role inside the experiment (baseline vs treatment); "" when unspecified. */ + role: RunRole; + /** One-line description of what this arm tests (e.g. "slide→flash after 8 turns"). */ + note: string; + status: RunStatus; + pid: number | null; + exitCode: number | null; + createdAt: number; + finishedAt: number | null; + nTotal: number; + done: number; + pass: number; + fail: number; + error: number; + running: number; + costUsd: number; + tokIn: number; + tokOut: number; + tokCache: number; +} + +export interface TrialRow { + jobName: string; + name: string; + task: string; + status: string; + reward: number | null; + costUsd: number; + durationMs: number; + detail: string; + updatedAt: number; +} + +export interface LaunchRecord { + jobName: string; + dataset: string; + agent: string; + models: string[]; + slide?: { model: string; turns?: number; onAction?: boolean; plan?: boolean }; + pid: number; + role?: RunRole; + note?: string; +} + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS runs ( + job_name TEXT PRIMARY KEY, + dataset TEXT NOT NULL DEFAULT '', + agent TEXT NOT NULL DEFAULT 'omp', + models TEXT NOT NULL DEFAULT '', + slide TEXT, + role TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'running', + pid INTEGER, + exit_code INTEGER, + created_at INTEGER NOT NULL, + finished_at INTEGER, + n_total INTEGER NOT NULL DEFAULT 0, + done INTEGER NOT NULL DEFAULT 0, + pass INTEGER NOT NULL DEFAULT 0, + fail INTEGER NOT NULL DEFAULT 0, + error INTEGER NOT NULL DEFAULT 0, + running INTEGER NOT NULL DEFAULT 0, + cost_usd REAL NOT NULL DEFAULT 0, + tok_in INTEGER NOT NULL DEFAULT 0, + tok_out INTEGER NOT NULL DEFAULT 0, + tok_cache INTEGER NOT NULL DEFAULT 0 +); +CREATE TABLE IF NOT EXISTS trials ( + job_name TEXT NOT NULL, + name TEXT NOT NULL, + task TEXT NOT NULL, + status TEXT NOT NULL, + reward REAL, + cost_usd REAL NOT NULL DEFAULT 0, + duration_ms INTEGER NOT NULL DEFAULT 0, + detail TEXT NOT NULL DEFAULT '', + updated_at INTEGER NOT NULL, + PRIMARY KEY (job_name, name) +); +CREATE INDEX IF NOT EXISTS idx_trials_job ON trials(job_name); +CREATE TABLE IF NOT EXISTS experiments ( + id TEXT PRIMARY KEY, + goal TEXT NOT NULL DEFAULT '', + updated_at INTEGER NOT NULL +); +`; + +/** Directory names inside the jobs root that are not Harbor job dirs. */ +const NON_JOB_DIRS = new Set(["_bench", "_manager"]); + +export class RunStore { + #db: Database; + readonly jobsDir: string; + + constructor(jobsDir: string, dbPath?: string) { + this.jobsDir = jobsDir; + fs.mkdirSync(path.join(jobsDir, "_manager"), { recursive: true }); + this.#db = new Database(dbPath ?? path.join(jobsDir, "_manager", "harbor-manager.sqlite")); + this.#db.run("PRAGMA journal_mode = WAL"); + this.#db.run(SCHEMA); + // Migration for stores created before run roles/notes existed. + const columns = new Set( + (this.#db.query("PRAGMA table_info(runs)").all() as Array<{ name: string }>).map(c => c.name), + ); + if (!columns.has("role")) this.#db.run("ALTER TABLE runs ADD COLUMN role TEXT NOT NULL DEFAULT ''"); + if (!columns.has("note")) this.#db.run("ALTER TABLE runs ADD COLUMN note TEXT NOT NULL DEFAULT ''"); + } + + close(): void { + this.#db.close(); + } + + /** Register a run this manager just launched (pid-owning). */ + registerLaunch(launch: LaunchRecord): void { + this.#db + .query( + `INSERT INTO runs (job_name, dataset, agent, models, slide, role, note, status, pid, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 'running', ?, ?) + ON CONFLICT(job_name) DO UPDATE SET + pid = excluded.pid, status = 'running', + role = CASE WHEN excluded.role != '' THEN excluded.role ELSE runs.role END, + note = CASE WHEN excluded.note != '' THEN excluded.note ELSE runs.note END`, + ) + .run( + launch.jobName, + launch.dataset, + launch.agent, + launch.models.join(","), + launch.slide ? JSON.stringify(launch.slide) : null, + launch.role ?? "", + launch.note ?? "", + launch.pid, + Date.now(), + ); + } + + /** Upsert the experiment's stated goal. */ + setExperimentGoal(id: string, goal: string): void { + this.#db + .query( + `INSERT INTO experiments (id, goal, updated_at) VALUES (?, ?, ?) + ON CONFLICT(id) DO UPDATE SET goal = excluded.goal, updated_at = excluded.updated_at`, + ) + .run(id, goal, Date.now()); + } + + getExperimentGoal(id: string): string { + const row = this.#db.query("SELECT goal FROM experiments WHERE id = ?").get(id) as { goal: string } | null; + return row?.goal ?? ""; + } + + /** Set role/note metadata on an existing run row. */ + setRunMeta(jobName: string, meta: { role?: RunRole; note?: string }): boolean { + const existing = this.getRun(jobName); + if (!existing) return false; + this.#db + .query("UPDATE runs SET role = ?, note = ? WHERE job_name = ?") + .run(meta.role ?? existing.role, meta.note ?? existing.note, jobName); + return true; + } + + /** Mark a launched run's terminal state (called when its child process exits). */ + markExit(jobName: string, exitCode: number | null, cancelled = false): void { + const status: RunStatus = cancelled ? "cancelled" : exitCode === 0 ? "complete" : "failed"; + this.#db + .query("UPDATE runs SET status = ?, exit_code = ?, finished_at = ?, pid = NULL WHERE job_name = ?") + .run(status, exitCode, Date.now(), jobName); + } + + /** + * Discover job dirs on disk that have no run row yet (runs launched by the + * CLI or a previous manager instance) and backfill them as historical rows. + */ + discover(): number { + let entries: fs.Dirent[] = []; + try { + entries = fs.readdirSync(this.jobsDir, { withFileTypes: true }); + } catch { + return 0; + } + const known = new Set( + (this.#db.query("SELECT job_name FROM runs").all() as Array<{ job_name: string }>).map(r => r.job_name), + ); + let added = 0; + for (const e of entries) { + if (!e.isDirectory() || NON_JOB_DIRS.has(e.name) || known.has(e.name)) continue; + const jobDir = path.join(this.jobsDir, e.name); + const meta = readHarborConfig(jobDir); + const createdAt = dirCreatedAt(jobDir); + this.#db + .query( + `INSERT INTO runs (job_name, dataset, agent, models, status, created_at) + VALUES (?, ?, ?, ?, 'running', ?)`, + ) + .run(e.name, meta.dataset, meta.agent, meta.models, createdAt); + this.syncRun(e.name); + added++; + } + return added; + } + + /** Re-read a job dir from disk and mirror trial + rollup state into the DB. */ + syncRun(jobName: string): RunRow | null { + const jobDir = path.join(this.jobsDir, jobName); + if (!fs.existsSync(jobDir)) return this.getRun(jobName); + const trials = readTrials(jobDir); + const job = readJobResult(jobDir); + const totals = aggregate(trials, job, job?.nTotal ?? trials.length); + const now = Date.now(); + const upsert = this.#db.query( + `INSERT INTO trials (job_name, name, task, status, reward, cost_usd, duration_ms, detail, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_name, name) DO UPDATE SET + status = excluded.status, reward = excluded.reward, cost_usd = excluded.cost_usd, + duration_ms = excluded.duration_ms, detail = excluded.detail, updated_at = excluded.updated_at`, + ); + const tx = this.#db.transaction(() => { + for (const t of trials) { + upsert.run( + jobName, + t.name, + t.name.replace(/__[^_]+$/, ""), + t.status, + t.reward, + t.costUsd, + t.durationMs, + t.detail, + now, + ); + } + this.#db + .query( + `UPDATE runs SET n_total = ?, done = ?, pass = ?, fail = ?, error = ?, running = ?, + cost_usd = ?, tok_in = ?, tok_out = ?, tok_cache = ? WHERE job_name = ?`, + ) + .run( + totals.total, + totals.done, + totals.pass, + totals.fail, + totals.error, + totals.running, + totals.costUsd, + totals.tokIn, + totals.tokOut, + totals.tokCache, + jobName, + ); + // Foreign runs (no owning pid and never finalized by markExit) infer + // their lifecycle from Harbor's job-level result: `finished_at` is + // terminal; a missing terminal marker with a fresh job dir means still + // running; stale for >30 min means the harness died mid-run. + const row = this.getRun(jobName); + if (row && row.pid === null && row.finishedAt === null && row.status !== "cancelled") { + const job2 = readJobResult(jobDir); + let status: RunStatus; + let finishedAt: number | null = null; + if (job2?.finishedAt != null) { + status = "complete"; + finishedAt = job2.finishedAt; + } else if (jobDirFresh(jobDir)) { + status = "running"; + } else { + status = totals.done > 0 && totals.done >= totals.total ? "complete" : "failed"; + finishedAt = jobDirMtime(jobDir); + } + if (status !== row.status) { + this.#db + .query("UPDATE runs SET status = ?, finished_at = ? WHERE job_name = ?") + .run(status, finishedAt, jobName); + } + } + }); + tx(); + return this.getRun(jobName); + } + + /** Sync every run currently marked running; returns the refreshed rows. */ + syncActive(): RunRow[] { + const active = this.#db.query("SELECT job_name FROM runs WHERE status = 'running'").all() as Array<{ + job_name: string; + }>; + const out: RunRow[] = []; + for (const { job_name } of active) { + // A pid-owning run whose process died without markExit (manager restart) + // is finalized here so it doesn't stay "running" forever. + const row = this.getRun(job_name); + if (row?.pid != null && !processAlive(row.pid)) { + this.markExit(job_name, null); + } + const synced = this.syncRun(job_name); + if (synced) out.push(synced); + } + return out; + } + + /** + * Sync every known run once — startup reconciliation. Rows stamped before a + * status-inference change (or by an older manager) self-correct here, since + * the periodic ticker only revisits rows already marked running. + */ + syncAll(): void { + const rows = this.#db.query("SELECT job_name FROM runs").all() as Array<{ job_name: string }>; + for (const { job_name } of rows) this.syncRun(job_name); + } + + getRun(jobName: string): RunRow | null { + const r = this.#db.query("SELECT * FROM runs WHERE job_name = ?").get(jobName) as Record | null; + return r ? rowToRun(r) : null; + } + + listRuns(): RunRow[] { + const rows = this.#db.query("SELECT * FROM runs ORDER BY created_at DESC").all() as Array< + Record + >; + return rows.map(rowToRun); + } + + listTrials(jobName: string): TrialRow[] { + const rows = this.#db.query("SELECT * FROM trials WHERE job_name = ? ORDER BY name").all(jobName) as Array< + Record + >; + return rows.map(r => ({ + jobName: String(r.job_name), + name: String(r.name), + task: String(r.task), + status: String(r.status), + reward: r.reward === null ? null : Number(r.reward), + costUsd: Number(r.cost_usd), + durationMs: Number(r.duration_ms), + detail: String(r.detail), + updatedAt: Number(r.updated_at), + })); + } +} + +function rowToRun(r: Record): RunRow { + return { + jobName: String(r.job_name), + dataset: String(r.dataset), + agent: String(r.agent), + models: String(r.models), + slide: r.slide === null ? null : String(r.slide), + role: String(r.role ?? "") as RunRole, + note: String(r.note ?? ""), + status: String(r.status) as RunStatus, + pid: r.pid === null ? null : Number(r.pid), + exitCode: r.exit_code === null ? null : Number(r.exit_code), + createdAt: Number(r.created_at), + finishedAt: r.finished_at === null ? null : Number(r.finished_at), + nTotal: Number(r.n_total), + done: Number(r.done), + pass: Number(r.pass), + fail: Number(r.fail), + error: Number(r.error), + running: Number(r.running), + costUsd: Number(r.cost_usd), + tokIn: Number(r.tok_in), + tokOut: Number(r.tok_out), + tokCache: Number(r.tok_cache), + }; +} + +/** Best-effort launch metadata for historical (CLI-launched) job dirs. */ +function readHarborConfig(jobDir: string): { dataset: string; agent: string; models: string } { + try { + const raw = JSON.parse(fs.readFileSync(path.join(jobDir, "config.json"), "utf8")) as Record; + const dataset = + typeof raw.dataset === "string" + ? raw.dataset + : (((raw.datasets as Array> | undefined)?.[0]?.name as string | undefined) ?? ""); + const agents = raw.agents as Array> | undefined; + const agent = (agents?.[0]?.name as string | undefined) ?? "omp"; + const models = (agents?.[0]?.model_name as string | undefined) ?? ""; + return { dataset: String(dataset), agent, models }; + } catch { + return { dataset: "", agent: "omp", models: "" }; + } +} + +function dirCreatedAt(dir: string): number { + try { + return Math.round(fs.statSync(dir).birthtimeMs || fs.statSync(dir).mtimeMs); + } catch { + return Date.now(); + } +} + +/** Stale threshold for foreign runs without a terminal marker. */ +const JOB_DIR_STALE_MS = 30 * 60 * 1000; + +/** Newest mtime across the job dir and its result.json (cheap freshness probe). */ +function jobDirMtime(dir: string): number { + let newest = 0; + for (const p of [dir, path.join(dir, "result.json")]) { + try { + newest = Math.max(newest, fs.statSync(p).mtimeMs); + } catch {} + } + return Math.round(newest) || Date.now(); +} + +function jobDirFresh(dir: string): boolean { + return Date.now() - jobDirMtime(dir) < JOB_DIR_STALE_MS; +} + +function processAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} diff --git a/packages/harbor-manager/src/web/app.tsx b/packages/harbor-manager/src/web/app.tsx new file mode 100644 index 000000000..f1aed9356 --- /dev/null +++ b/packages/harbor-manager/src/web/app.tsx @@ -0,0 +1,817 @@ +/** + * harbor-manager dashboard. + * + * Views (hash-routed): + * #/ experiments index — runs grouped by job-name prefix + * #/exp/ experiment detail — arm table, dithered comparison charts + * (projected values for in-flight arms, dimmed), task matrix + * #/runs flat run list (legacy view) + * #/runs/ run detail — trial grid + live transcript tail + */ +import { useCallback, useEffect, useRef, useState } from "react"; +import { createRoot } from "react-dom/client"; + +// ── api types (mirrors server modules) ────────────────────────────────────── + +interface RunRow { + jobName: string; + dataset: string; + agent: string; + models: string; + slide: string | null; + role: "baseline" | "variant" | ""; + note: string; + status: "running" | "complete" | "failed" | "cancelled"; + pid: number | null; + createdAt: number; + finishedAt: number | null; + nTotal: number; + done: number; + pass: number; + fail: number; + error: number; + running: number; + costUsd: number; +} + +interface TrialRow { + name: string; + task: string; + status: string; + reward: number | null; + costUsd: number; + durationMs: number; + detail: string; +} + +interface ArmProjection { + etaMs: number | null; + passPct: number; + costPerTask: number; + totalCostUsd: number; + meanTrialMs: number; +} + +interface ArmSummary { + run: RunRow; + arm: string; + config: string; + passPct: number | null; + costPerTask: number | null; + meanTrialMs: number | null; + projected: ArmProjection | null; +} + +interface ExperimentSummary { + id: string; + goal: string; + arms: number; + runningArms: number; + datasets: string[]; + nTotal: number; + done: number; + pass: number; + fail: number; + error: number; + costUsd: number; + updatedAt: number; +} + +interface ExperimentDetail { + id: string; + goal: string; + arms: ArmSummary[]; + tasks: string[]; + matrix: Record>; +} + +interface TranscriptEntry { + kind: string; + model?: string; + tool?: string; + isError?: boolean; + text?: string; + tools?: string[]; +} + +// ── helpers ────────────────────────────────────────────────────────────────── + +const fmtUsd = (v: number) => (v >= 100 ? `$${v.toFixed(0)}` : v >= 1 ? `$${v.toFixed(2)}` : `$${v.toFixed(3)}`); +const fmtMin = (ms: number) => `${(ms / 60000).toFixed(1)}m`; +const fmtEta = (etaMs: number | null) => { + if (etaMs === null) return "—"; + const mins = Math.max(0, Math.round((etaMs - Date.now()) / 60000)); + return mins >= 90 ? `~${(mins / 60).toFixed(1)}h` : `~${mins}m`; +}; + +async function getJson(url: string): Promise { + const res = await fetch(url); + if (!res.ok) throw new Error(`${url}: ${res.status}`); + return (await res.json()) as T; +} + +function useHashRoute(): string { + const [hash, setHash] = useState(location.hash || "#/"); + useEffect(() => { + const onChange = () => setHash(location.hash || "#/"); + window.addEventListener("hashchange", onChange); + return () => window.removeEventListener("hashchange", onChange); + }, []); + return hash; +} + +/** Poll a JSON endpoint on an interval (SSE covers the run list; details poll). */ +function usePolled(url: string | null, intervalMs: number): T | null { + const [data, setData] = useState(null); + useEffect(() => { + if (!url) return; + let live = true; + const load = () => + getJson(url) + .then(d => live && setData(d)) + .catch(() => {}); + load(); + const timer = setInterval(load, intervalMs); + return () => { + live = false; + clearInterval(timer); + }; + }, [url, intervalMs]); + return data; +} + +const STATUS_CLASS: Record = { + running: "text-sky-400 border-sky-400", + complete: "text-emerald-400 border-emerald-400", + failed: "text-red-400 border-red-400", + cancelled: "text-zinc-500 border-zinc-500", + pass: "text-emerald-400 border-emerald-400", + fail: "text-red-400 border-red-400", + error: "text-amber-400 border-amber-400", +}; + +function Chip({ label }: { label: string }) { + return ( + + {label} + + ); +} + +function Progress({ + run, +}: { + run: RunRow | { pass: number; fail: number; error: number; running: number; done: number; nTotal: number }; +}) { + const total = Math.max(run.nTotal, run.done + run.running, 1); + const seg = (n: number) => `${(100 * n) / total}%`; + return ( + + + + + + + + + {run.done}/{run.nTotal || "?"} + + + ); +} + +// ── experiments index ──────────────────────────────────────────────────────── + +function ExperimentsIndex() { + const experiments = usePolled("/api/experiments", 3000); + if (!experiments) return

      loading…
      ; + return ( +       + ); +} + +// ── experiment detail ──────────────────────────────────────────────────────── + +type MetricRow = Record; + +/** + * One value per arm, keyed by its role so the chart colours the comparison: + * baselines blue, variants green, running arms grey-dotted at their projected + * value. The dataKeys stack into a single slot; exactly one is nonzero per row. + */ +function metricRows( + arms: ArmSummary[], + actual: (arm: ArmSummary) => number | null, + projected: (proj: ArmProjection) => number, +): MetricRow[] { + return arms.map(arm => { + const running = arm.run.status === "running"; + const value = running ? 0 : (actual(arm) ?? 0); + return { + arm: arm.arm, + baseline: !running && arm.run.role === "baseline" ? value : 0, + variant: !running && arm.run.role !== "baseline" ? value : 0, + projected: running && arm.projected ? projected(arm.projected) : 0, + }; + }); +} + +function MetricChart({ title, rows, format }: { title: string; rows: MetricRow[]; format?: (v: number) => string }) { + const maxVal = Math.max( + ...rows.map(r => (Number(r.baseline) || 0) + (Number(r.variant) || 0) + (Number(r.projected) || 0)), + 0, + ); + + const percentages = [1.0, 0.75, 0.5, 0.25, 0.0]; + + return ( +
      + {/* Header with Title and Legend */} +
      +
      {title}
      + {/* Legend */} +
      +
      + + baseline +
      +
      + + variant +
      +
      + + projected +
      +
      +
      + + {/* Main Chart Area */} +
      + {/* Y-Axis Column */} +
      + {percentages.map(p => { + const tick = maxVal * p; + const formattedTick = format ? format(tick) : tick.toFixed(1); + return ( + + {formattedTick} + + ); + })} +
      + + {/* Chart Plot Area */} +
      + {/* Grid Lines */} +
      + {percentages.map(p => ( +
      + ))} +
      + + {/* Bars Container */} +
      + {rows.map(row => { + const baselineVal = Number(row.baseline) || 0; + const variantVal = Number(row.variant) || 0; + const projectedVal = Number(row.projected) || 0; + + let val = 0; + let type: "baseline" | "variant" | "projected" = "baseline"; + + if (baselineVal > 0) { + val = baselineVal; + type = "baseline"; + } else if (variantVal > 0) { + val = variantVal; + type = "variant"; + } else if (projectedVal > 0) { + val = projectedVal; + type = "projected"; + } + + const heightPercent = maxVal > 0 ? (val / maxVal) * 100 : 0; + const formattedValue = format ? format(val) : val.toFixed(1); + const barStyles = { + baseline: "bg-gradient-to-t from-sky-600 to-sky-400 border border-sky-400/30", + variant: "bg-gradient-to-t from-emerald-600 to-emerald-400 border border-emerald-400/30", + projected: "bg-zinc-800/80 border-2 border-dashed border-zinc-500", + }; + + return ( +
      + {/* Tooltip on Hover */} +
      +
      +
      {row.arm as string}
      +
      + {type}: {formattedValue} +
      +
      +
      +
      + + {/* The Bar */} +
      + + {/* Arm Name / X-Axis Label */} +
      + {row.arm as string} +
      +
      + ); + })} +
      +
      +
      + {/* Spacer to push X-Axis labels from overflow bottom */} +
      +
      + ); +} + +const CELL_CLASS: Record = { + pass: "bg-emerald-500", + fail: "bg-red-500", + error: "bg-amber-500", + running: "bg-sky-500 animate-pulse", +}; + +function ExperimentPage({ id }: { id: string }) { + const detail = usePolled(`/api/experiments/${encodeURIComponent(id)}`, 3000); + if (!detail) return
      loading…
      ; + const { arms, tasks, matrix, goal } = detail; + const passRows = metricRows( + arms, + a => a.passPct, + p => p.passPct, + ); + const costRows = metricRows( + arms, + a => a.costPerTask, + p => p.costPerTask, + ); + const timeRows = metricRows( + arms, + a => (a.meanTrialMs === null ? null : a.meanTrialMs / 60000), + p => p.meanTrialMs / 60000, + ); + return ( +
      +
      +

      {id}

      + + {arms.length} arms · {tasks.length} tasks + +
      + {goal &&

      {goal}

      } + + + + + + + + + + + + + + + + {arms.map(arm => ( + + + + + + + + + + + + ))} + +
      armtestsstatusprogressetapass%$/taskmean +
      + {arm.arm} + {arm.run.role && ( + + {arm.run.role} + + )} + + {arm.run.note || arm.config || "—"} + + + + + {arm.projected ? fmtEta(arm.projected.etaMs) : "—"} + {arm.passPct !== null ? `${arm.passPct.toFixed(0)}%` : "—"} + {arm.projected && →{arm.projected.passPct.toFixed(0)}%} + + {arm.costPerTask !== null ? fmtUsd(arm.costPerTask) : "—"} + {arm.projected && Σ{fmtUsd(arm.projected.totalCostUsd)}} + {arm.meanTrialMs !== null ? fmtMin(arm.meanTrialMs) : "—"} + + trials + +
      + +
      + `${v.toFixed(1)}%`} /> + fmtUsd(v)} /> + `${v.toFixed(1)}m`} /> +
      + +
      +
      task matrix
      +
      + + + + + {arms.map(arm => ( + + ))} + + + + {tasks.map(task => ( + + + {arms.map(arm => { + const cell = matrix[arm.arm]?.[task]; + return ( + + ); + })} + + ))} + +
      task + {arm.arm} +
      {task} + +
      +
      +
      +
      + ); +} + +// ── runs (legacy flat view) ────────────────────────────────────────────────── + +function useRunsSse(): RunRow[] | null { + const [runs, setRuns] = useState(null); + useEffect(() => { + const es = new EventSource("/api/events"); + es.onmessage = ev => setRuns(JSON.parse(ev.data) as RunRow[]); + return () => es.close(); + }, []); + return runs; +} + +function RunsPage({ selected }: { selected: string | null }) { + const runs = useRunsSse(); + const detail = usePolled<{ run: RunRow; trials: TrialRow[] }>( + selected ? `/api/runs/${encodeURIComponent(selected)}` : null, + 2500, + ); + const [trial, setTrial] = useState(null); + const transcript = usePolled<{ entries: TranscriptEntry[] }>( + selected && trial + ? `/api/runs/${encodeURIComponent(selected)}/trials/${encodeURIComponent(trial)}/transcript?tail=60` + : null, + 2500, + ); + const transcriptRef = useRef(null); + useEffect(() => { + if (!transcript) return; + const el = transcriptRef.current; + if (el) el.scrollTop = el.scrollHeight; + }, [transcript]); + const cancel = useCallback(async (name: string) => { + if (confirm(`stop ${name}?`)) await fetch(`/api/runs/${encodeURIComponent(name)}`, { method: "DELETE" }); + }, []); + + if (!runs) return
      loading…
      ; + return ( +
      +
      + + + + + + + + + + + {runs.map(r => ( + (location.hash = `#/runs/${encodeURIComponent(r.jobName)}`)} + className={`cursor-pointer border-t border-zinc-800/60 hover:bg-zinc-900 ${r.jobName === selected ? "bg-zinc-900" : ""}`} + > + + + + + + + ))} + +
      runstatusprogressspend +
      + {r.jobName} + {(r.note || r.role) && ( +
      + {r.role && ( + + {r.role} + + )} + {r.role && r.note ? " · " : ""} + {r.note} +
      + )} +       		+ + + + {fmtUsd(r.costUsd)} + {r.status === "running" && ( + + )} +
      +
      +
      + {detail ? ( + <> +
      + {detail.run.jobName} {" "} + + {detail.run.dataset} · {detail.run.models} + {detail.run.slide ? ` → ${detail.run.slide}` : ""} + +
      +
      + + + {detail.trials.map(t => ( + setTrial(t.name)} + className={`cursor-pointer border-t border-zinc-800/60 hover:bg-zinc-900 ${t.name === trial ? "bg-zinc-900" : ""}`} + > + + + + + + + ))} + +
      {t.task} + + {fmtUsd(t.costUsd)}{t.durationMs ? fmtMin(t.durationMs) : "—"}{t.detail}
      +
      + {trial && ( +
      + {(transcript?.entries ?? []).map((e, i) => ( + // biome-ignore lint/suspicious/noArrayIndexKey: tail window, entries have no ids +
      +
      + {e.kind === "assistant" ? (e.model ?? "assistant") : (e.tool ?? e.kind)} + {e.isError ? " · error" : ""} +
      + {e.text && ( + 
      +												{e.text}
+
      + )} + {e.tools && e.tools.length > 0 && ( +
      → {e.tools.join(", ")}
      + )} +
      + ))} +
      + )} + + ) : ( +
      select a run
      + )} +
      +
      + ); +} + +// ── launch form ────────────────────────────────────────────────────────────── + +function LaunchForm({ onDone }: { onDone: () => void }) { + const [msg, setMsg] = useState(""); + const submit = useCallback( + async (ev: React.FormEvent) => { + ev.preventDefault(); + const f = new FormData(ev.currentTarget); + const body: Record = { model: f.get("model") }; + if (f.get("jobName")) body.jobName = f.get("jobName"); + if (f.get("dataset")) body.dataset = f.get("dataset"); + if (f.get("tasks")) body.tasks = Number(f.get("tasks")); + if (f.get("concurrency")) body.concurrency = Number(f.get("concurrency")); + if (f.get("timeoutMultiplier")) body.timeoutMultiplier = Number(f.get("timeoutMultiplier")); + if (f.get("include")) { + body.include = String(f.get("include")) + .split(",") + .map(s => s.trim()) + .filter(Boolean); + } + if (f.get("goal")) body.goal = f.get("goal"); + if (f.get("role")) body.role = f.get("role"); + if (f.get("note")) body.note = f.get("note"); + const trigger = f.get("slideTrigger"); + if (f.get("slideModel") && trigger) { + const slide: Record = { model: f.get("slideModel"), plan: !!f.get("slidePlan") }; + if (trigger === "on-action") slide.onAction = true; + else slide.turns = Number(f.get("slideTurns") || 8); + body.slide = slide; + } + setMsg("launching…"); + const res = await fetch("/api/runs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const out = (await res.json()) as { jobName?: string; error?: string }; + setMsg(res.ok ? `launched ${out.jobName}` : `error: ${out.error}`); + if (res.ok) setTimeout(onDone, 800); + }, + [onDone], + ); + const input = "rounded border border-zinc-700 bg-zinc-950 px-2 py-1 text-sm"; + return ( +
      + + + + + + + + + + + + + + +
      + + {msg} +
      +
      + ); +} + +// ── shell ──────────────────────────────────────────────────────────────────── + +function App() { + const hash = useHashRoute(); + const [showLaunch, setShowLaunch] = useState(false); + useEffect(() => { + if (hash !== undefined) { + window.scrollTo(0, 0); + } + }, [hash]); + const expMatch = hash.match(/^#\/exp\/(.+)$/); + const runMatch = hash.match(/^#\/runs(?:\/(.+))?$/); + const view = expMatch ? ( + + ) : runMatch ? ( + + ) : ( + + ); + const tab = (href: string, label: string, active: boolean) => ( + + {label} + + ); + return ( + <> +
      +

      harbor-manager

      + +
      + +
      +
      + {showLaunch && setShowLaunch(false)} />} + {view} + + ); +} + +const rootEl = document.getElementById("root"); +if (rootEl) createRoot(rootEl).render(); diff --git a/packages/harbor-manager/src/web/index.html b/packages/harbor-manager/src/web/index.html new file mode 100644 index 000000000..0855d9306 --- /dev/null +++ b/packages/harbor-manager/src/web/index.html @@ -0,0 +1,16 @@ + + + + + +harbor-manager + + + + +
      + + + diff --git a/packages/harbor-manager/tsconfig.json b/packages/harbor-manager/tsconfig.json new file mode 100644 index 000000000..253d814f6 --- /dev/null +++ b/packages/harbor-manager/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.workspace.json", + "compilerOptions": { + "jsx": "react-jsx", + "lib": ["ES2024", "DOM", "DOM.Iterable", "DOM.AsyncIterable"] + }, + "include": [ + "src" + ] +} diff --git a/packages/terminal-bench/README.md b/packages/terminal-bench/README.md deleted file mode 100644 index 4dd12f551..000000000 --- a/packages/terminal-bench/README.md +++ /dev/null @@ -1,131 +0,0 @@ -# @oh-my-pi/terminal-bench - -Run [harbor-framework/terminal-bench-2](https://github.com/harbor-framework/terminal-bench-2) -against the **local `omp` build** with a live progress / spend / success dashboard. - -It drives [Harbor](https://github.com/laude-institute/harbor) (the official TB-2 -harness) under the hood and renders its own dashboard by polling each trial's -`result.json`. - -``` -bun src/runner.ts --model anthropic/claude-sonnet-4-6 --tasks 20 --concurrency 4 -``` - -``` -terminal-bench-2 · omp · anthropic/claude-sonnet-4-6 · conc=4 k=1 -████████████░░░░░░░░ 12/20 elapsed 14:32 eta ~6:10 -pass 9 (75%) fail 2 err 1 run 4 pend 4 -spend $1.84 in 1.2M out 84k cache 3.1M -────────────────────────────────────────────────────── - ✓ fix-git r1.00 $0.12 1:40 - ✗ regex-chess r0.00 $0.31 4:10 - ! qemu-startup — $0.04 0:30 TimeoutError - ⠙ path-tracing · $0.05 2:01 -────────────────────────────────────────────────────── -harbor: ... -``` - -## How it works - -1. **Local omp, not npm.** Harbor's built-in `pi` agent installs the upstream - `@mariozechner/pi-coding-agent`. This runner instead packs the working tree — - `bun pm pack` in `packages/coding-agent`, which bundles every workspace TS - package into `dist/cli.js` — and a custom Harbor agent - ([`agent/omp_local.py`](./agent/omp_local.py)) uploads that tarball into each - task container, installs Bun, `bun install`s the bundle's external deps + the - matching `@oh-my-pi/pi-natives-linux-` prebuilt, and runs - `bun .../dist/cli.js --print --mode json --no-session --auto-approve`. -2. **Auth via the host gateway — no keys in containers.** A generated - `~/.omp/agent/models.yml` routes the model providers' `baseUrl` at the host - pm2 `omp-auth-gateway` (`http://host.docker.internal:4000`, `transport: - pi-native`). The gateway resolves credentials host-side; containers only ever - see a dummy `apiKey`. Cost/tokens are parsed from omp's `message_end` events. -3. **Live dashboard.** Harbor's own output is redirected to `harbor.log`; this - process owns the terminal and polls `/result.json` (authoritative - totals) + `//result.json` (per-task reward/cost/tokens). - -## Prerequisites - -- **Docker** running (Docker Desktop on macOS — provides `host.docker.internal`). -- **Harbor**: `uv tool install harbor` (provides `harbor` on `PATH`). -- **Auth gateway** running on the host (pm2 `omp-auth-gateway`, default - `127.0.0.1:4000`, started `--no-auth` on loopback). Check: - `curl -s 127.0.0.1:4000/healthz`. - -## Usage - -``` -bun src/runner.ts [options] [-- ] -``` - -| Option | Default | Notes | -|---|---|---| -| `-m, --model ` | `anthropic/claude-sonnet-4-6` | Repeatable | -| `-l, --tasks ` | `20` | Max tasks | -| `-n, --concurrency ` | `4` | Concurrent trials | -| `-k, --attempts ` | `1` | Attempts per task (pass@k) | -| `-i/-x, --include/--exclude ` | — | Task filters (repeatable) | -| `--thinking ` | — | `off…max` | -| `--advisor-model

      ` | — | Second model reviewing the primary; spend summed in | -| `--agent ` | `omp` | `oracle`/`nop`/any harbor agent (bypasses omp) | -| `--install ` | `local` | `published` = npm `@oh-my-pi/pi-coding-agent` | -| `--tarball ` / `--no-build` | — | Reuse a prebuilt omp tarball | -| `--gateway-url ` | `http://host.docker.internal:4000` | | -| `--no-gateway` | off | Pass host provider keys into containers instead | -| `--allow-host ` | — | `harbor --allow-agent-host` (allowlist tasks) | -| `-o, --jobs-dir ` | `/runs/tb2` | | -| `--timeout-multiplier ` | — | Scale task timeouts | -| `--dry-run` | off | Print the harbor command + models.yml and exit | - -### Examples - -```bash -# Default 20-task run -bun src/runner.ts -m anthropic/claude-sonnet-4-6 -l 20 -n 4 - -# Compare models (separate runs / job dirs) -bun src/runner.ts -m openai-codex/gpt-5.1-codex -l 20 --thinking high - -# Primary model + advisor model (combined spend reported) -bun src/runner.ts -m anthropic/claude-sonnet-4-6 --advisor-model anthropic/claude-haiku-4-5 -l 20 - -# Cheap pipeline smoke (no model spend — uses task reference solutions) -bun src/runner.ts --agent oracle -l 2 - -# Inspect exactly what will run -bun src/runner.ts --dry-run -l 20 -``` - -## Output - -Per run, under `--jobs-dir`: - -- `/` — Harbor's trial dirs (`result.json` per trial). -- `_bench//report.md` — markdown summary table. -- `_bench//harbor.log` — full Harbor output. -- `_bench//models.yml` — generated gateway routing (dummy key). - -## Caveats - -- **Network policy.** On Harbor's local Docker backend only **public** - agent-phase tasks can reach the host gateway. `no_network` tasks cut off - `host.docker.internal`; allowlist tasks aren't supported by the Docker backend - (use `--allow-host` only when you know the task allows it). Such tasks surface - as agent errors in the dashboard. -- **Architecture.** The native prebuilt is selected from the container's - `uname -m` (linux-arm64 / linux-x64). Alpine/musl base images are unsupported - (no musl prebuilt) and fail fast in `install()`. -- **`web_search` is off by default.** It can't authenticate through the gateway - (it uses dedicated search-provider creds), so it's disabled in the container - config to avoid 401s and false negatives. Re-enable with `--web-search` only - if the container can reach a search provider. The report shows `web_search=on/off`. -- **Advisor spend accuracy vs speed.** `--advisor-model` runs a second model - (separate spend, summed into the reported total + an `(advisor $…)` breakdown). - Its turns are flushed to `__advisor.jsonl` only when the primary stays caught - up, so the default `--advisor-sync 1` waits per turn for accurate spend. - `--advisor-sync off` is faster but can drop end-of-run advisor backlog, - undercounting advisor cost. -- **`--install local` reflects local TS changes** across the monorepo (they're - inlined into `dist/cli.js`), but **not** uncommitted changes to the Rust - natives or other externalized deps (mupdf, puppeteer) — those resolve from npm - at the bundle's version. diff --git a/packages/terminal-bench/package.json b/packages/terminal-bench/package.json deleted file mode 100644 index 33758fe50..000000000 --- a/packages/terminal-bench/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "module", - "private": true, - "name": "@oh-my-pi/terminal-bench", - "version": "0.0.1", - "description": "Run harbor-framework/terminal-bench-2 against the local omp build with a live progress/spend/success dashboard", - "homepage": "https://omp.sh", - "author": "Can Boluk", - "license": "MIT", - "repository": { - "type": "git", - "url": "git+https://github.com/can1357/oh-my-pi.git", - "directory": "packages/terminal-bench" - }, - "bin": { - "tb2": "src/runner.ts" - }, - "scripts": { - "check": "biome check . && bun run check:types", - "check:types": "tsgo -p tsconfig.json --noEmit", - "lint": "biome lint .", - "start": "bun run src/runner.ts" - }, - "devDependencies": { - "@types/bun": "catalog:" - }, - "engines": { - "bun": ">=1.3.14" - } -} diff --git a/packages/terminal-bench/tsconfig.json b/packages/terminal-bench/tsconfig.json deleted file mode 100644 index d9a6e62ad..000000000 --- a/packages/terminal-bench/tsconfig.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "extends": "../tsconfig.workspace.json", - "include": [ - "src" - ] -} From 42d81f189f9b00672e82a515e8049f4b1eae5d3c Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 03:03:26 +0200 Subject: [PATCH 55/80] feat(coding-agent): redesigned agent hub layout for task clarity - Redesigned agent hub entries as two-line cards, separating identity and status from task descriptions. - Added explicit model and thinking level badges to agent status information. - Updated agent hub rendering to support multi-line entries and adaptive vertical scrolling based on row height. - Simplified status display by using glyphs instead of redundant status labels. - Adjusted rendering logic to prioritize displaying agent task summaries on an independent indented line. --- packages/coding-agent/CHANGELOG.md | 1 + .../src/modes/components/agent-hub.ts | 132 ++++++++++++++---- .../test/agent-hub-ordering.test.ts | 17 ++- 3 files changed, 110 insertions(+), 40 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 23e6cc216..cecfd3d86 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -6,6 +6,7 @@ - Added display setting to toggle between collapsing or keeping compacted history inline, now applied to live session displays - Added a compact session-only model picker (Alt+P) for quick model switching without changing roles +- Redesigned Agent Hub entries as two-line cards: identity (status glyph, name, agent type, parent when nested) on the left, active model + reasoning level and age right-aligned, with the task description on its own line; dropped the redundant `sub · of Main` noise ### Changed diff --git a/packages/coding-agent/src/modes/components/agent-hub.ts b/packages/coding-agent/src/modes/components/agent-hub.ts index 1c8413605..5b09acf76 100644 --- a/packages/coding-agent/src/modes/components/agent-hub.ts +++ b/packages/coding-agent/src/modes/components/agent-hub.ts @@ -15,8 +15,8 @@ */ import * as fs from "node:fs"; import * as path from "node:path"; -import type { AgentTool } from "@oh-my-pi/pi-agent-core"; -import { Container, Ellipsis, matchesKey, type OverlayHandle, type TUI } from "@oh-my-pi/pi-tui"; +import { type AgentTool, ThinkingLevel } from "@oh-my-pi/pi-agent-core"; +import { Container, Ellipsis, matchesKey, type OverlayHandle, padding, type TUI, visibleWidth } from "@oh-my-pi/pi-tui"; import { formatAge, getProjectDir, logger } from "@oh-my-pi/pi-utils"; import { ADVISOR_TRANSCRIPT_FILENAME, isAdvisorTranscriptName } from "../../advisor"; import type { KeyId } from "../../config/keybindings"; @@ -25,6 +25,7 @@ import { IrcBus } from "../../irc/bus"; import { AgentLifecycleManager } from "../../registry/agent-lifecycle"; import { type AgentRef, AgentRegistry, type AgentStatus, MAIN_AGENT_ID } from "../../registry/agent-registry"; import { USER_INTERRUPT_LABEL } from "../../session/messages"; +import { parseThinkingLevel } from "../../thinking"; import { replaceTabs, TRUNCATE_LENGTHS, truncateToWidth } from "../../tools/render-utils"; import type { ObservableSession, SessionObserverRegistry } from "../session-observer-registry"; import { theme } from "../theme/theme"; @@ -55,20 +56,50 @@ function clampHubLine(line: string, width: number): string { const STATUS_ORDER: Record = { running: 0, idle: 1, parked: 2, aborted: 3 }; -/** Glyph + status word, colored per theme status conventions. */ -function statusBadge(status: AgentStatus): string { +/** Status glyph, colored per theme status conventions. The title-line counts spell out the words. */ +function statusGlyph(status: AgentStatus): string { switch (status) { case "running": - return theme.fg("accent", `${theme.status.running} running`); + return theme.fg("accent", theme.status.running); case "idle": - return theme.fg("success", `${theme.status.enabled} idle`); + return theme.fg("success", theme.status.enabled); case "parked": - return theme.fg("muted", `${theme.status.shadowed} parked`); + return theme.fg("muted", theme.status.shadowed); case "aborted": - return theme.fg("error", `${theme.status.aborted} aborted`); + return theme.fg("error", theme.status.aborted); } } +/** Model id + thinking level (`sonnet-4-6 ◒ high`), level colored per theme. */ +function formatModelBadge(modelId: string, level: ThinkingLevel | undefined): string { + const model = theme.fg("muted", replaceTabs(modelId)); + if (!level || level === ThinkingLevel.Off || level === ThinkingLevel.Inherit) return model; + const display = theme.thinking[level as keyof typeof theme.thinking] ?? level; + return `${model} ${theme.getThinkingBorderColor(level)(display)}`; +} + +/** + * Active model + reasoning level for a hub row: live session state when the + * agent is attached, else the executor-reported `resolvedModel` selector + * (`provider/id`, optionally `:`). Undefined when neither is known + * (e.g. a parked historical agent restored from disk). + */ +function modelBadge(ref: AgentRef, observed: ObservableSession | undefined): string | undefined { + const model = ref.session?.model; + if (model) { + const level = model.thinking ? ref.session?.thinkingLevel : undefined; + return formatModelBadge(model.id, level); + } + const resolved = observed?.progress?.resolvedModel; + if (!resolved) return undefined; + // Model ids may themselves contain colons (`qwen3:14b`), so only treat the + // suffix as a thinking level when it parses as one. + const colon = resolved.lastIndexOf(":"); + const level = colon >= 0 ? parseThinkingLevel(resolved.slice(colon + 1)) : undefined; + const selector = level !== undefined ? resolved.slice(0, colon) : resolved; + return formatModelBadge(selector.slice(selector.indexOf("/") + 1), level); +} + async function registerPersistedSubagents( registry: AgentRegistry, sessionFile: string | null | undefined, @@ -427,17 +458,31 @@ export class AgentHubOverlayComponent extends Container { } else { const termHeight = process.stdout.rows || 40; // Chrome: 2 borders + title + notice? + blank + hints + border - const maxVisible = Math.max(3, termHeight - 7 - (this.#notice ? 1 : 0)); - let start = 0; - if (this.#rows.length > maxVisible) { - start = Math.min( - Math.max(0, this.#selectedRow - Math.floor(maxVisible / 2)), - this.#rows.length - maxVisible, - ); + const budget = Math.max(4, termHeight - 7 - (this.#notice ? 1 : 0)); + const entries = this.#rows.map((ref, i) => this.#renderEntry(ref, i === this.#selectedRow, width)); + // Entries are 1-2 lines tall; grow a window around the selection until + // the line budget is spent, so the selected entry stays centered. + let start = this.#selectedRow; + let end = this.#selectedRow + 1; + let used = entries[start]?.length ?? 0; + for (let grew = true; grew; ) { + grew = false; + if (end < entries.length && used + entries[end].length <= budget) { + used += entries[end].length; + end++; + grew = true; + } + if (start > 0 && used + entries[start - 1].length <= budget) { + start--; + used += entries[start].length; + grew = true; + } + } + if (start > 0) { + lines.push(` ${theme.fg("dim", `… ${start} more`)}`); } - const end = Math.min(start + maxVisible, this.#rows.length); for (let i = start; i < end; i++) { - lines.push(this.#renderRow(this.#rows[i], i === this.#selectedRow, width)); + lines.push(...entries[i]); } if (end < this.#rows.length) { lines.push(` ${theme.fg("dim", `… ${this.#rows.length - end} more`)}`); @@ -466,26 +511,51 @@ export class AgentHubOverlayComponent extends Container { return parts.join(theme.sep.dot); } - #renderRow(ref: AgentRef, selected: boolean, width: number): string { + /** + * One agent entry, 1-2 lines: + * `❯ ⟳ Name type ↳ parent ⧉ 2 ········ model ◒ level · age` — identity + * left, metadata right-aligned (inlined when the terminal is too narrow) — + * plus an indented dim task line when the agent's work is known. + */ + #renderEntry(ref: AgentRef, selected: boolean, width: number): string[] { + const max = Math.max(1, width - 2); const cursor = selected ? theme.fg("accent", theme.nav.cursor) : " "; - const parts: string[] = [statusBadge(ref.status), theme.bold(replaceTabs(ref.id))]; - parts.push(theme.fg("dim", replaceTabs(ref.displayName))); - parts.push(theme.fg("dim", ref.parentId ? `${ref.kind} · of ${ref.parentId}` : ref.kind)); - if (ref.kind === "advisor") { - parts.push(theme.fg("warning", "read-only")); + const fields: string[] = [`${cursor} ${statusGlyph(ref.status)} ${theme.bold(replaceTabs(ref.id))}`]; + if (ref.displayName && ref.displayName !== ref.id) { + fields.push(theme.fg("dim", replaceTabs(ref.displayName))); } - const observed = this.#observableFor(ref.id); - const task = observed?.description ?? observed?.progress?.task; - if (task) { - parts.push(theme.fg("muted", sanitizeLine(task, TRUNCATE_LENGTHS.TITLE))); + if (ref.parentId && ref.parentId !== MAIN_AGENT_ID) { + fields.push(theme.fg("dim", `↳ ${replaceTabs(ref.parentId)}`)); + } + if (ref.kind === "advisor") { + fields.push(theme.fg("warning", "read-only")); } const unread = this.#irc.unreadCount(ref.id); if (unread > 0) { - parts.push(theme.fg("warning", `⧉ ${unread}`)); + fields.push(theme.fg("warning", `⧉ ${unread}`)); } - parts.push(theme.fg("dim", formatAge(Math.max(1, Math.round((Date.now() - ref.lastActivity) / 1000))))); - const rawLine = ` ${cursor} ${parts.join(theme.sep.dot)}`; - return truncateToWidth(rawLine.replace(/[\r\n]+/g, " "), Math.max(1, width - 1)); + const left = ` ${fields.join(" ")}`; + + const observed = this.#observableFor(ref.id); + const meta: string[] = []; + const badge = modelBadge(ref, observed); + if (badge) meta.push(badge); + meta.push(theme.fg("dim", formatAge(Math.max(1, Math.round((Date.now() - ref.lastActivity) / 1000))))); + const right = meta.join(theme.sep.dot); + + const leftWidth = visibleWidth(left); + const rightWidth = visibleWidth(right); + const line = + leftWidth + 2 + rightWidth <= max + ? left + padding(max - leftWidth - rightWidth) + right + : truncateToWidth(`${left} ${right}`.replace(/[\r\n]+/g, " "), max); + const entry = [line]; + + const task = observed?.description ?? observed?.progress?.task ?? ref.activity; + if (task) { + entry.push(` ${theme.fg("muted", sanitizeLine(task, Math.max(10, max - 5)))}`); + } + return entry; } #handleTableInput(keyData: string): void { diff --git a/packages/coding-agent/test/agent-hub-ordering.test.ts b/packages/coding-agent/test/agent-hub-ordering.test.ts index 1c34634ad..c246af4c0 100644 --- a/packages/coding-agent/test/agent-hub-ordering.test.ts +++ b/packages/coding-agent/test/agent-hub-ordering.test.ts @@ -53,15 +53,14 @@ function makeHub(agents: AgentRegistry) { } function renderedAgentIds(hub: AgentHubOverlayComponent): string[] { - return hub - .render(120) - .map(line => Bun.stripANSI(line)) - .map(line => line.split(" · ")) - .filter( - parts => - parts.length >= 4 && ["running", "idle", "parked", "aborted"].some(status => parts[0].endsWith(status)), - ) - .map(parts => parts[1]!); + // Entry first lines are ` …`; task lines are + // indented deeper and chrome lines never carry the cursor slot. + const ids: string[] = []; + for (const raw of hub.render(120)) { + const match = /^ (?:❯| ) (\S+) (\S+)/u.exec(Bun.stripANSI(raw)); + if (match) ids.push(match[2]!); + } + return ids; } describe("Agent hub row ordering", () => { From ac1625361301e025cd9972306a2f068c57597f73 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 03:08:21 +0200 Subject: [PATCH 56/80] wip: rslide (experimental) --- packages/coding-agent/CHANGELOG.md | 4 + packages/coding-agent/src/cli/args.ts | 12 + packages/coding-agent/src/cli/flag-tables.ts | 12 + packages/coding-agent/src/commands/launch.ts | 18 + packages/coding-agent/src/main.ts | 58 ++ .../system/reasoning-slide-checklist.md | 7 + .../system/reasoning-slide-continue.md | 1 + .../prompts/system/reasoning-slide-plan.md | 11 + packages/coding-agent/src/sdk.ts | 5 +- .../coding-agent/src/session/agent-session.ts | 234 +++++++- .../agent-session-reasoning-slide.test.ts | 502 ++++++++++++++++++ .../test/agent-session-retry-fallback.test.ts | 2 +- 12 files changed, 861 insertions(+), 5 deletions(-) create mode 100644 packages/coding-agent/src/prompts/system/reasoning-slide-checklist.md create mode 100644 packages/coding-agent/src/prompts/system/reasoning-slide-continue.md create mode 100644 packages/coding-agent/src/prompts/system/reasoning-slide-plan.md create mode 100644 packages/coding-agent/test/agent-session-reasoning-slide.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index cecfd3d86..fa441dfcc 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -25,11 +25,15 @@ - Fixed transcript rebuilds (compaction, `/compact`, and toggling history display) repainting content below stale scrollback when collapsing history; rebuilds now correctly clear the scrollback buffer when history is collapsed - Improved auto-compaction to automatically drop images and elide content when context is tight, and added persistent warning badges to the compaction divider when manual intervention is required - Fixed backgrounded Bash blocks continuing to repaint with live and final job output; they now freeze with a compact job notice while completion is delivered separately +- Fixed `--reasoning-slide-plan` silently ending the run with no code written when the model answered the plan nudge with a text-only reply (no tool call): the agent loop treats a tool-call-free turn as a natural stop and never prompts again, which the nudge's own "write the plan in your next reply" instruction makes common. The nudge now explicitly tells the model this is a checkpoint, not a final answer, and the session forces one more turn whenever a post-nudge reply lands with zero tool calls ## [16.4.8] - 2026-07-12 ### Added - Added a predicate form to the browser run's `wait()` helper: `wait(fn, { timeout?, interval? })` polls the function (sync or async) until truthy and resolves with that value, failing with a named timeout error (deadline clamped under the cell budget so it always beats the opaque whole-cell timeout) instead of Bun's `sleep expects a number` or a whole-cell stall from in-page polling Promises; both `wait` forms now register in the stall diagnosis of cell timeouts +- Added `--reasoning-slide-model` and `--reasoning-slide-turns` to switch a running agent from its initial model after a fixed number of completed assistant turns +- Added `--reasoning-slide-plan` (with `--reasoning-slide-plan-at`) to steer a hidden deep-planning nudge into the run before the reasoning slide; the switch is held until a substantial plan turn actually lands (bounded by a grace window) and the nudge is scrubbed from the LLM context at the switch so the fast model inherits only the produced plan +- Added `--reasoning-slide-on-action` to trigger the reasoning slide at the first completed turn that ran an edit/write tool instead of a fixed turn count (bash is excluded — it doubles as exploration) ### Changed diff --git a/packages/coding-agent/src/cli/args.ts b/packages/coding-agent/src/cli/args.ts index d5a0b18d4..f3b473c74 100644 --- a/packages/coding-agent/src/cli/args.ts +++ b/packages/coding-agent/src/cli/args.ts @@ -27,6 +27,12 @@ export interface Args { smol?: string; slow?: string; plan?: string; + reasoningSlideModel?: string; + reasoningSlideTurns?: string; + reasoningSlideOnAction?: boolean; + reasoningSlidePlan?: boolean; + reasoningSlidePlanAt?: string; + reasoningSlideChecklist?: boolean; maxTime?: number; apiKey?: string; systemPrompt?: string; @@ -231,6 +237,12 @@ export function parseArgs(inputArgs: string[], extensionFlags?: Map = { "--plan": (result, value) => { result.plan = value; }, + "--reasoning-slide-model": (result, value) => { + result.reasoningSlideModel = value; + }, + "--reasoning-slide-turns": (result, value) => { + result.reasoningSlideTurns = value; + }, + "--reasoning-slide-plan-at": (result, value) => { + result.reasoningSlidePlanAt = value; + }, "--max-time": (result, value, deps) => { const seconds = Number(value); if (Number.isFinite(seconds) && seconds > 0) { @@ -275,6 +284,9 @@ export const VALUELESS_FLAGS: ReadonlySet = new Set([ "--no-pty", "--hide-thinking", "--advisor", + "--reasoning-slide-plan", + "--reasoning-slide-on-action", + "--reasoning-slide-checklist", "--print", "--print-thoughts", "--no-extensions", diff --git a/packages/coding-agent/src/commands/launch.ts b/packages/coding-agent/src/commands/launch.ts index 5c559d6ea..0e24e1328 100644 --- a/packages/coding-agent/src/commands/launch.ts +++ b/packages/coding-agent/src/commands/launch.ts @@ -34,6 +34,24 @@ export default class Index extends Command { plan: Flags.string({ description: "Plan model for architectural planning (or PI_PLAN_MODEL env)", }), + "reasoning-slide-model": Flags.string({ + description: "Switch to this model after --reasoning-slide-turns completed assistant turns", + }), + "reasoning-slide-turns": Flags.string({ + description: "Positive number of completed assistant turns before the reasoning slide", + }), + "reasoning-slide-on-action": Flags.boolean({ + description: "Trigger the reasoning slide at the first completed turn that ran edit/write", + }), + "reasoning-slide-plan": Flags.boolean({ + description: "Inject a hidden deep-plan nudge before the reasoning slide, scrubbed at the switch", + }), + "reasoning-slide-plan-at": Flags.string({ + description: "Completed assistant turns before the plan nudge is injected (default 1)", + }), + "reasoning-slide-checklist": Flags.boolean({ + description: "Steer a hidden verify-before-finishing checklist into the run at the switch", + }), provider: Flags.string({ description: "Provider to use (legacy; prefer --model)", }), diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 0b2b119d9..8cb37eb3d 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -903,6 +903,64 @@ export async function buildSessionOptions( if (!options.model) options.model = scopedModels[0].model; } + const hasReasoningSlideModel = parsed.reasoningSlideModel !== undefined; + const hasReasoningSlideTurns = parsed.reasoningSlideTurns !== undefined; + const hasReasoningSlideOnAction = parsed.reasoningSlideOnAction === true; + if (hasReasoningSlideModel && hasReasoningSlideTurns === hasReasoningSlideOnAction) { + throw new Error( + "--reasoning-slide-model requires exactly one trigger: --reasoning-slide-turns or --reasoning-slide-on-action", + ); + } + if ((hasReasoningSlideTurns || hasReasoningSlideOnAction) && !hasReasoningSlideModel) { + throw new Error("--reasoning-slide-turns/--reasoning-slide-on-action require --reasoning-slide-model"); + } + if (parsed.reasoningSlidePlan && !hasReasoningSlideModel) { + throw new Error("--reasoning-slide-plan requires a reasoning slide (--reasoning-slide-model + trigger)"); + } + if (parsed.reasoningSlidePlanAt !== undefined && !parsed.reasoningSlidePlan) { + throw new Error("--reasoning-slide-plan-at requires --reasoning-slide-plan"); + } + if (hasReasoningSlideModel) { + let afterTurns: number | undefined; + if (hasReasoningSlideTurns) { + afterTurns = Number(parsed.reasoningSlideTurns); + if (!Number.isSafeInteger(afterTurns) || afterTurns < 1) { + throw new Error("--reasoning-slide-turns must be a positive integer"); + } + } + let planAtTurn: number | undefined; + if (parsed.reasoningSlidePlanAt !== undefined) { + planAtTurn = Number(parsed.reasoningSlidePlanAt); + const belowTurns = afterTurns === undefined || planAtTurn < afterTurns; + if (!Number.isSafeInteger(planAtTurn) || planAtTurn < 1 || !belowTurns) { + throw new Error("--reasoning-slide-plan-at must be a positive integer below --reasoning-slide-turns"); + } + } + const resolved = resolveCliModel({ + cliModel: parsed.reasoningSlideModel, + modelRegistry, + preferences: modelMatchPreferences, + }); + if (resolved.warning) { + process.stderr.write(`${chalk.yellow(`Warning: ${resolved.warning}`)}\n`); + } + if (resolved.error || !resolved.model) { + throw new Error(resolved.error ?? `Model "${parsed.reasoningSlideModel}" not found`); + } + if (!modelRegistry.hasConfiguredAuth(resolved.model)) { + throw new Error(`No API key for ${resolved.model.provider}/${resolved.model.id}`); + } + options.reasoningSlide = { + target: resolved.model, + afterTurns, + onFirstAction: hasReasoningSlideOnAction || undefined, + thinkingLevel: resolved.thinkingLevel, + plan: parsed.reasoningSlidePlan === true, + planAtTurn, + checklist: parsed.reasoningSlideChecklist === true, + }; + } + // Thinking level if (parsed.thinking) { options.thinkingLevel = parsed.thinking; diff --git a/packages/coding-agent/src/prompts/system/reasoning-slide-checklist.md b/packages/coding-agent/src/prompts/system/reasoning-slide-checklist.md new file mode 100644 index 000000000..5b1fc7bfb --- /dev/null +++ b/packages/coding-agent/src/prompts/system/reasoning-slide-checklist.md @@ -0,0 +1,7 @@ +Before you consider this task finished, verify: + +- Consistency: if you changed a pattern, signature, or check in one place, grep for every other call site or duplicate copy that needs the identical change. A fix applied to only some of the matching sites is still a failure. +- Scope: if your diff does more than the minimal change needed to resolve the issue, confirm you have not altered behavior for any case outside the reported issue. Prefer the smallest correct diff over a broader rewrite. +- Verification: run the full test module or file the issue lives in, not just the one test you expect to flip. A change that breaks a sibling test is not a fix. + +Do not claim the task is complete until you have done these three checks. diff --git a/packages/coding-agent/src/prompts/system/reasoning-slide-continue.md b/packages/coding-agent/src/prompts/system/reasoning-slide-continue.md new file mode 100644 index 000000000..04dd27c9c --- /dev/null +++ b/packages/coding-agent/src/prompts/system/reasoning-slide-continue.md @@ -0,0 +1 @@ +Continue the task now — do not end your turn here. diff --git a/packages/coding-agent/src/prompts/system/reasoning-slide-plan.md b/packages/coding-agent/src/prompts/system/reasoning-slide-plan.md new file mode 100644 index 000000000..f41d99893 --- /dev/null +++ b/packages/coding-agent/src/prompts/system/reasoning-slide-plan.md @@ -0,0 +1,11 @@ +Stop and write the complete plan in your NEXT reply — before any further exploration. You have already seen enough to commit to a plan; do not defer this. + +Lay out, explicitly and comprehensively: + +- Every remaining step in execution order, with the exact files, commands, and checks involved. +- Known risks, edge cases, and how you will verify each step actually landed (specific commands, expected outputs). Never modify tests or verification assets to make checks pass. +- What is already done, stated briefly, so no step gets repeated. + +Be thorough and concrete — this plan is the reference for the remainder of the run. You may verify details with tools after the plan is written, never before. + +This is a checkpoint, not a final answer: do not end your turn on the plan alone. In the SAME reply, immediately after the plan, call whatever tool starts the first step — do not stop here. diff --git a/packages/coding-agent/src/sdk.ts b/packages/coding-agent/src/sdk.ts index d03b414a4..80cc4e80d 100644 --- a/packages/coding-agent/src/sdk.ts +++ b/packages/coding-agent/src/sdk.ts @@ -109,7 +109,7 @@ import { obfuscateProviderContext, SecretObfuscator, } from "./secrets"; -import { AgentSession } from "./session/agent-session"; +import { AgentSession, type ReasoningSlide } from "./session/agent-session"; import { discoverAuthStorage as discoverAuthStorageFromConfig } from "./session/auth-broker-config"; import type { AuthStorage } from "./session/auth-storage"; import { @@ -407,6 +407,8 @@ export interface CreateAgentSessionOptions { thinkingLevel?: ConfiguredThinkingLevel; /** Models available for cycling (Ctrl+P in interactive mode) */ scopedModels?: Array<{ model: Model; thinkingLevel?: ThinkingLevel }>; + /** One-way model switch after a fixed number of completed assistant turns. */ + reasoningSlide?: ReasoningSlide; /** Provider-facing system prompt override. Replaces the fully rendered default blocks. */ systemPrompt?: string | string[] | ((defaultPrompt: string[]) => string | string[]); @@ -2863,6 +2865,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} agent, pruneToolDescriptions: inlineToolDescriptors, thinkingLevel: autoThinking ? AUTO_THINKING : effectiveThinkingLevel, + reasoningSlide: options.reasoningSlide, serviceTierByFamily: initialServiceTierByFamily, sessionManager, settings, diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index f800310a6..b8363c07b 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -270,6 +270,9 @@ import planModeReferencePrompt from "../prompts/system/plan-mode-reference.md" w import planModeToolDecisionReminderPrompt from "../prompts/system/plan-mode-tool-decision-reminder.md" with { type: "text", }; +import reasoningSlideChecklistPrompt from "../prompts/system/reasoning-slide-checklist.md" with { type: "text" }; +import reasoningSlideContinuePrompt from "../prompts/system/reasoning-slide-continue.md" with { type: "text" }; +import reasoningSlidePlanPrompt from "../prompts/system/reasoning-slide-plan.md" with { type: "text" }; import rewindReportTemplate from "../prompts/system/rewind-report.md" with { type: "text" }; import sideChannelNoToolsReminder from "../prompts/system/side-channel-no-tools.md" with { type: "text" }; import thinkingLoopRedirectTemplate from "../prompts/system/thinking-loop-redirect.md" with { type: "text" }; @@ -413,7 +416,30 @@ const MID_RUN_TODO_NUDGE_MUTATING_TOOLS: Record = { /** `customType` for the hidden mid-run todo nudge; `display: false`, so it reaches * the model but never renders in the TUI or transcript. */ const MID_RUN_TODO_NUDGE_MESSAGE_TYPE = "mid-run-todo-nudge"; - +/** Hidden plan-burst nudge injected by the reasoning slide; scrubbed from the + * LLM context when the slide switches models. */ +const REASONING_SLIDE_PLAN_MESSAGE_TYPE = "reasoning-slide-plan"; +/** Hidden safety-net nudge forcing one more turn after a text-only reply to + * the plan nudge, which would otherwise end the run with no code written. */ +const REASONING_SLIDE_CONTINUE_MESSAGE_TYPE = "reasoning-slide-continue"; +/** Hidden "verify before finishing" checklist steered into the run at the + * switch, aimed at the fast model's specific failure patterns: partial + * multi-site fixes, unnecessarily broad rewrites, and reported-test-only + * verification. */ +const REASONING_SLIDE_CHECKLIST_MESSAGE_TYPE = "reasoning-slide-checklist"; +/** Minimum visible text length for a post-nudge assistant turn to count as the + * delivered plan; exploration turns emit short connective text and stay under. */ +const REASONING_SLIDE_PLAN_MIN_CHARS = 400; +/** Extra turns past `afterTurns` the slide waits for the plan before switching anyway. */ +const REASONING_SLIDE_PLAN_GRACE_TURNS = 4; +/** Tools whose first successful call marks the execution phase for + * {@link ReasoningSlide.onFirstAction}-triggered switches. Bash is + * deliberately excluded: it doubles as exploration (ls/cat) and fired + * turn-1 switches in practice. */ +const REASONING_SLIDE_ACTION_TOOLS: Record = { + edit: true, + write: true, +}; /** Abort reason for the Gemini reasoning-header runaway interrupt. Surfaced on the * discarded assistant turn only; never reaches the model. */ const GEMINI_HEADER_INTERRUPT_REASON = "Interrupted: emit a tool call instead of more planning"; @@ -599,8 +625,9 @@ const COMPACTION_CHECK_BLOCK_AUTOMATIC_CONTINUATION: CompactionCheckResult = { /** * User-facing notice for a compaction dead end: maintenance freed too little - * to retry safely. `remedies` names the recovery actions available on the - * emitting path (the shake-rescue path can additionally offer `/shake images`). + * to retry safely. `remedies` names the recovery actions left on the emitting + * path — by the time the post-pass dead end fires, the tiered rescue has + * already attempted both elide and image-drop automatically. */ function compactionDeadEndWarning(remedies: string): string { return ( @@ -680,6 +707,36 @@ export interface AsyncJobSnapshot { } export type { ShakeMode, ShakeResult }; +/** + * Switches an active session from its initial model one-way at a completed + * assistant-turn boundary. Trigger is either a fixed turn count + * ({@link afterTurns}) or the first turn that ran an action tool + * ({@link onFirstAction}); exactly one must be set. + */ +export interface ReasoningSlide { + target: Model; + /** Switch after this many completed assistant turns. */ + afterTurns?: number; + /** Switch at the first completed turn that ran an edit/write tool. */ + onFirstAction?: boolean; + thinkingLevel?: ConfiguredThinkingLevel; + /** + * Plan burst: after {@link planAtTurn} completed turns, steer a hidden + * "lay out the complete plan" nudge into the run so the primary model + * spends its remaining turns producing a comprehensive plan. The nudge is + * removed from the LLM context at the switch; the plan itself stays. + */ + plan?: boolean; + /** Completed assistant turns before the plan nudge is injected (default 1). */ + planAtTurn?: number; + /** + * Steer a hidden "before you finish, verify..." checklist into the run at + * the moment of the switch — consistency across matching call sites, + * diff scope vs. the reported issue, running the full test module rather + * than only the reported test. Independent of {@link plan}. + */ + checklist?: boolean; +} // ============================================================================ // Types @@ -695,6 +752,9 @@ export interface AgentSessionConfig { scopedModels?: Array<{ model: Model; thinkingLevel?: ThinkingLevel }>; /** Initial session thinking selector. */ thinkingLevel?: ConfiguredThinkingLevel; + /** Switch model after this many completed assistant turns. */ + reasoningSlide?: ReasoningSlide; + /** Initial per-family service tiers (OpenAI / Anthropic / Google) for the live session. */ serviceTierByFamily?: ServiceTierByFamily; /** Prompt templates for expansion */ @@ -1607,6 +1667,13 @@ export class AgentSession { #autoThinking: boolean = false; /** The level `auto` last resolved to (for UI); undefined until a turn is classified. */ #autoResolvedLevel: Effort | undefined; + #reasoningSlide: ReasoningSlide | undefined; + #reasoningSlideTurnCount = 0; + /** True once the plan-burst nudge has been queued; scrubbed from context at the switch. */ + #reasoningSlidePlanInjected = false; + /** True once a post-nudge assistant turn delivered a substantial written plan. */ + #reasoningSlidePlanDelivered = false; + #promptTemplates: PromptTemplate[]; #slashCommands: FileSlashCommand[]; @@ -2091,6 +2158,155 @@ export class AgentSession { this.#emit(pending); } + /** Advance the configured one-way model switch at a successful assistant-turn boundary. */ + async #advanceReasoningSlide(liveMessages: AgentMessage[], context: AgentTurnEndContext | undefined): Promise { + const reasoningSlide = this.#reasoningSlide; + if (!reasoningSlide || context?.message.role !== "assistant") return; + + this.#reasoningSlideTurnCount++; + // Structural safety net, scoped to the plan nudge specifically: every + // branch below assumes the agent loop will run another turn. It won't + // if THIS turn had no tool calls — the loop treats a text-only turn as + // "the agent is done" and ends the session with no further prompting. + // A bare fixed-turn/action slide never provokes that (a genuine + // text-only stop there is normal agent behavior and must be honored), + // but the plan nudge explicitly asks for a prose reply, which makes a + // text-only turn common right after it — observed silently killing + // production SWE-bench runs before any code was ever written. Force + // one more turn only in that specific, self-created hazard window. + if (reasoningSlide.plan && this.#reasoningSlidePlanInjected && context.toolResults.length === 0) { + this.agent.steer({ + role: "custom", + customType: REASONING_SLIDE_CONTINUE_MESSAGE_TYPE, + content: reasoningSlideContinuePrompt, + attribution: "agent", + display: false, + timestamp: Date.now(), + }); + } + this.#noteReasoningSlidePlanDelivery(reasoningSlide, context.message); + let trigger: string; + if (reasoningSlide.onFirstAction) { + // Action trigger: the first turn that mutates the world (edit/write/ + // bash) marks the start of the execution phase — switch right there. + const action = context.toolResults.find(result => REASONING_SLIDE_ACTION_TOOLS[result.toolName]); + if (!action) { + this.#maybeInjectReasoningSlidePlanNudge(reasoningSlide); + return; + } + trigger = `first ${action.toolName} call (turn ${this.#reasoningSlideTurnCount})`; + } else { + const afterTurns = reasoningSlide.afterTurns ?? 1; + if (this.#reasoningSlideTurnCount < afterTurns) { + this.#maybeInjectReasoningSlidePlanNudge(reasoningSlide); + return; + } + // Hold the switch until the nudged plan actually lands: sliding mid- + // exploration hands the fast model a transcript with an unfulfilled + // promise to plan (observed as test-doctoring drift on SWE-bench). + // Bounded so a model that refuses to plan still slides eventually. + if ( + reasoningSlide.plan && + this.#reasoningSlidePlanInjected && + !this.#reasoningSlidePlanDelivered && + this.#reasoningSlideTurnCount < afterTurns + REASONING_SLIDE_PLAN_GRACE_TURNS + ) { + return; + } + trigger = `${this.#reasoningSlideTurnCount} completed turns`; + } + await this.#waitForSessionMessagePersistence(context.message); + for (const toolResult of context.toolResults) { + await this.#waitForSessionMessagePersistence(toolResult); + } + + this.#scrubReasoningSlidePlanNudge(liveMessages); + const target = reasoningSlide.target; + if (this.model && modelsAreEqual(this.model, target)) { + this.#reasoningSlide = undefined; + return; + } + + await this.setModelTemporary(target, reasoningSlide.thinkingLevel, { ephemeral: true }); + this.#reasoningSlide = undefined; + this.emitNotice( + "info", + `Reasoning slide: switched to ${target.provider}/${target.id} after ${trigger}.`, + "reasoning-slide", + ); + if (reasoningSlide.checklist) { + this.agent.steer({ + role: "custom", + customType: REASONING_SLIDE_CHECKLIST_MESSAGE_TYPE, + content: reasoningSlideChecklistPrompt, + attribution: "agent", + display: false, + timestamp: Date.now(), + }); + } + } + + /** + * Plan-delivery detector: a post-nudge assistant turn whose visible text is + * substantial ({@link REASONING_SLIDE_PLAN_MIN_CHARS}+) is taken as the + * written plan. Exploration turns emit short connective text ("Let me read + * X first") and never trip this. + */ + #noteReasoningSlidePlanDelivery(reasoningSlide: ReasoningSlide, message: AgentTurnEndContext["message"]): void { + if (!reasoningSlide.plan || !this.#reasoningSlidePlanInjected || this.#reasoningSlidePlanDelivered) return; + if (message.role !== "assistant") return; + let textChars = 0; + for (const block of message.content) { + if (block.type === "text") textChars += block.text.length; + } + if (textChars >= REASONING_SLIDE_PLAN_MIN_CHARS) { + this.#reasoningSlidePlanDelivered = true; + this.emitNotice("info", "Reasoning slide: plan delivered.", "reasoning-slide"); + } + } + + /** + * Plan burst: once {@link ReasoningSlide.planAtTurn} turns have completed, + * steer a hidden deep-planning nudge into the run so the primary model's + * remaining pre-slide turns produce a comprehensive plan for the fast model + * to execute. Injected at most once per slide. + */ + #maybeInjectReasoningSlidePlanNudge(reasoningSlide: ReasoningSlide): void { + if (!reasoningSlide.plan || this.#reasoningSlidePlanInjected) return; + if (this.#reasoningSlideTurnCount < (reasoningSlide.planAtTurn ?? 1)) return; + this.#reasoningSlidePlanInjected = true; + this.agent.steer({ + role: "custom", + customType: REASONING_SLIDE_PLAN_MESSAGE_TYPE, + content: reasoningSlidePlanPrompt, + display: false, + attribution: "agent", + timestamp: Date.now(), + }); + this.emitNotice("info", "Reasoning slide: injected deep-plan nudge.", "reasoning-slide"); + } + + /** + * Remove the plan-burst nudge from the LLM context before the model + * switch: the fast model inherits the plan the nudge produced, not the + * nudge itself. Splices the loop's live context array in place (the run + * streams from it) and mirrors the removal into agent state. The persisted + * transcript keeps the message for audit; a session reload re-materializes + * it, which is acceptable for the benchmark-oriented single-run lifecycle + * this feature targets. + */ + #scrubReasoningSlidePlanNudge(liveMessages: AgentMessage[]): void { + if (!this.#reasoningSlidePlanInjected) return; + const isPlanNudge = (m: AgentMessage): boolean => + m.role === "custom" && m.customType === REASONING_SLIDE_PLAN_MESSAGE_TYPE; + for (let i = liveMessages.length - 1; i >= 0; i--) { + if (isPlanNudge(liveMessages[i])) liveMessages.splice(i, 1); + } + const stateMessages = this.agent.state.messages; + const filtered = stateMessages.filter(m => !isPlanNudge(m)); + if (filtered.length !== stateMessages.length) this.agent.replaceMessages(filtered); + } + constructor(config: AgentSessionConfig) { this.agent = config.agent; this.sessionManager = config.sessionManager; @@ -2111,7 +2327,18 @@ export class AgentSession { } else { this.#thinkingLevel = config.thinkingLevel; } + if (config.reasoningSlide) { + const { afterTurns, onFirstAction } = config.reasoningSlide; + if ((afterTurns !== undefined) === (onFirstAction === true)) { + throw new Error("reasoningSlide requires exactly one trigger: afterTurns or onFirstAction"); + } + if (afterTurns !== undefined && (!Number.isSafeInteger(afterTurns) || afterTurns < 1)) { + throw new Error("reasoningSlide.afterTurns must be a positive integer"); + } + this.#reasoningSlide = config.reasoningSlide; + } this.#applyThinkingLevelToAgent(this.#thinkingLevel); + this.#promptTemplates = config.promptTemplates ?? []; this.#slashCommands = config.slashCommands ?? []; this.#extensionRunner = config.extensionRunner; @@ -2184,6 +2411,7 @@ export class AgentSession { }); if (detection) this.#maybeInjectToolCallLoopRedirect(messages, detection); } + await this.#advanceReasoningSlide(messages, context); this.#advisorPrimaryTurnsCompleted++; if (this.#advisors.length > 0) { for (const a of this.#advisors) { diff --git a/packages/coding-agent/test/agent-session-reasoning-slide.test.ts b/packages/coding-agent/test/agent-session-reasoning-slide.test.ts new file mode 100644 index 000000000..eedd871e1 --- /dev/null +++ b/packages/coding-agent/test/agent-session-reasoning-slide.test.ts @@ -0,0 +1,502 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import * as path from "node:path"; +import { Agent, type AgentTool } from "@oh-my-pi/pi-agent-core"; +import { type Api, Effort, type Model, z } from "@oh-my-pi/pi-ai"; +import { createMockModel, type MockResponse } from "@oh-my-pi/pi-ai/providers/mock"; +import { getBundledModel } from "@oh-my-pi/pi-catalog/models"; +import { ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry"; +import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import { AgentSession } from "@oh-my-pi/pi-coding-agent/session/agent-session"; +import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage"; +import { convertToLlm } from "@oh-my-pi/pi-coding-agent/session/messages"; +import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager"; +import { TempDir } from "@oh-my-pi/pi-utils"; + +describe("AgentSession reasoning slide", () => { + let tempDir: TempDir; + let authStorage: AuthStorage; + let session: AgentSession | undefined; + + beforeEach(async () => { + tempDir = TempDir.createSync("@pi-reasoning-slide-"); + authStorage = await AuthStorage.create(path.join(tempDir.path(), "auth.db")); + authStorage.setRuntimeApiKey("anthropic", "test-key"); + }); + + afterEach(async () => { + if (session) await session.dispose(); + authStorage.close(); + tempDir.removeSync(); + }); + + function modelOrThrow(id: string): Model { + const model = getBundledModel("anthropic", id); + if (!model) throw new Error(`Expected bundled model ${id}`); + return model; + } + + it("uses the primary model for N completed turns before sliding to the target", async () => { + const primary = modelOrThrow("claude-sonnet-4-5"); + const target = modelOrThrow("claude-sonnet-4-6"); + const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir.path(), "models.yml")); + const requestedModels: string[] = []; + const mock = createMockModel({ + responses: [{ content: ["first"] }, { content: ["second"] }, { content: ["third"] }], + }); + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model: primary, + systemPrompt: ["Test"], + tools: [], + messages: [], + thinkingLevel: Effort.Medium, + }, + streamFn: (model, context, options) => { + requestedModels.push(`${model.provider}/${model.id}`); + return mock.stream(model, context, options); + }, + }); + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry, + reasoningSlide: { target, afterTurns: 2 }, + }); + + await session.prompt("first task"); + await session.prompt("second task"); + await session.prompt("third task"); + + expect(requestedModels).toEqual([ + `${primary.provider}/${primary.id}`, + `${primary.provider}/${primary.id}`, + `${target.provider}/${target.id}`, + ]); + expect(session.model?.id).toBe(target.id); + }); + + it("plan burst: injects the deep-plan nudge mid-run and scrubs it at the switch", async () => { + const primary = modelOrThrow("claude-sonnet-4-5"); + const target = modelOrThrow("claude-sonnet-4-6"); + const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir.path(), "models.yml")); + + const recordToolSchema = z.object({}); + const recordTool: AgentTool = { + name: "record", + label: "Record", + description: "Record a step", + parameters: recordToolSchema, + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + }; + const step = (id: string): MockResponse => ({ + content: [{ type: "toolCall", id, name: "record", arguments: {} }], + stopReason: "toolUse", + }); + // A plan-shaped turn: substantial text (clears the delivery threshold) + // plus a tool call so the loop keeps running. + const planText = `Plan: ${"step-by-step execution detail. ".repeat(20)}`; + const planStep = (id: string): MockResponse => ({ + content: [ + { type: "text", text: planText }, + { type: "toolCall", id, name: "record", arguments: {} }, + ], + stopReason: "toolUse", + }); + // Turn 1 explores; turn 2 answers the nudge with the plan; turn 3 keeps + // working; turn 4 runs on the target. + const mock = createMockModel({ + responses: [step("t1"), planStep("t2"), step("t3"), { content: ["done"] }], + }); + + const calls: Array<{ model: string; hasNudge: boolean }> = []; + const nudgeMarker = "complete plan in your NEXT reply"; + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model: primary, + systemPrompt: ["Test"], + tools: [recordTool as AgentTool], + messages: [], + thinkingLevel: Effort.Medium, + }, + convertToLlm, + streamFn: (model, context, options) => { + const hasNudge = context.messages.some(message => { + // The custom nudge converts to a developer-role LLM message. + if (message.role !== "user" && message.role !== "developer") return false; + const content = message.content; + if (typeof content === "string") return content.includes(nudgeMarker); + return content.some(block => block.type === "text" && block.text.includes(nudgeMarker)); + }); + calls.push({ model: `${model.provider}/${model.id}`, hasNudge }); + return mock.stream(model, context, options); + }, + }); + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry, + toolRegistry: new Map([[recordTool.name, recordTool as AgentTool]]), + reasoningSlide: { target, afterTurns: 3, plan: true, planAtTurn: 1 }, + }); + + await session.prompt("do the task"); + + const primaryId = `${primary.provider}/${primary.id}`; + expect(calls.map(call => call.model)).toEqual([ + primaryId, + primaryId, + primaryId, + `${target.provider}/${target.id}`, + ]); + // Nudge lands after turn planAtTurn=1, stays through the pre-slide turns, + // and is scrubbed from the context the target model sees. + expect(calls.map(call => call.hasNudge)).toEqual([false, true, true, false]); + expect(session.model?.id).toBe(target.id); + const residual = session.agent.state.messages.filter( + message => message.role === "custom" && message.customType === "reasoning-slide-plan", + ); + expect(residual).toHaveLength(0); + }); + + it("plan burst: holds the slide until the plan lands, then switches", async () => { + const primary = modelOrThrow("claude-sonnet-4-5"); + const target = modelOrThrow("claude-sonnet-4-6"); + const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir.path(), "models.yml")); + const recordToolSchema = z.object({}); + const recordTool: AgentTool = { + name: "record", + label: "Record", + description: "Record a step", + parameters: recordToolSchema, + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + }; + const step = (id: string): MockResponse => ({ + content: [{ type: "toolCall", id, name: "record", arguments: {} }], + stopReason: "toolUse", + }); + const planText = `Plan: ${"comprehensive step detail. ".repeat(20)}`; + // afterTurns=2, but the plan only lands on turn 4 — the slide must wait + // for it instead of switching mid-exploration, then fire right after. + const mock = createMockModel({ + responses: [ + step("t1"), + step("t2"), + step("t3"), + { + content: [ + { type: "text", text: planText }, + { type: "toolCall", id: "t4", name: "record", arguments: {} }, + ], + stopReason: "toolUse", + }, + { content: ["done"] }, + ], + }); + const requested: string[] = []; + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model: primary, + systemPrompt: ["Test"], + tools: [recordTool as AgentTool], + messages: [], + thinkingLevel: Effort.Medium, + }, + convertToLlm, + streamFn: (model, context, options) => { + requested.push(model.id); + return mock.stream(model, context, options); + }, + }); + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry, + toolRegistry: new Map([[recordTool.name, recordTool as AgentTool]]), + reasoningSlide: { target, afterTurns: 2, plan: true, planAtTurn: 1 }, + }); + + await session.prompt("do the task"); + + // Turns 1-4 stay on the primary (plan not yet delivered at the turn-2/3 + // boundaries); turn 5 runs on the target. + expect(requested).toEqual([primary.id, primary.id, primary.id, primary.id, target.id]); + expect(session.model?.id).toBe(target.id); + }); + + it("plan burst: grace ceiling slides even when no plan is ever written", async () => { + const primary = modelOrThrow("claude-sonnet-4-5"); + const target = modelOrThrow("claude-sonnet-4-6"); + const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir.path(), "models.yml")); + const recordToolSchema = z.object({}); + const recordTool: AgentTool = { + name: "record", + label: "Record", + description: "Record a step", + parameters: recordToolSchema, + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + }; + const step = (id: string): MockResponse => ({ + content: [{ type: "toolCall", id, name: "record", arguments: {} }], + stopReason: "toolUse", + }); + // afterTurns=2 + grace 4 → the hold expires at turn 6 even with no plan. + const mock = createMockModel({ + responses: [step("t1"), step("t2"), step("t3"), step("t4"), step("t5"), step("t6"), { content: ["done"] }], + }); + const requested: string[] = []; + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model: primary, + systemPrompt: ["Test"], + tools: [recordTool as AgentTool], + messages: [], + thinkingLevel: Effort.Medium, + }, + convertToLlm, + streamFn: (model, context, options) => { + requested.push(model.id); + return mock.stream(model, context, options); + }, + }); + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry, + toolRegistry: new Map([[recordTool.name, recordTool as AgentTool]]), + reasoningSlide: { target, afterTurns: 2, plan: true, planAtTurn: 1 }, + }); + + await session.prompt("do the task"); + + expect(requested).toEqual([primary.id, primary.id, primary.id, primary.id, primary.id, primary.id, target.id]); + expect(session.model?.id).toBe(target.id); + }); + + it("onFirstAction: bash stays on the primary; the first edit/write slides", async () => { + const primary = modelOrThrow("claude-sonnet-4-5"); + const target = modelOrThrow("claude-sonnet-4-6"); + const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir.path(), "models.yml")); + const readSchema = z.object({}); + const lookTool: AgentTool = { + name: "record", + label: "Record", + description: "Read-only step", + parameters: readSchema, + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + }; + const bashSchema = z.object({}); + const bashTool: AgentTool = { + name: "bash", + label: "Bash", + description: "Run a command", + parameters: bashSchema, + async execute() { + return { content: [{ type: "text", text: "ran" }], details: undefined }; + }, + }; + const writeSchema = z.object({}); + const writeTool: AgentTool = { + name: "write", + label: "Write", + description: "Write a file", + parameters: writeSchema, + async execute() { + return { content: [{ type: "text", text: "wrote" }], details: undefined }; + }, + }; + const call = (id: string, name: string): MockResponse => ({ + content: [{ type: "toolCall", id, name, arguments: {} }], + stopReason: "toolUse", + }); + // Turn 1 reads, turn 2 runs bash (exploration — must NOT trigger), + // turn 3 writes (first action) → slide; turn 4 on target. + const mock = createMockModel({ + responses: [call("t1", "record"), call("t2", "bash"), call("t3", "write"), { content: ["done"] }], + }); + const requested: string[] = []; + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model: primary, + systemPrompt: ["Test"], + tools: [lookTool as AgentTool, bashTool as AgentTool, writeTool as AgentTool], + messages: [], + thinkingLevel: Effort.Medium, + }, + convertToLlm, + streamFn: (model, context, options) => { + requested.push(model.id); + return mock.stream(model, context, options); + }, + }); + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry, + toolRegistry: new Map([ + [lookTool.name, lookTool as AgentTool], + [bashTool.name, bashTool as AgentTool], + [writeTool.name, writeTool as AgentTool], + ]), + reasoningSlide: { target, onFirstAction: true }, + }); + + await session.prompt("do the task"); + + expect(requested).toEqual([primary.id, primary.id, primary.id, target.id]); + expect(session.model?.id).toBe(target.id); + }); + + it("plan burst: a text-only plan-delivery turn does not end the run before the target model runs", async () => { + // Regression: the agent loop treats a turn with zero tool calls as a + // natural stop boundary and ends the whole session with no further + // prompting. The plan nudge explicitly asks for a prose reply, making + // this the COMMON case, not an edge case — observed killing production + // SWE-bench runs before any code was ever written. The switch must + // force a continuation so the target model actually gets a turn. + const primary = modelOrThrow("claude-sonnet-4-5"); + const target = modelOrThrow("claude-sonnet-4-6"); + const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir.path(), "models.yml")); + const recordToolSchema = z.object({}); + const recordTool: AgentTool = { + name: "record", + label: "Record", + description: "Record a step", + parameters: recordToolSchema, + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + }; + const step = (id: string): MockResponse => ({ + content: [{ type: "toolCall", id, name: "record", arguments: {} }], + stopReason: "toolUse", + }); + const planText = `Plan: ${"comprehensive step detail. ".repeat(20)}`; + // Turn 1 explores (tool call). Turn 2 answers the nudge with the plan + // as PURE TEXT — no tool call, stopReason "stop". afterTurns=2 means + // this same turn also satisfies the switch criteria: without the + // safety net, the run would end right here and the target model + // would never be invoked. + const mock = createMockModel({ + responses: [ + step("t1"), + { content: [{ type: "text", text: planText }], stopReason: "stop" }, + { content: ["done"] }, + ], + }); + const requested: string[] = []; + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model: primary, + systemPrompt: ["Test"], + tools: [recordTool as AgentTool], + messages: [], + thinkingLevel: Effort.Medium, + }, + convertToLlm, + streamFn: (model, context, options) => { + requested.push(model.id); + return mock.stream(model, context, options); + }, + }); + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry, + toolRegistry: new Map([[recordTool.name, recordTool as AgentTool]]), + reasoningSlide: { target, afterTurns: 2, plan: true, planAtTurn: 1 }, + }); + + await session.prompt("do the task"); + + // The run must survive the text-only plan turn and give the target + // model an actual turn — not stop dead after 2 primary-model calls. + expect(requested).toEqual([primary.id, primary.id, target.id]); + expect(session.model?.id).toBe(target.id); + }); + + it("checklist: steered only at the switch, present in post-switch context, independent of plan", async () => { + const primary = modelOrThrow("claude-sonnet-4-5"); + const target = modelOrThrow("claude-sonnet-4-6"); + const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir.path(), "models.yml")); + const recordToolSchema = z.object({}); + const recordTool: AgentTool = { + name: "record", + label: "Record", + description: "Record a step", + parameters: recordToolSchema, + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + }; + const step = (id: string): MockResponse => ({ + content: [{ type: "toolCall", id, name: "record", arguments: {} }], + stopReason: "toolUse", + }); + const mock = createMockModel({ + responses: [step("t1"), step("t2"), { content: ["done"] }], + }); + const checklistMarker = "grep for every other call site"; + const calls: Array<{ model: string; hasChecklist: boolean }> = []; + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model: primary, + systemPrompt: ["Test"], + tools: [recordTool as AgentTool], + messages: [], + thinkingLevel: Effort.Medium, + }, + convertToLlm, + streamFn: (model, context, options) => { + const hasChecklist = context.messages.some(message => { + if (message.role !== "user" && message.role !== "developer") return false; + const content = message.content; + if (typeof content === "string") return content.includes(checklistMarker); + return content.some(block => block.type === "text" && block.text.includes(checklistMarker)); + }); + calls.push({ model: `${model.provider}/${model.id}`, hasChecklist }); + return mock.stream(model, context, options); + }, + }); + session = new AgentSession({ + agent, + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry, + toolRegistry: new Map([[recordTool.name, recordTool as AgentTool]]), + // No `plan: true` — checklist must work standalone. + reasoningSlide: { target, afterTurns: 2, checklist: true }, + }); + + await session.prompt("do the task"); + + expect(calls.map(call => call.model)).toEqual([ + `${primary.provider}/${primary.id}`, + `${primary.provider}/${primary.id}`, + `${target.provider}/${target.id}`, + ]); + // Absent pre-switch, present only once the target model is running. + expect(calls.map(call => call.hasChecklist)).toEqual([false, false, true]); + expect(session.model?.id).toBe(target.id); + }); +}); diff --git a/packages/coding-agent/test/agent-session-retry-fallback.test.ts b/packages/coding-agent/test/agent-session-retry-fallback.test.ts index 1e5494cb4..d9713b1fa 100644 --- a/packages/coding-agent/test/agent-session-retry-fallback.test.ts +++ b/packages/coding-agent/test/agent-session-retry-fallback.test.ts @@ -1567,7 +1567,7 @@ describe("AgentSession retry fallback", () => { settings, modelRegistry, }); - const { retryStartEvents, retryEndEvents } = trackRetryEvents(session); + const { retryStartEvents } = trackRetryEvents(session); session.subscribe(event => { if (event.type === "retry_fallback_applied") { fallbackAppliedEvents.push(event); From 0856055dfed52d7aa95b076152525b01a992b839 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 03:39:05 +0200 Subject: [PATCH 57/80] feat(harbor-manager): unified benchmark normalization and reporting - Implemented a unified benchmark normalization layer to handle metrics and artifacts across harbor, edit, and snapcompact benchmarks. - Integrated the TypeScript edit benchmark directly into the manager, migrating logic and deprecating the standalone package. - Updated the API and database schema to support standardized benchmark configurations, metrics, and trace-based reporting. - Enhanced the UI to visualize comparative performance metrics, including pass rate, cost, and latency deltas for benchmark runs. --- bun.lock | 12 +- package.json | 1 - packages/harbor-manager/README.md | 48 +- .../adapters/edit/bun-imports.d.ts | 4 + packages/harbor-manager/adapters/edit/cli.ts | 97 +++ .../adapters/edit}/prompts/benchmark-retry.md | 0 .../edit}/prompts/benchmark-system.md | 0 .../adapters/edit}/prompts/benchmark-task.md | 0 .../adapters/edit}/report.ts | 0 .../adapters/edit}/runner.test.ts | 8 +- .../adapters/edit}/runner.ts | 17 +- .../adapters/edit/tsconfig.json | 4 + packages/harbor-manager/package.json | 15 +- .../src/adapters/snapcompact.py} | 39 +- .../harbor-manager/src/benchmarks.test.ts | 85 ++ packages/harbor-manager/src/benchmarks.ts | 273 +++++++ .../harbor-manager/src/experiments.test.ts | 17 +- packages/harbor-manager/src/experiments.ts | 10 +- packages/harbor-manager/src/manager.test.ts | 97 ++- packages/harbor-manager/src/runner.ts | 8 +- packages/harbor-manager/src/server.ts | 205 +++-- packages/harbor-manager/src/store.ts | 139 ++-- packages/harbor-manager/src/web/app.tsx | 154 +++- .../typescript-edit-benchmark/package.json | 11 +- .../typescript-edit-benchmark/src/index.ts | 758 ------------------ 25 files changed, 1010 insertions(+), 992 deletions(-) create mode 100644 packages/harbor-manager/adapters/edit/bun-imports.d.ts create mode 100644 packages/harbor-manager/adapters/edit/cli.ts rename packages/{typescript-edit-benchmark/src => harbor-manager/adapters/edit}/prompts/benchmark-retry.md (100%) rename packages/{typescript-edit-benchmark/src => harbor-manager/adapters/edit}/prompts/benchmark-system.md (100%) rename packages/{typescript-edit-benchmark/src => harbor-manager/adapters/edit}/prompts/benchmark-task.md (100%) rename packages/{typescript-edit-benchmark/src => harbor-manager/adapters/edit}/report.ts (100%) rename packages/{typescript-edit-benchmark/test => harbor-manager/adapters/edit}/runner.test.ts (98%) rename packages/{typescript-edit-benchmark/src => harbor-manager/adapters/edit}/runner.ts (99%) create mode 100644 packages/harbor-manager/adapters/edit/tsconfig.json rename packages/{snapcompact/research/run.py => harbor-manager/src/adapters/snapcompact.py} (93%) create mode 100644 packages/harbor-manager/src/benchmarks.test.ts create mode 100644 packages/harbor-manager/src/benchmarks.ts delete mode 100755 packages/typescript-edit-benchmark/src/index.ts diff --git a/bun.lock b/bun.lock index 80270a3e9..b9ec794d5 100644 --- a/bun.lock +++ b/bun.lock @@ -140,12 +140,19 @@ "name": "@oh-my-pi/harbor-manager", "version": "0.0.1", "bin": { - "harbor-manager": "src/runner.ts", + "harbor-manager": "src/server.ts", }, "dependencies": { + "@oh-my-pi/hashline": "catalog:", + "@oh-my-pi/pi-agent-core": "catalog:", + "@oh-my-pi/pi-ai": "catalog:", + "@oh-my-pi/pi-coding-agent": "catalog:", + "@oh-my-pi/pi-utils": "catalog:", + "@oh-my-pi/typescript-edit-benchmark": "workspace:*", "clsx": "^2.1.1", "d3-scale": "^4.0.2", "d3-shape": "^3.2.0", + "diff": "catalog:", "motion": "^12.15.0", "react": "^19.1.0", "react-dom": "^19.1.0", @@ -276,9 +283,6 @@ "packages/typescript-edit-benchmark": { "name": "@oh-my-pi/typescript-edit-benchmark", "version": "0.0.1", - "bin": { - "typescript-edit-benchmark": "src/index.ts", - }, "dependencies": { "@babel/generator": "catalog:", "@babel/parser": "catalog:", diff --git a/package.json b/package.json index 5865a13bc..91b40ea2a 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,6 @@ "ci:release:publish": "bun scripts/ci-release-publish.ts", "ci:release:publish-native-leaf": "bun scripts/ci-release-publish.ts --native-leaf", "bench:gen-fixtures": "bun --cwd=packages/typescript-edit-benchmark run src/generate.ts --typescript-dir /tmp/typescript-source --count-per-type 8", - "bench:edit": "bun --cwd=packages/typescript-edit-benchmark run start", "stats:sync": "python3 scripts/session-stats/sync.py", "stats:tools": "python3 scripts/session-stats/analyze.py tools", "stats:edits": "python3 scripts/session-stats/analyze.py edits", diff --git a/packages/harbor-manager/README.md b/packages/harbor-manager/README.md index 107a06389..f289964ea 100644 --- a/packages/harbor-manager/README.md +++ b/packages/harbor-manager/README.md @@ -1,22 +1,16 @@ # @oh-my-pi/harbor-manager -Manage [Harbor](https://github.com/laude-institute/harbor) benchmark runs against -the **local `omp` build**: a CLI runner with a live terminal dashboard, a -SQLite-backed run store, a REST/SSE API, and a web dashboard with live -run → trial → transcript drill-down. - -Works with any Harbor dataset (`terminal-bench@2.0` by default, -`swe-bench/swe-bench-verified`, …). +One manager for repository benchmarks. Harbor, TypeScript edit, and SnapCompact +runs use the same experiment → run → trace model, SQLite store, REST/SSE API, +and dashboard. Benchmark-native artifacts remain on disk; adapters normalize +their live progress, scores, token usage, costs, and traces. ```bash -# CLI run (owns the terminal, writes a markdown report) -bun src/runner.ts --model anthropic/claude-sonnet-4-6 --tasks 20 --concurrency 4 - -# Web manager: dashboard + API on :4700 -bun run serve # bun src/server.ts [--port 4700] [--jobs-dir ] +# Dashboard + API on :4700; launch every benchmark from the same “new run” form +bun run serve --port 4700 ``` -## How runs execute +## How Harbor runs execute 1. **Local omp, not npm.** By default the runner bind-mounts the repo read-only into each task container (`--install source`) and runs omp @@ -35,34 +29,36 @@ bun run serve # bun src/server.ts [--port 4700] [--jobs-dir ] ## Server -- `GET /` — dashboard: run list (SSE-live), trial grid, transcript viewer - that tails live sessions. -- `GET /api/runs` — run rows (rollups: pass/fail/error/spend/tokens). -- `POST /api/runs` — launch. Body: +- `GET /` — experiments, runs, normalized traces, and a launch form for every benchmark. +- `GET /api/experiments` — experiment summaries across all benchmark types. +- `GET /api/runs` — uniform run rows with benchmark, score, progress, spend, and tokens. +- `POST /api/runs` — launch through a benchmark adapter. Body: ```json { + "benchmark": "edit", "model": "anthropic/claude-opus-4-8", - "dataset": "swe-bench/swe-bench-verified", "tasks": 20, "concurrency": 4, - "timeoutMultiplier": 2, - "include": ["swe-bench/astropy__astropy-14995"], - "slide": { "model": "google/gemini-3.5-flash", "onAction": true, "plan": true } + "attempts": 2, + "jobName": "edit-baseline", + "role": "baseline", + "goal": "compare edit strategies" } ``` - `slide.turns` and `slide.onAction` are mutually exclusive triggers. -- `GET /api/runs/:name` — `{ run, trials }` (syncs from disk on read). + `benchmark` is `harbor`, `edit`, or `snapcompact`. Harbor uses `dataset`, + `include`, `timeoutMultiplier`, and `slide`; edit uses `include` as task IDs; + SnapCompact uses `conditions` and treats `tasks` as the passage limit. +- `GET /api/runs/:name` — `{ run, traces }` (syncs native artifacts on read). - `DELETE /api/runs/:name` — cancel a manager-launched run. -- `GET /api/runs/:name/trials/:trial/transcript?tail=N[&raw=1]` — compact - (or raw JSONL) view of the trial's live session log. +- `GET /api/runs/:name/traces/:trace[?raw=1]` — normalized or native trace. - `GET /api/events` — SSE stream of run-list snapshots (sent on change). State lives in `/_manager/harbor-manager.sqlite`; the filesystem stays the source of truth and historical CLI runs are auto-discovered. -## Runner options (excerpt) +## Harbor runner options (excerpt) | Option | Default | Notes | |---|---|---| diff --git a/packages/harbor-manager/adapters/edit/bun-imports.d.ts b/packages/harbor-manager/adapters/edit/bun-imports.d.ts new file mode 100644 index 000000000..7c90c3d17 --- /dev/null +++ b/packages/harbor-manager/adapters/edit/bun-imports.d.ts @@ -0,0 +1,4 @@ +declare module "*.tar.gz" { + const content: string; + export default content; +} diff --git a/packages/harbor-manager/adapters/edit/cli.ts b/packages/harbor-manager/adapters/edit/cli.ts new file mode 100644 index 000000000..dbbe8ce04 --- /dev/null +++ b/packages/harbor-manager/adapters/edit/cli.ts @@ -0,0 +1,97 @@ +#!/usr/bin/env bun +/** Manager-owned executable adapter for the TypeScript edit benchmark. */ +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { parseArgs } from "node:util"; +import { TempDir } from "@oh-my-pi/pi-utils"; +import { loadTasksFromDir } from "@oh-my-pi/typescript-edit-benchmark/tasks"; +import { generateJsonReport } from "./report"; +import { type BenchmarkConfig, runBenchmark } from "./runner"; + +const EDIT_PACKAGE = path.resolve(import.meta.dir, "..", "..", "..", "typescript-edit-benchmark"); + +async function extractFixtures(): Promise<{ dir: string; temp: TempDir }> { + const temp = await TempDir.create("@harbor-edit-fixtures-"); + const archive = new Bun.Archive(await Bun.file(path.join(EDIT_PACKAGE, "fixtures.tar.gz")).arrayBuffer()); + for (const [filePath, file] of await archive.files()) { + await Bun.write(path.join(temp.path(), filePath), file); + } + const entries = await fs.readdir(temp.path(), { withFileTypes: true }); + const directories = entries.filter(entry => entry.isDirectory()); + const files = entries.filter(entry => entry.isFile()); + return { dir: directories.length === 1 && files.length === 0 ? path.join(temp.path(), directories[0]!.name) : temp.path(), temp }; +} + +/** Execute an edit benchmark and continuously materialize its normalized source artifact. */ +export async function main(argv = process.argv.slice(2)): Promise { + const { values } = parseArgs({ + args: argv, + options: { + model: { type: "string" }, + output: { type: "string" }, + "max-tasks": { type: "string", default: "80" }, + tasks: { type: "string" }, + "task-concurrency": { type: "string", default: "32" }, + runs: { type: "string", default: "1" }, + list: { type: "boolean", default: false }, + }, + strict: true, + }); + const fixtures = await extractFixtures(); + try { + let tasks = await loadTasksFromDir(fixtures.dir); + if (values.list) { + process.stdout.write(`${JSON.stringify(tasks.map(task => ({ id: task.id, name: task.name })))}\n`); + return; + } + if (!values.model || !values.output) throw new Error("edit adapter requires --model and --output"); + if (values.tasks) { + const selected = new Set(values.tasks.split(",").map(value => value.trim())); + tasks = tasks.filter(task => selected.has(task.id)); + if (tasks.length !== selected.size) throw new Error("one or more edit task ids were not found"); + } else { + const limit = Number(values["max-tasks"]); + if (limit > 0 && tasks.length > limit) { + const sorted = tasks.slice().sort((a, b) => a.id.localeCompare(b.id)); + const step = sorted.length / limit; + tasks = Array.from({ length: limit }, (_, index) => sorted[Math.floor(index * step)]!); + } + } + const slash = values.model.indexOf("/"); + const config: BenchmarkConfig = { + provider: slash === -1 ? "anthropic" : values.model.slice(0, slash), + model: values.model, + runsPerTask: Number(values.runs), + timeout: 120_000, + connectionTimeout: 30_000, + maxTurns: 30, + taskConcurrency: Number(values["task-concurrency"]), + guided: false, + maxAttempts: 1, + noOpRetryLimit: 2, + maxTimeoutRetries: 3, + maxProviderFailureRetries: 3, + mutationScopeWindow: 20, + conversationDumpDir: path.join(path.dirname(values.output), "result.dump"), + inProcess: true, + earlyStopOnMatch: true, + }; + let writes = Promise.resolve(); + const result = await runBenchmark(tasks, config, undefined, snapshot => { + writes = writes.then(async () => { + await Bun.write(values.output!, generateJsonReport(snapshot)); + }); + }); + await writes; + await Bun.write(values.output, generateJsonReport(result)); + } finally { + await fixtures.temp.remove(); + } +} + +if (import.meta.main) { + main().catch(error => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/packages/typescript-edit-benchmark/src/prompts/benchmark-retry.md b/packages/harbor-manager/adapters/edit/prompts/benchmark-retry.md similarity index 100% rename from packages/typescript-edit-benchmark/src/prompts/benchmark-retry.md rename to packages/harbor-manager/adapters/edit/prompts/benchmark-retry.md diff --git a/packages/typescript-edit-benchmark/src/prompts/benchmark-system.md b/packages/harbor-manager/adapters/edit/prompts/benchmark-system.md similarity index 100% rename from packages/typescript-edit-benchmark/src/prompts/benchmark-system.md rename to packages/harbor-manager/adapters/edit/prompts/benchmark-system.md diff --git a/packages/typescript-edit-benchmark/src/prompts/benchmark-task.md b/packages/harbor-manager/adapters/edit/prompts/benchmark-task.md similarity index 100% rename from packages/typescript-edit-benchmark/src/prompts/benchmark-task.md rename to packages/harbor-manager/adapters/edit/prompts/benchmark-task.md diff --git a/packages/typescript-edit-benchmark/src/report.ts b/packages/harbor-manager/adapters/edit/report.ts similarity index 100% rename from packages/typescript-edit-benchmark/src/report.ts rename to packages/harbor-manager/adapters/edit/report.ts diff --git a/packages/typescript-edit-benchmark/test/runner.test.ts b/packages/harbor-manager/adapters/edit/runner.test.ts similarity index 98% rename from packages/typescript-edit-benchmark/test/runner.test.ts rename to packages/harbor-manager/adapters/edit/runner.test.ts index 2ff9434ad..b30cccd84 100644 --- a/packages/typescript-edit-benchmark/test/runner.test.ts +++ b/packages/harbor-manager/adapters/edit/runner.test.ts @@ -4,12 +4,8 @@ import * as path from "node:path"; import type { AgentMessage } from "@oh-my-pi/pi-agent-core"; import { formatSessionDumpText, SessionManager } from "@oh-my-pi/pi-coding-agent"; import { TempDir } from "@oh-my-pi/pi-utils"; -import { generateReport } from "@oh-my-pi/typescript-edit-benchmark/report"; -import { - buildBenchmarkResult, - type TaskRunResult, - writeConversationDump, -} from "@oh-my-pi/typescript-edit-benchmark/runner"; +import { generateReport } from "./report"; +import { buildBenchmarkResult, type TaskRunResult, writeConversationDump } from "./runner"; import type { EditTask } from "@oh-my-pi/typescript-edit-benchmark/tasks"; const tempDirs: TempDir[] = []; diff --git a/packages/typescript-edit-benchmark/src/runner.ts b/packages/harbor-manager/adapters/edit/runner.ts similarity index 99% rename from packages/typescript-edit-benchmark/src/runner.ts rename to packages/harbor-manager/adapters/edit/runner.ts index e401c5c97..47568e6af 100644 --- a/packages/typescript-edit-benchmark/src/runner.ts +++ b/packages/harbor-manager/adapters/edit/runner.ts @@ -13,15 +13,22 @@ import type { Model, ToolExample } from "@oh-my-pi/pi-ai"; import { formatSessionDumpText, RpcClient } from "@oh-my-pi/pi-coding-agent"; import { prompt } from "@oh-my-pi/pi-utils"; import { diffLines } from "diff"; -import { formatDirectory } from "./formatter"; -import { discoverSharedInfra, InProcessClient, type SharedInfra } from "./in-process-client"; +import { formatDirectory } from "@oh-my-pi/typescript-edit-benchmark/formatter"; +import { + discoverSharedInfra, + InProcessClient, + type SharedInfra, +} from "@oh-my-pi/typescript-edit-benchmark/in-process-client"; import benchmarkRetryPrompt from "./prompts/benchmark-retry.md" with { type: "text" }; import benchmarkSystemPrompt from "./prompts/benchmark-system.md" with { type: "text" }; import benchmarkTaskPrompt from "./prompts/benchmark-task.md" with { type: "text" }; -import type { EditTask } from "./tasks"; -import { verifyExpectedFileSubset, verifyExpectedFiles } from "./verify"; +import type { EditTask } from "@oh-my-pi/typescript-edit-benchmark/tasks"; +import { + verifyExpectedFileSubset, + verifyExpectedFiles, +} from "@oh-my-pi/typescript-edit-benchmark/verify"; -const REPO_ROOT = path.resolve(import.meta.dir, "..", "..", ".."); +const REPO_ROOT = path.resolve(import.meta.dir, "..", "..", "..", ".."); const RUNS_DIR = path.join(REPO_ROOT, "runs"); const TMP = path.join(RUNS_DIR, `rb-${Math.random().toString(36).slice(2, 10)}`); const CLI_PATH = Bun.fileURLToPath(import.meta.resolve("@oh-my-pi/pi-coding-agent/cli")); diff --git a/packages/harbor-manager/adapters/edit/tsconfig.json b/packages/harbor-manager/adapters/edit/tsconfig.json new file mode 100644 index 000000000..936fe3f3c --- /dev/null +++ b/packages/harbor-manager/adapters/edit/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../../tsconfig.workspace.json", + "include": ["."] +} diff --git a/packages/harbor-manager/package.json b/packages/harbor-manager/package.json index 42146e3f4..c54a1be65 100644 --- a/packages/harbor-manager/package.json +++ b/packages/harbor-manager/package.json @@ -3,7 +3,7 @@ "private": true, "name": "@oh-my-pi/harbor-manager", "version": "0.0.1", - "description": "Manage Harbor benchmark runs against the local omp build: CLI runner, SQLite-backed run store, REST/SSE API, and a live web dashboard", + "description": "Unified benchmark runners plus Harbor run storage, REST/SSE APIs, and a live web dashboard", "homepage": "https://omp.sh", "author": "Can Boluk", "license": "MIT", @@ -13,20 +13,27 @@ "directory": "packages/harbor-manager" }, "bin": { - "harbor-manager": "src/runner.ts" + "harbor-manager": "src/server.ts" }, "scripts": { "check": "biome check . && bun run check:types", - "check:types": "tsgo -p tsconfig.json --noEmit", + "check:types": "tsgo -p tsconfig.json --noEmit && tsgo -p adapters/edit/tsconfig.json --noEmit", "lint": "biome lint .", - "start": "bun run src/runner.ts", + "start": "bun run src/server.ts", "serve": "bun run src/server.ts", "test": "bun test" }, "dependencies": { + "@oh-my-pi/hashline": "catalog:", + "@oh-my-pi/pi-agent-core": "catalog:", + "@oh-my-pi/pi-ai": "catalog:", + "@oh-my-pi/pi-coding-agent": "catalog:", + "@oh-my-pi/pi-utils": "catalog:", + "@oh-my-pi/typescript-edit-benchmark": "workspace:*", "clsx": "^2.1.1", "d3-scale": "^4.0.2", "d3-shape": "^3.2.0", + "diff": "catalog:", "motion": "^12.15.0", "react": "^19.1.0", "react-dom": "^19.1.0", diff --git a/packages/snapcompact/research/run.py b/packages/harbor-manager/src/adapters/snapcompact.py similarity index 93% rename from packages/snapcompact/research/run.py rename to packages/harbor-manager/src/adapters/snapcompact.py index efc570d75..b45daf69b 100644 --- a/packages/snapcompact/research/run.py +++ b/packages/harbor-manager/src/adapters/snapcompact.py @@ -36,6 +36,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path HERE = Path(__file__).resolve().parent +RESEARCH = HERE.parents[2] / "snapcompact" / "research" def find_agent_prompts() -> Path: for parent in HERE.parents: @@ -48,16 +49,16 @@ def find_agent_prompts() -> Path: raise FileNotFoundError("Could not find agent compaction prompts") -sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(RESEARCH)) import squad # noqa: E402 from anthropic_api import complete, image_block, load_api_key # noqa: E402 from bdf import VARIANTS, FontCfg, capacity, render # noqa: E402 AGENT_PROMPTS = find_agent_prompts() -CACHE = HERE / ".cache" +CACHE = RESEARCH / ".cache" QA_CACHE = CACHE / "qa" -RESULTS = HERE / "results" +RESULTS = RESEARCH / "results" FONTS = { "8x13": FontCfg("8x13", "8x13", 8, 13), @@ -100,7 +101,7 @@ def sha8(*parts: str) -> str: def load_prompt(name: str) -> str: - return (HERE / "prompts" / name).read_text() + return (RESEARCH / "prompts" / name).read_text() def agent_prompt(name: str) -> str: @@ -276,6 +277,7 @@ def main() -> None: ap.add_argument("--price-in", type=float, default=10.0, help="$ per 1M input tokens") ap.add_argument("--price-out", type=float, default=50.0, help="$ per 1M output tokens") ap.add_argument("--env", default="~/.env") + ap.add_argument("--output-dir", help="write records.jsonl and summary.json directly to this directory") args = ap.parse_args() CACHE.mkdir(exist_ok=True) @@ -288,7 +290,11 @@ def main() -> None: f"-e{args.effort}" if args.effort else "", ] ) - run_dir = RESULTS / f"{args.model}-seed{args.seed}-qpc{args.qpc}-{scope}{tag}" + run_dir = ( + Path(args.output_dir).expanduser().resolve() + if args.output_dir + else RESULTS / f"{args.model}-seed{args.seed}-qpc{args.qpc}-{scope}{tag}" + ) run_dir.mkdir(parents=True, exist_ok=True) paras = squad.load_paragraphs(CACHE) @@ -312,17 +318,18 @@ def main() -> None: ctx_args = {"args": args, "flow": flow, "paras": paras, "offsets": offsets, "api_key": api_key} records: list[dict] = [] done = 0 - with ThreadPoolExecutor(args.workers) as pool: - futures = [pool.submit(run_chunk, cond, start, end, ctx_args) for cond, start, end in tasks] - for fut in futures: - records.extend(fut.result()) - done += 1 - if done % 20 == 0: - print(f" {done}/{len(tasks)} chunks", flush=True) - - with (run_dir / "records.jsonl").open("w") as fh: - for r in records: - fh.write(json.dumps(r) + "\n") + with (run_dir / "records.jsonl").open("w") as records_file: + with ThreadPoolExecutor(args.workers) as pool: + futures = [pool.submit(run_chunk, cond, start, end, ctx_args) for cond, start, end in tasks] + for fut in futures: + chunk_records = fut.result() + records.extend(chunk_records) + for record in chunk_records: + records_file.write(json.dumps(record) + "\n") + records_file.flush() + done += 1 + if done % 20 == 0: + print(f" {done}/{len(tasks)} chunks", flush=True) rows = [ aggregate(cond["name"], [r for r in records if r["cond"] == cond["name"]], args.price_in, args.price_out) diff --git a/packages/harbor-manager/src/benchmarks.test.ts b/packages/harbor-manager/src/benchmarks.test.ts new file mode 100644 index 000000000..d7a00018a --- /dev/null +++ b/packages/harbor-manager/src/benchmarks.test.ts @@ -0,0 +1,85 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { BENCHMARK_DEFINITIONS, readBenchmarkSnapshot } from "./benchmarks"; + +const cleanups: string[] = []; + +function jobDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "harbor-benchmark-")); + cleanups.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of cleanups.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe("benchmark adapters", () => { + it("normalizes edit attempts, traces, tokens, and declared metrics", () => { + const dir = jobDir(); + fs.writeFileSync( + path.join(dir, "result.json"), + JSON.stringify({ + tasks: [ + { + id: "rename-symbol", + name: "Rename symbol", + runs: [ + { + runIndex: 0, + success: true, + duration: 1200, + tokens: { input: 100, output: 20, reasoning: 5 }, + }, + ], + }, + ], + summary: { + totalRuns: 1, + successfulRuns: 1, + taskSuccessRate: 1, + editSuccessRate: 0.75, + totalTokens: { input: 100, output: 20 }, + }, + }), + ); + + const snapshot = readBenchmarkSnapshot("edit", dir); + expect(snapshot.metrics).toEqual({ task_success_rate: 1, edit_success_rate: 0.75 }); + expect(snapshot.traces[0]).toMatchObject({ + name: "rename-symbol__1", + status: "pass", + tracePath: path.join("result.dump", "rename-symbol", "run-1.md"), + }); + expect([snapshot.tokIn, snapshot.tokOut]).toEqual([100, 20]); + }); + + it("normalizes SnapCompact records and weighted quality metrics", () => { + const dir = jobDir(); + fs.writeFileSync( + path.join(dir, "records.jsonl"), + `${JSON.stringify({ cond: "text", chunk: 0, pos_rel: 0.2, q: "q", answer: "a", golds: ["a"], em: 1, f1: 1 })}\n`, + ); + fs.writeFileSync( + path.join(dir, "summary.json"), + JSON.stringify({ + rows: [ + { n: 2, f1: 0.75, em: 0.5, cost_usd: 1.25, tokens_in: 200, tokens_out: 30, cache_w: 10, cache_r: 20 }, + ], + }), + ); + + const snapshot = readBenchmarkSnapshot("snapcompact", dir); + expect(snapshot.metrics).toEqual({ f1: 0.75, exact_match: 0.5 }); + expect(snapshot.traces[0]).toMatchObject({ status: "pass", reward: 1, tracePath: "record:1" }); + expect(snapshot.costUsd).toBe(1.25); + expect(snapshot.tokCache).toBe(30); + }); + + it("publishes metric definitions for every managed benchmark", () => { + expect(BENCHMARK_DEFINITIONS.map(definition => definition.kind)).toEqual(["harbor", "edit", "snapcompact"]); + expect(BENCHMARK_DEFINITIONS.every(definition => definition.metrics.length > 0)).toBe(true); + }); +}); diff --git a/packages/harbor-manager/src/benchmarks.ts b/packages/harbor-manager/src/benchmarks.ts new file mode 100644 index 000000000..93523ddfa --- /dev/null +++ b/packages/harbor-manager/src/benchmarks.ts @@ -0,0 +1,273 @@ +/** Benchmark adapters normalize native artifacts into manager runs and traces. */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { aggregate, readJobResult, readTrials } from "./runner"; +import type { BenchmarkKind } from "./store"; + +/** Describes a benchmark metric so storage and UI do not hard-code benchmark semantics. */ +export interface MetricDefinition { + key: string; + label: string; + format: "percent" | "number" | "usd"; + higherIsBetter: boolean; +} + +/** Adapter metadata exposed to launch clients and the dashboard. */ +export interface BenchmarkDefinition { + kind: BenchmarkKind; + label: string; + metrics: MetricDefinition[]; +} + +/** Built-in benchmark adapters and their native score definitions. */ +export const BENCHMARK_DEFINITIONS: BenchmarkDefinition[] = [ + { + kind: "harbor", + label: "Harbor", + metrics: [{ key: "success_rate", label: "Success rate", format: "percent", higherIsBetter: true }], + }, + { + kind: "edit", + label: "TypeScript edit", + metrics: [ + { key: "task_success_rate", label: "Task success", format: "percent", higherIsBetter: true }, + { key: "edit_success_rate", label: "Edit success", format: "percent", higherIsBetter: true }, + ], + }, + { + kind: "snapcompact", + label: "SnapCompact", + metrics: [ + { key: "f1", label: "F1", format: "percent", higherIsBetter: true }, + { key: "exact_match", label: "Exact match", format: "percent", higherIsBetter: true }, + ], + }, +]; + +/** A normalized trace emitted by any benchmark adapter. */ +export interface BenchmarkTrace { + name: string; + task: string; + status: "pass" | "fail" | "error" | "running"; + reward: number | null; + costUsd: number; + durationMs: number; + detail: string; + tracePath: string | null; +} + +/** Uniform aggregate and traces read from benchmark-native artifacts. */ +export interface BenchmarkSnapshot { + traces: BenchmarkTrace[]; + total: number; + done: number; + pass: number; + fail: number; + error: number; + running: number; + costUsd: number; + tokIn: number; + tokOut: number; + tokCache: number; + score: number | null; + metrics: Record; +} + +interface EditRun { + runIndex: number; + success: boolean; + error?: string; + duration: number; + tokens: { input: number; output: number; reasoning: number }; + toolCalls?: { read: number; edit: number; write: number }; +} + +interface EditTask { + id: string; + name: string; + runs: EditRun[]; +} + +interface EditResult { + tasks: EditTask[]; + summary: { + totalRuns: number; + successfulRuns: number; + taskSuccessRate: number; + editSuccessRate: number; + totalTokens: { input: number; output: number }; + }; +} + +interface SnapRecord { + cond: string; + chunk: number; + pos_rel: number; + q: string; + answer: string; + golds: string[]; + em: number; + f1: number; +} + +interface SnapSummaryRow { + n: number; + f1: number; + em: number; + cost_usd: number; + tokens_in: number; + tokens_out: number; + cache_w: number; + cache_r: number; +} + +interface SnapSummary { + rows: SnapSummaryRow[]; +} + +function emptySnapshot(): BenchmarkSnapshot { + return { + traces: [], + total: 0, + done: 0, + pass: 0, + fail: 0, + error: 0, + running: 0, + costUsd: 0, + tokIn: 0, + tokOut: 0, + tokCache: 0, + score: null, + metrics: {}, + }; +} + +function readEditSnapshot(jobDir: string): BenchmarkSnapshot { + const file = path.join(jobDir, "result.json"); + if (!fs.existsSync(file)) return emptySnapshot(); + const result: EditResult = JSON.parse(fs.readFileSync(file, "utf8")); + const traces: BenchmarkTrace[] = []; + let tokIn = 0; + let tokOut = 0; + for (const task of result.tasks) { + for (const run of task.runs) { + tokIn += run.tokens.input; + tokOut += run.tokens.output; + const runNumber = run.runIndex + 1; + traces.push({ + name: `${task.id}__${runNumber}`, + task: task.id, + status: run.success ? "pass" : run.error ? "error" : "fail", + reward: run.success ? 1 : 0, + costUsd: 0, + durationMs: run.duration, + detail: JSON.stringify({ name: task.name, error: run.error ?? null, tools: run.toolCalls ?? null }), + tracePath: path.join("result.dump", task.id.replace(/[^a-zA-Z0-9._-]/g, "_"), `run-${runNumber}.md`), + }); + } + } + const pass = traces.filter(trace => trace.status === "pass").length; + const error = traces.filter(trace => trace.status === "error").length; + return { + traces, + total: result.summary.totalRuns, + done: traces.length, + pass, + fail: traces.length - pass, + error, + running: Math.max(0, result.summary.totalRuns - traces.length), + costUsd: 0, + tokIn, + tokOut, + tokCache: 0, + score: result.summary.taskSuccessRate, + metrics: { + task_success_rate: result.summary.taskSuccessRate, + edit_success_rate: result.summary.editSuccessRate, + }, + }; +} + +function readSnapcompactSnapshot(jobDir: string): BenchmarkSnapshot { + const recordsFile = path.join(jobDir, "records.jsonl"); + const summaryFile = path.join(jobDir, "summary.json"); + if (!fs.existsSync(recordsFile)) return emptySnapshot(); + const records = fs + .readFileSync(recordsFile, "utf8") + .split("\n") + .filter(Boolean) + .map(line => JSON.parse(line) as SnapRecord); + const traces = records.map( + (record, index): BenchmarkTrace => ({ + name: `${record.cond}__${record.chunk}__${index + 1}`, + task: `${record.cond}:${record.chunk}`, + status: record.f1 > 0 ? "pass" : "fail", + reward: record.f1, + costUsd: 0, + durationMs: 0, + detail: JSON.stringify({ question: record.q, answer: record.answer, golds: record.golds, em: record.em }), + tracePath: `record:${index + 1}`, + }), + ); + let rows: SnapSummaryRow[] = []; + if (fs.existsSync(summaryFile)) { + const summary: SnapSummary = JSON.parse(fs.readFileSync(summaryFile, "utf8")); + rows = summary.rows; + } + const samples = rows.reduce((sum, row) => sum + row.n, 0); + const weightedF1 = rows.reduce((sum, row) => sum + row.f1 * row.n, 0); + const weightedEm = rows.reduce((sum, row) => sum + row.em * row.n, 0); + const pass = traces.filter(trace => trace.status === "pass").length; + return { + traces, + total: traces.length, + done: traces.length, + pass, + fail: traces.length - pass, + error: 0, + running: 0, + costUsd: rows.reduce((sum, row) => sum + row.cost_usd, 0), + tokIn: rows.reduce((sum, row) => sum + row.tokens_in, 0), + tokOut: rows.reduce((sum, row) => sum + row.tokens_out, 0), + tokCache: rows.reduce((sum, row) => sum + row.cache_w + row.cache_r, 0), + score: samples > 0 ? weightedF1 / samples : null, + metrics: { + f1: samples > 0 ? weightedF1 / samples : null, + exact_match: samples > 0 ? weightedEm / samples : null, + }, + }; +} + +/** Read and normalize the latest artifacts for a benchmark run. */ +export function readBenchmarkSnapshot(benchmark: BenchmarkKind, jobDir: string): BenchmarkSnapshot { + if (benchmark === "edit") return readEditSnapshot(jobDir); + if (benchmark === "snapcompact") return readSnapcompactSnapshot(jobDir); + const trials = readTrials(jobDir); + const job = readJobResult(jobDir); + const totals = aggregate(trials, job, job?.nTotal ?? trials.length); + return { + traces: trials.map(trial => ({ + name: trial.name, + task: trial.name.replace(/__[^_]+$/, ""), + status: trial.status, + reward: trial.reward, + costUsd: trial.costUsd, + durationMs: trial.durationMs, + detail: trial.detail, + tracePath: path.join(trial.name, "agent", "omp.txt"), + })), + total: totals.total, + done: totals.done, + pass: totals.pass, + fail: totals.fail, + error: totals.error, + running: totals.running, + costUsd: totals.costUsd, + tokIn: totals.tokIn, + tokOut: totals.tokOut, + tokCache: totals.tokCache, + score: totals.done > 0 ? totals.pass / totals.done : null, + metrics: { success_rate: totals.done > 0 ? totals.pass / totals.done : null }, + }; +} diff --git a/packages/harbor-manager/src/experiments.test.ts b/packages/harbor-manager/src/experiments.test.ts index d659168ca..454df656a 100644 --- a/packages/harbor-manager/src/experiments.test.ts +++ b/packages/harbor-manager/src/experiments.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "bun:test"; import { armOf, experimentOf, summarizeArm } from "./experiments"; -import type { RunRow, TrialRow } from "./store"; +import type { RunRow, TraceRow } from "./store"; /** * Contracts under test: @@ -11,11 +11,13 @@ import type { RunRow, TrialRow } from "./store"; function runRow(overrides: Partial): RunRow { return { + benchmark: "harbor", jobName: "exp-arm", dataset: "d", agent: "omp", models: "anthropic/claude-opus-4-8", slide: null, + config: {}, role: "", note: "", status: "running", @@ -33,11 +35,13 @@ function runRow(overrides: Partial): RunRow { tokIn: 0, tokOut: 0, tokCache: 0, + score: null, + metrics: {}, ...overrides, }; } -function trialRow(overrides: Partial): TrialRow { +function traceRow(overrides: Partial): TraceRow { return { jobName: "exp-arm", name: "task__x", @@ -48,6 +52,7 @@ function trialRow(overrides: Partial): TrialRow { durationMs: 60_000, detail: "", updatedAt: Date.now(), + tracePath: null, ...overrides, }; } @@ -75,8 +80,8 @@ describe("summarizeArm", () => { costUsd: 5, }), [ - trialRow({ status: "pass", durationMs: 120_000 }), - trialRow({ name: "b__x", task: "b", status: "fail", reward: 0, durationMs: 240_000 }), + traceRow({ status: "pass", durationMs: 120_000 }), + traceRow({ name: "b__x", task: "b", status: "fail", reward: 0, durationMs: 240_000 }), ], ); expect(running.arm).toBe("n8"); @@ -94,7 +99,7 @@ describe("summarizeArm", () => { const finished = summarizeArm( runRow({ jobName: "sb2-opus", status: "complete", nTotal: 20, done: 20, pass: 15, costUsd: 30 }), - [trialRow({})], + [traceRow({})], ); expect(finished.projected).toBeNull(); expect(finished.costPerTask).toBeCloseTo(1.5, 5); @@ -108,6 +113,6 @@ describe("summarizeArm", () => { }), [], ); - expect(arm.config).toBe("anthropic/claude-opus-4-8 → google/gemini-3.5-flash on first edit/write +plan"); + expect(arm.config).toBe("harbor · anthropic/claude-opus-4-8 → google/gemini-3.5-flash on first edit/write +plan"); }); }); diff --git a/packages/harbor-manager/src/experiments.ts b/packages/harbor-manager/src/experiments.ts index 277128d54..829d0a021 100644 --- a/packages/harbor-manager/src/experiments.ts +++ b/packages/harbor-manager/src/experiments.ts @@ -3,7 +3,7 @@ * `sb2-gemini` → experiment `sb2`) so comparable arms can be charted together, * with linear projections for arms still in flight. */ -import type { RunRow, RunStore, TrialRow } from "./store"; +import type { RunRow, RunStore, TraceRow } from "./store"; /** Linear extrapolation of a running arm to its full task count. */ export interface ArmProjection { @@ -79,8 +79,8 @@ function slideLabel(slideJson: string | null): string { } } -export function summarizeArm(run: RunRow, trials: TrialRow[]): ArmSummary { - const decided = trials.filter(t => t.status === "pass" || t.status === "fail" || t.status === "error"); +export function summarizeArm(run: RunRow, traces: TraceRow[]): ArmSummary { + const decided = traces.filter(t => t.status === "pass" || t.status === "fail" || t.status === "error"); const durations = decided.filter(t => t.durationMs > 0).map(t => t.durationMs); const meanTrialMs = durations.length > 0 ? durations.reduce((a, b) => a + b, 0) / durations.length : null; const passPct = decided.length > 0 ? (100 * run.pass) / decided.length : null; @@ -102,7 +102,7 @@ export function summarizeArm(run: RunRow, trials: TrialRow[]): ArmSummary { return { run, arm: armOf(run.jobName), - config: `${run.models}${slideLabel(run.slide)}`, + config: `${run.benchmark} · ${run.models}${slideLabel(run.slide)}`, passPct, costPerTask, meanTrialMs, @@ -150,7 +150,7 @@ export function experimentDetail(store: RunStore, id: string): ExperimentDetail const matrix: ExperimentDetail["matrix"] = {}; const tasks = new Set(); for (const run of runs) { - const trials = store.listTrials(run.jobName); + const trials = store.listTraces(run.jobName); arms.push(summarizeArm(run, trials)); const cells: Record = {}; for (const t of trials) { diff --git a/packages/harbor-manager/src/manager.test.ts b/packages/harbor-manager/src/manager.test.ts index 26974c862..3d26fb274 100644 --- a/packages/harbor-manager/src/manager.test.ts +++ b/packages/harbor-manager/src/manager.test.ts @@ -104,13 +104,13 @@ describe("RunStore", () => { expect(run?.running).toBe(1); expect(run?.costUsd).toBeCloseTo(0.7, 5); - const trials = store.listTrials("job-a"); - expect(trials.map(t => [t.task, t.status])).toEqual([ + const traces = store.listTraces("job-a"); + expect(traces.map(t => [t.task, t.status])).toEqual([ ["alpha", "pass"], ["beta", "error"], ["gamma", "running"], ]); - expect(trials[1].detail).toBe("AgentTimeoutError"); + expect(traces[1].detail).toBe("AgentTimeoutError"); // re-discover is idempotent expect(store.discover()).toBe(0); @@ -162,6 +162,7 @@ describe("RunStore", () => { const store = new RunStore(jobsDir); cleanups.push(() => store.close()); store.registerLaunch({ + benchmark: "harbor", jobName: "job-b", dataset: "test-dataset@1.0", agent: "omp", @@ -175,7 +176,7 @@ describe("RunStore", () => { }); describe("ManagerServer API", () => { - it("serves runs, trials, transcripts, and validates launches", async () => { + it("serves uniform runs, traces, and rejects invalid launches", async () => { const jobsDir = makeJobsDir(); writeFixtureJob(jobsDir, "job-api"); const manager = new ManagerServer(jobsDir); @@ -190,15 +191,15 @@ describe("ManagerServer API", () => { const detailRes = await fetch(`${base}/api/runs/job-api`); expect(detailRes.status).toBe(200); - const detail = (await detailRes.json()) as { run: { pass: number }; trials: Array<{ status: string }> }; + const detail = (await detailRes.json()) as { run: { pass: number }; traces: Array<{ status: string }> }; expect(detail.run.pass).toBe(1); - expect(detail.trials).toHaveLength(3); + expect(detail.traces).toHaveLength(3); - const tr = await fetch(`${base}/api/runs/job-api/trials/alpha__abc/transcript?tail=10`); + const tr = await fetch(`${base}/api/runs/job-api/traces/alpha__abc?tail=10`); expect(tr.status).toBe(200); - const transcript = (await tr.json()) as { entries: Array<{ kind: string; tools?: string[] }> }; - expect(transcript.entries.map(e => e.kind)).toEqual(["assistant", "toolResult"]); - expect(transcript.entries[0].tools).toEqual(["read"]); + const trace = (await tr.json()) as { entries: Array<{ kind: string; tools?: string[] }> }; + expect(trace.entries.map(e => e.kind)).toEqual(["assistant", "toolResult"]); + expect(trace.entries[0].tools).toEqual(["read"]); const missing = await fetch(`${base}/api/runs/nope`); expect(missing.status).toBe(404); @@ -215,4 +216,80 @@ describe("ManagerServer API", () => { }; expect(cancelUnknown.cancelled).toBe(false); }); + + it("serves edit and SnapCompact metrics and native traces through one API", async () => { + const jobsDir = makeJobsDir(); + const manager = new ManagerServer(jobsDir); + for (const benchmark of ["edit", "snapcompact"] as const) { + const jobName = `${benchmark}-arm`; + manager.store.registerLaunch({ + benchmark, + jobName, + dataset: benchmark === "edit" ? "typescript-edit" : "squad-dev", + agent: benchmark, + models: ["test/model"], + pid: process.pid, + }); + manager.store.markExit(jobName, 0); + } + const editDir = path.join(jobsDir, "edit-arm"); + fs.writeFileSync( + path.join(editDir, "result.json"), + JSON.stringify({ + tasks: [ + { + id: "rename", + name: "Rename", + runs: [{ runIndex: 0, success: true, duration: 10, tokens: { input: 8, output: 2, reasoning: 0 } }], + }, + ], + summary: { + totalRuns: 1, + successfulRuns: 1, + taskSuccessRate: 1, + editSuccessRate: 1, + totalTokens: { input: 8, output: 2 }, + }, + }), + ); + fs.mkdirSync(path.join(editDir, "result.dump", "rename"), { recursive: true }); + fs.writeFileSync(path.join(editDir, "result.dump", "rename", "run-1.md"), "# conversation\n\nassistant answer"); + const snapDir = path.join(jobsDir, "snapcompact-arm"); + fs.writeFileSync( + path.join(snapDir, "records.jsonl"), + `${JSON.stringify({ cond: "text", chunk: 0, pos_rel: 0, q: "question", answer: "answer", golds: ["gold"], em: 0, f1: 0.5 })}\n`, + ); + fs.writeFileSync( + path.join(snapDir, "summary.json"), + JSON.stringify({ + rows: [{ n: 1, f1: 0.5, em: 0, cost_usd: 0.1, tokens_in: 10, tokens_out: 2, cache_w: 0, cache_r: 0 }], + }), + ); + manager.store.syncAll(); + const server = manager.start(0); + cleanups.push(() => { + void manager.stop(); + }); + const base = `http://localhost:${server.port}`; + + const edit = (await (await fetch(`${base}/api/runs/edit-arm`)).json()) as { + run: { benchmark: string; metrics: Record }; + traces: Array<{ name: string }>; + }; + expect(edit.run).toMatchObject({ benchmark: "edit", metrics: { task_success_rate: 1, edit_success_rate: 1 } }); + const editTrace = (await ( + await fetch(`${base}/api/runs/edit-arm/traces/${encodeURIComponent(edit.traces[0].name)}`) + ).json()) as { entries: Array<{ kind: string; text: string }> }; + expect(editTrace.entries).toEqual([{ kind: "conversation", text: "# conversation\n\nassistant answer" }]); + + const snap = (await (await fetch(`${base}/api/runs/snapcompact-arm`)).json()) as { + run: { benchmark: string; metrics: Record }; + traces: Array<{ name: string }>; + }; + expect(snap.run).toMatchObject({ benchmark: "snapcompact", metrics: { f1: 0.5, exact_match: 0 } }); + const snapTrace = (await ( + await fetch(`${base}/api/runs/snapcompact-arm/traces/${encodeURIComponent(snap.traces[0].name)}`) + ).json()) as { entries: Array<{ kind: string }> }; + expect(snapTrace.entries.map(entry => entry.kind)).toEqual(["question", "answer", "reference"]); + }); }); diff --git a/packages/harbor-manager/src/runner.ts b/packages/harbor-manager/src/runner.ts index a716fd39e..b1bd1472c 100755 --- a/packages/harbor-manager/src/runner.ts +++ b/packages/harbor-manager/src/runner.ts @@ -11,9 +11,9 @@ * process renders a live dashboard (progress / success% / spend / tokens / ETA) * by polling each trial's `result.json`. On completion it writes a markdown report. * - * bun src/runner.ts --model anthropic/claude-sonnet-4-6 --tasks 20 --concurrency 4 - * bun src/runner.ts --agent oracle --tasks 2 # cheap pipeline smoke - * bun src/runner.ts --help + * harbor-manager harbor --model anthropic/claude-sonnet-4-6 --tasks 20 --concurrency 4 + * harbor-manager harbor --agent oracle --tasks 2 # cheap pipeline smoke + * harbor-manager harbor --help */ import { spawnSync } from "node:child_process"; import * as fs from "node:fs"; @@ -108,7 +108,7 @@ function defaultConfig(): Config { const HELP = `harbor-manager runner (local omp) -Usage: bun src/runner.ts [options] [-- ] +Usage: harbor-manager harbor [options] [-- ] Commands: cleanup Force-remove ALL leftover Harbor containers + networks, then exit diff --git a/packages/harbor-manager/src/server.ts b/packages/harbor-manager/src/server.ts index 7068d555c..a96353d9a 100755 --- a/packages/harbor-manager/src/server.ts +++ b/packages/harbor-manager/src/server.ts @@ -6,18 +6,20 @@ * bun src/server.ts [--port 4700] [--jobs-dir ] * * API: + * GET /api/experiments → experiment summaries across all benchmarks * GET /api/runs → RunRow[] - * POST /api/runs → launch a run (JSON body, see LaunchRequest) - * GET /api/runs/:name → { run, trials } - * DELETE /api/runs/:name → cancel a manager-launched run - * GET /api/runs/:name/trials/:trial/transcript?tail=N[&raw=1] + * POST /api/runs → launch any benchmark + * GET /api/runs/:name → { run, traces } + * DELETE /api/runs/:name → cancel a managed run + * GET /api/runs/:name/traces/:trace → normalized trace * GET /api/events → SSE: run-list snapshots on change */ import * as fs from "node:fs"; import * as path from "node:path"; import type { Server, Subprocess } from "bun"; +import { BENCHMARK_DEFINITIONS } from "./benchmarks"; import { buildExperiments, experimentDetail, experimentOf } from "./experiments"; -import { type RunRole, RunStore } from "./store"; +import { type BenchmarkKind, type RunRole, RunStore } from "./store"; /** PUT /api/experiments/:id body — goal and per-run role/note metadata. */ export interface ExperimentMetaUpdate { @@ -33,6 +35,8 @@ const DEFAULT_JOBS_DIR = path.join(REPO_ROOT, "runs", "harbor"); /** POST /api/runs body. Mirrors the runner CLI surface we actually use. */ export interface LaunchRequest { + /** Benchmark adapter to execute. */ + benchmark?: BenchmarkKind; model: string; dataset?: string; /** Task count for a dataset sample, or omit when `include` is given. */ @@ -40,12 +44,14 @@ export interface LaunchRequest { /** Explicit task names (passed as repeated --include). */ include?: string[]; concurrency?: number; + /** SnapCompact conditions; ignored by other benchmarks. */ + conditions?: string[]; timeoutMultiplier?: number; attempts?: number; agent?: string; jobName?: string; webSearch?: boolean; - slide?: { model: string; turns?: number; onAction?: boolean; plan?: boolean }; + slide?: { model: string; turns?: number; onAction?: boolean; plan?: boolean; checklist?: boolean }; /** Role of this run inside its experiment (baseline vs treatment). */ role?: RunRole; /** One-line description of what this arm tests. */ @@ -180,6 +186,9 @@ export class ManagerServer { }); } if (p === "/api/events") return this.#sseResponse(); + if (p === "/api/benchmarks" && request.method === "GET") { + return Response.json(BENCHMARK_DEFINITIONS); + } if (p === "/api/experiments" && request.method === "GET") { return Response.json(buildExperiments(this.#store)); } @@ -207,15 +216,15 @@ export class ManagerServer { if (request.method === "DELETE") return Response.json(this.cancel(jobName)); const run = this.#store.syncRun(jobName); if (!run) return Response.json({ error: "run not found" }, { status: 404 }); - return Response.json({ run, trials: this.#store.listTrials(jobName) }); + return Response.json({ run, traces: this.#store.listTraces(jobName) }); } - const trialMatch = p.match(/^\/api\/runs\/([^/]+)\/trials\/([^/]+)\/transcript$/); - if (trialMatch) { - const jobName = decodeURIComponent(trialMatch[1]); - const trial = decodeURIComponent(trialMatch[2]); + const traceMatch = p.match(/^\/api\/runs\/([^/]+)\/traces\/([^/]+)$/); + if (traceMatch) { + const jobName = decodeURIComponent(traceMatch[1]); + const trace = decodeURIComponent(traceMatch[2]); const tail = Number(url.searchParams.get("tail") ?? "120"); const raw = url.searchParams.get("raw") === "1"; - return this.#transcript(jobName, trial, tail, raw); + return this.#trace(jobName, trace, tail, raw); } return Response.json({ error: "not found" }, { status: 404 }); } catch (err) { @@ -248,43 +257,78 @@ export class ManagerServer { }); } - /** Spawn the CLI runner for `request` and register the run. */ + /** Launch any supported benchmark and register it in the uniform run store. */ launch(request: LaunchRequest): { jobName: string; pid: number } { if (!request.model) throw new Error("model is required"); - const dataset = request.dataset ?? "terminal-bench@2.0"; + const benchmark = request.benchmark ?? "harbor"; + if (benchmark !== "harbor" && benchmark !== "edit" && benchmark !== "snapcompact") { + throw new Error(`unsupported benchmark: ${benchmark}`); + } + const dataset = + request.dataset ?? + (benchmark === "harbor" ? "terminal-bench@2.0" : benchmark === "edit" ? "typescript-edit" : "squad-dev"); const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); const modelSlug = request.model.replace(/[^a-zA-Z0-9]+/g, "-"); const jobName = request.jobName ?? `${modelSlug}-${stamp}`; if (this.#children.has(jobName) || this.#store.getRun(jobName)?.status === "running") { throw new Error(`run ${jobName} is already running`); } + const jobDir = path.join(this.jobsDir, jobName); + fs.mkdirSync(jobDir, { recursive: true }); - const argv = ["bun", "src/runner.ts", "--model", request.model, "-d", dataset, "--job-name", jobName]; - if (request.agent) argv.push("--agent", request.agent); - if (request.tasks !== undefined) argv.push("--tasks", String(request.tasks)); - if (request.concurrency !== undefined) argv.push("--concurrency", String(request.concurrency)); - if (request.attempts !== undefined) argv.push("--attempts", String(request.attempts)); - if (request.timeoutMultiplier !== undefined) argv.push("--timeout-multiplier", String(request.timeoutMultiplier)); - if (request.webSearch) argv.push("--web-search"); - for (const task of request.include ?? []) argv.push("--include", task); - if (request.slide) { - argv.push("--agent-arg", "--reasoning-slide-model", "--agent-arg", request.slide.model); - if (request.slide.onAction) argv.push("--agent-arg", "--reasoning-slide-on-action"); - else if (request.slide.turns !== undefined) { - argv.push("--agent-arg", "--reasoning-slide-turns", "--agent-arg", String(request.slide.turns)); - } else throw new Error("slide requires turns or onAction"); - if (request.slide.plan) argv.push("--agent-arg", "--reasoning-slide-plan"); - // The runner only auto-routes gateway auth for the primary model's - // provider; declare the slide model's provider explicitly so its - // requests reach the gateway too. - const slideProvider = request.slide.model.split("/", 1)[0]; - if (slideProvider) argv.push("--providers", slideProvider); - } - // Default to source mode (repo bind-mount, no rebuild); prebuilt binaries only on request. - if (request.prebuiltBinaries) { - for (const name of ["omp-linux-arm64", "omp-linux-x64"]) { - const binary = path.join(REPO_ROOT, "packages", "coding-agent", "dist", name); - if (fs.existsSync(binary)) argv.push("--binary", binary); + let argv: string[]; + let cwd: string; + if (benchmark === "edit") { + cwd = PKG_DIR; + argv = ["bun", "adapters/edit/cli.ts", "--model", request.model, "--output", path.join(jobDir, "result.json")]; + if (request.tasks !== undefined) argv.push("--max-tasks", String(request.tasks)); + if (request.include?.length) argv.push("--tasks", request.include.join(",")); + if (request.concurrency !== undefined) argv.push("--task-concurrency", String(request.concurrency)); + if (request.attempts !== undefined) argv.push("--runs", String(request.attempts)); + } else if (benchmark === "snapcompact") { + cwd = PKG_DIR; + argv = ["uv", "run", "src/adapters/snapcompact.py", "--model", request.model, "--output-dir", jobDir]; + if (request.tasks !== undefined) argv.push("--limit-paras", String(request.tasks)); + if (request.concurrency !== undefined) argv.push("--workers", String(request.concurrency)); + if (request.conditions?.length) argv.push("--conditions", request.conditions.join(",")); + } else { + cwd = PKG_DIR; + argv = [ + "bun", + "src/runner.ts", + "--model", + request.model, + "-d", + dataset, + "--job-name", + jobName, + "--jobs-dir", + this.jobsDir, + ]; + if (request.agent) argv.push("--agent", request.agent); + if (request.tasks !== undefined) argv.push("--tasks", String(request.tasks)); + if (request.concurrency !== undefined) argv.push("--concurrency", String(request.concurrency)); + if (request.attempts !== undefined) argv.push("--attempts", String(request.attempts)); + if (request.timeoutMultiplier !== undefined) + argv.push("--timeout-multiplier", String(request.timeoutMultiplier)); + if (request.webSearch) argv.push("--web-search"); + for (const task of request.include ?? []) argv.push("--include", task); + if (request.slide) { + argv.push("--agent-arg", "--reasoning-slide-model", "--agent-arg", request.slide.model); + if (request.slide.onAction) argv.push("--agent-arg", "--reasoning-slide-on-action"); + else if (request.slide.turns !== undefined) { + argv.push("--agent-arg", "--reasoning-slide-turns", "--agent-arg", String(request.slide.turns)); + } else throw new Error("slide requires turns or onAction"); + if (request.slide.plan) argv.push("--agent-arg", "--reasoning-slide-plan"); + if (request.slide.checklist) argv.push("--agent-arg", "--reasoning-slide-checklist"); + const slideProvider = request.slide.model.split("/", 1)[0]; + if (slideProvider) argv.push("--providers", slideProvider); + } + if (request.prebuiltBinaries) { + for (const name of ["omp-linux-arm64", "omp-linux-x64"]) { + const binary = path.join(REPO_ROOT, "packages", "coding-agent", "dist", name); + if (fs.existsSync(binary)) argv.push("--binary", binary); + } } } argv.push(...(request.extraArgs ?? [])); @@ -293,7 +337,7 @@ export class ManagerServer { fs.mkdirSync(logDir, { recursive: true }); const logFile = fs.openSync(path.join(logDir, `${jobName}.log`), "w"); const proc = Bun.spawn(argv, { - cwd: PKG_DIR, + cwd, stdout: logFile, stderr: logFile, env: { ...process.env }, @@ -309,11 +353,13 @@ export class ManagerServer { this.#tick(); }); this.#store.registerLaunch({ + benchmark, jobName, dataset, agent: request.agent ?? "omp", models: [request.model], slide: request.slide, + config: { ...request }, pid: proc.pid, role: request.role, note: request.note, @@ -354,12 +400,44 @@ export class ManagerServer { return { jobName, cancelled: false }; } - /** Compact transcript view of a trial's omp.txt session JSONL. */ - #transcript(jobName: string, trial: string, tail: number, raw: boolean): Response { - const file = path.join(this.jobsDir, jobName, trial, "agent", "omp.txt"); - if (!fs.existsSync(file)) return Response.json({ error: "transcript not found" }, { status: 404 }); - const lines = fs.readFileSync(file, "utf8").split("\n").filter(Boolean); + /** Return a normalized trace regardless of the benchmark's native artifact format. */ + #trace(jobName: string, traceName: string, tail: number, raw: boolean): Response { + const trace = this.#store.listTraces(jobName).find(item => item.name === traceName); + if (!trace?.tracePath) return Response.json({ error: "trace not found" }, { status: 404 }); + const jobDir = path.join(this.jobsDir, jobName); const n = Number.isSafeInteger(tail) && tail > 0 ? Math.min(tail, 2000) : 120; + if (trace.tracePath.startsWith("record:")) { + const lineNumber = Number(trace.tracePath.slice("record:".length)); + const line = fs.readFileSync(path.join(jobDir, "records.jsonl"), "utf8").split("\n")[lineNumber - 1]; + if (!line) return Response.json({ error: "trace not found" }, { status: 404 }); + if (raw) return new Response(line, { headers: { "content-type": "application/json" } }); + const record = JSON.parse(line) as Record; + return Response.json({ + jobName, + trace: traceName, + entries: [ + { kind: "question", text: String(record.q ?? "") }, + { kind: "answer", model: this.#store.getRun(jobName)?.models ?? "", text: String(record.answer ?? "") }, + { kind: "reference", text: JSON.stringify(record.golds ?? []) }, + ], + totalEvents: 3, + }); + } + const file = path.resolve(jobDir, trace.tracePath); + if (!file.startsWith(`${path.resolve(jobDir)}${path.sep}`) || !fs.existsSync(file)) { + return Response.json({ error: "trace not found" }, { status: 404 }); + } + const text = fs.readFileSync(file, "utf8"); + if (!file.endsWith(".txt")) { + if (raw) return new Response(text, { headers: { "content-type": "text/plain; charset=utf-8" } }); + return Response.json({ + jobName, + trace: traceName, + entries: [{ kind: "conversation", text }], + totalEvents: 1, + }); + } + const lines = text.split("\n").filter(Boolean); if (raw) { return new Response(lines.slice(-n).join("\n"), { headers: { "content-type": "application/x-ndjson" }, @@ -373,41 +451,30 @@ export class ManagerServer { } catch { continue; } - const type = event.type; - if (type === "message_end") { + if (event.type === "message_end") { const message = event.message as Record | undefined; if (!message) continue; - const role = message.role; - if (role === "assistant") { - const content = Array.isArray(message.content) - ? (message.content as Array>) - : []; - const text = content - .filter(block => block.type === "text") - .map(block => String(block.text ?? "")) - .join("\n"); + const content = Array.isArray(message.content) ? (message.content as Array>) : []; + const body = content + .filter(block => block.type === "text") + .map(block => String(block.text ?? "")) + .join("\n"); + if (message.role === "assistant") { const tools = content.filter(block => block.type === "toolCall").map(block => String(block.name ?? "?")); - entries.push({ kind: "assistant", model: message.model ?? "", text, tools }); - } else if (role === "toolResult") { - const content = Array.isArray(message.content) - ? (message.content as Array>) - : []; - const text = content - .filter(block => block.type === "text") - .map(block => String(block.text ?? "")) - .join("\n"); + entries.push({ kind: "assistant", model: message.model ?? "", text: body, tools }); + } else if (message.role === "toolResult") { entries.push({ kind: "toolResult", tool: message.toolName ?? "?", isError: message.isError === true, - text: text.length > 1600 ? `${text.slice(0, 1600)}…` : text, + text: body.length > 1600 ? `${body.slice(0, 1600)}…` : body, }); } - } else if (type === "notice") { + } else if (event.type === "notice") { entries.push({ kind: "notice", text: event.message ?? "" }); } } - return Response.json({ jobName, trial, entries: entries.slice(-n), totalEvents: lines.length }); + return Response.json({ jobName, trace: traceName, entries: entries.slice(-n), totalEvents: lines.length }); } } diff --git a/packages/harbor-manager/src/store.ts b/packages/harbor-manager/src/store.ts index 152a56d63..848cd6415 100644 --- a/packages/harbor-manager/src/store.ts +++ b/packages/harbor-manager/src/store.ts @@ -10,19 +10,26 @@ import { Database } from "bun:sqlite"; import * as fs from "node:fs"; import * as path from "node:path"; -import { aggregate, readJobResult, readTrials } from "./runner"; +import { readBenchmarkSnapshot } from "./benchmarks"; +import { readJobResult } from "./runner"; export type RunStatus = "running" | "complete" | "failed" | "cancelled"; +/** Benchmark implementation that produced a run. */ +export type BenchmarkKind = "harbor" | "edit" | "snapcompact"; + /** How a run relates to its experiment's question. */ export type RunRole = "baseline" | "variant" | ""; export interface RunRow { + benchmark: BenchmarkKind; jobName: string; dataset: string; agent: string; models: string; slide: string | null; + /** Benchmark-specific launch configuration. */ + config: Record; /** Role inside the experiment (baseline vs treatment); "" when unspecified. */ role: RunRole; /** One-line description of what this arm tests (e.g. "slide→flash after 8 turns"). */ @@ -42,9 +49,13 @@ export interface RunRow { tokIn: number; tokOut: number; tokCache: number; + /** Benchmark-native aggregate score, when the benchmark exposes one. */ + score: number | null; + /** Values keyed by the adapter's metric definitions. */ + metrics: Record; } -export interface TrialRow { +export interface TraceRow { jobName: string; name: string; task: string; @@ -54,9 +65,12 @@ export interface TrialRow { durationMs: number; detail: string; updatedAt: number; + /** Adapter-owned locator used by the uniform trace endpoint. */ + tracePath: string | null; } export interface LaunchRecord { + benchmark: BenchmarkKind; jobName: string; dataset: string; agent: string; @@ -65,17 +79,20 @@ export interface LaunchRecord { pid: number; role?: RunRole; note?: string; + config?: Record; } const SCHEMA = ` CREATE TABLE IF NOT EXISTS runs ( job_name TEXT PRIMARY KEY, + benchmark TEXT NOT NULL DEFAULT 'harbor', dataset TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL DEFAULT 'omp', models TEXT NOT NULL DEFAULT '', slide TEXT, role TEXT NOT NULL DEFAULT '', note TEXT NOT NULL DEFAULT '', + config_json TEXT NOT NULL DEFAULT '{}', status TEXT NOT NULL DEFAULT 'running', pid INTEGER, exit_code INTEGER, @@ -90,6 +107,8 @@ CREATE TABLE IF NOT EXISTS runs ( cost_usd REAL NOT NULL DEFAULT 0, tok_in INTEGER NOT NULL DEFAULT 0, tok_out INTEGER NOT NULL DEFAULT 0, + score REAL, + metrics_json TEXT NOT NULL DEFAULT '{}', tok_cache INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS trials ( @@ -101,6 +120,7 @@ CREATE TABLE IF NOT EXISTS trials ( cost_usd REAL NOT NULL DEFAULT 0, duration_ms INTEGER NOT NULL DEFAULT 0, detail TEXT NOT NULL DEFAULT '', + trace_path TEXT, updated_at INTEGER NOT NULL, PRIMARY KEY (job_name, name) ); @@ -125,12 +145,25 @@ export class RunStore { this.#db = new Database(dbPath ?? path.join(jobsDir, "_manager", "harbor-manager.sqlite")); this.#db.run("PRAGMA journal_mode = WAL"); this.#db.run(SCHEMA); - // Migration for stores created before run roles/notes existed. - const columns = new Set( + const runColumns = new Set( (this.#db.query("PRAGMA table_info(runs)").all() as Array<{ name: string }>).map(c => c.name), ); - if (!columns.has("role")) this.#db.run("ALTER TABLE runs ADD COLUMN role TEXT NOT NULL DEFAULT ''"); - if (!columns.has("note")) this.#db.run("ALTER TABLE runs ADD COLUMN note TEXT NOT NULL DEFAULT ''"); + if (!runColumns.has("role")) this.#db.run("ALTER TABLE runs ADD COLUMN role TEXT NOT NULL DEFAULT ''"); + if (!runColumns.has("note")) this.#db.run("ALTER TABLE runs ADD COLUMN note TEXT NOT NULL DEFAULT ''"); + if (!runColumns.has("benchmark")) { + this.#db.run("ALTER TABLE runs ADD COLUMN benchmark TEXT NOT NULL DEFAULT 'harbor'"); + } + if (!runColumns.has("config_json")) { + this.#db.run("ALTER TABLE runs ADD COLUMN config_json TEXT NOT NULL DEFAULT '{}'"); + } + if (!runColumns.has("score")) this.#db.run("ALTER TABLE runs ADD COLUMN score REAL"); + if (!runColumns.has("metrics_json")) { + this.#db.run("ALTER TABLE runs ADD COLUMN metrics_json TEXT NOT NULL DEFAULT '{}'"); + } + const traceColumns = new Set( + (this.#db.query("PRAGMA table_info(trials)").all() as Array<{ name: string }>).map(c => c.name), + ); + if (!traceColumns.has("trace_path")) this.#db.run("ALTER TABLE trials ADD COLUMN trace_path TEXT"); } close(): void { @@ -139,26 +172,34 @@ export class RunStore { /** Register a run this manager just launched (pid-owning). */ registerLaunch(launch: LaunchRecord): void { + this.#db.query("DELETE FROM trials WHERE job_name = ?").run(launch.jobName); this.#db .query( - `INSERT INTO runs (job_name, dataset, agent, models, slide, role, note, status, pid, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, 'running', ?, ?) + `INSERT INTO runs + (job_name, benchmark, dataset, agent, models, slide, role, note, config_json, status, pid, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', ?, ?) ON CONFLICT(job_name) DO UPDATE SET - pid = excluded.pid, status = 'running', + benchmark = excluded.benchmark, pid = excluded.pid, status = 'running', + config_json = excluded.config_json, role = CASE WHEN excluded.role != '' THEN excluded.role ELSE runs.role END, note = CASE WHEN excluded.note != '' THEN excluded.note ELSE runs.note END`, ) .run( launch.jobName, + launch.benchmark, launch.dataset, launch.agent, launch.models.join(","), launch.slide ? JSON.stringify(launch.slide) : null, launch.role ?? "", launch.note ?? "", + JSON.stringify(launch.config ?? {}), launch.pid, Date.now(), ); + const jobDir = path.join(this.jobsDir, launch.jobName); + fs.mkdirSync(jobDir, { recursive: true }); + fs.writeFileSync(path.join(jobDir, "manager.json"), JSON.stringify(launch, null, 2)); } /** Upsert the experiment's stated goal. */ @@ -230,65 +271,68 @@ export class RunStore { syncRun(jobName: string): RunRow | null { const jobDir = path.join(this.jobsDir, jobName); if (!fs.existsSync(jobDir)) return this.getRun(jobName); - const trials = readTrials(jobDir); - const job = readJobResult(jobDir); - const totals = aggregate(trials, job, job?.nTotal ?? trials.length); + const row = this.getRun(jobName); + if (!row) return null; + const snapshot = readBenchmarkSnapshot(row.benchmark, jobDir); const now = Date.now(); const upsert = this.#db.query( - `INSERT INTO trials (job_name, name, task, status, reward, cost_usd, duration_ms, detail, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `INSERT INTO trials + (job_name, name, task, status, reward, cost_usd, duration_ms, detail, trace_path, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(job_name, name) DO UPDATE SET status = excluded.status, reward = excluded.reward, cost_usd = excluded.cost_usd, - duration_ms = excluded.duration_ms, detail = excluded.detail, updated_at = excluded.updated_at`, + duration_ms = excluded.duration_ms, detail = excluded.detail, + trace_path = excluded.trace_path, updated_at = excluded.updated_at`, ); const tx = this.#db.transaction(() => { - for (const t of trials) { + for (const trace of snapshot.traces) { upsert.run( jobName, - t.name, - t.name.replace(/__[^_]+$/, ""), - t.status, - t.reward, - t.costUsd, - t.durationMs, - t.detail, + trace.name, + trace.task, + trace.status, + trace.reward, + trace.costUsd, + trace.durationMs, + trace.detail, + trace.tracePath, now, ); } this.#db .query( `UPDATE runs SET n_total = ?, done = ?, pass = ?, fail = ?, error = ?, running = ?, - cost_usd = ?, tok_in = ?, tok_out = ?, tok_cache = ? WHERE job_name = ?`, + cost_usd = ?, tok_in = ?, tok_out = ?, tok_cache = ?, score = ?, metrics_json = ? + WHERE job_name = ?`, ) .run( - totals.total, - totals.done, - totals.pass, - totals.fail, - totals.error, - totals.running, - totals.costUsd, - totals.tokIn, - totals.tokOut, - totals.tokCache, + snapshot.total, + snapshot.done, + snapshot.pass, + snapshot.fail, + snapshot.error, + snapshot.running, + snapshot.costUsd, + snapshot.tokIn, + snapshot.tokOut, + snapshot.tokCache, + snapshot.score, + JSON.stringify(snapshot.metrics), jobName, ); - // Foreign runs (no owning pid and never finalized by markExit) infer - // their lifecycle from Harbor's job-level result: `finished_at` is - // terminal; a missing terminal marker with a fresh job dir means still - // running; stale for >30 min means the harness died mid-run. - const row = this.getRun(jobName); - if (row && row.pid === null && row.finishedAt === null && row.status !== "cancelled") { - const job2 = readJobResult(jobDir); + // Historical Harbor runs have no owning process. Infer their terminal + // state from result metadata or directory freshness. + if (row.benchmark === "harbor" && row.pid === null && row.finishedAt === null && row.status !== "cancelled") { + const result = readJobResult(jobDir); let status: RunStatus; let finishedAt: number | null = null; - if (job2?.finishedAt != null) { + if (result?.finishedAt != null) { status = "complete"; - finishedAt = job2.finishedAt; + finishedAt = result.finishedAt; } else if (jobDirFresh(jobDir)) { status = "running"; } else { - status = totals.done > 0 && totals.done >= totals.total ? "complete" : "failed"; + status = snapshot.done > 0 && snapshot.done >= snapshot.total ? "complete" : "failed"; finishedAt = jobDirMtime(jobDir); } if (status !== row.status) { @@ -343,7 +387,7 @@ export class RunStore { return rows.map(rowToRun); } - listTrials(jobName: string): TrialRow[] { + listTraces(jobName: string): TraceRow[] { const rows = this.#db.query("SELECT * FROM trials WHERE job_name = ? ORDER BY name").all(jobName) as Array< Record >; @@ -357,17 +401,20 @@ export class RunStore { durationMs: Number(r.duration_ms), detail: String(r.detail), updatedAt: Number(r.updated_at), + tracePath: r.trace_path === null ? null : String(r.trace_path), })); } } function rowToRun(r: Record): RunRow { return { + benchmark: String(r.benchmark ?? "harbor") as BenchmarkKind, jobName: String(r.job_name), dataset: String(r.dataset), agent: String(r.agent), models: String(r.models), slide: r.slide === null ? null : String(r.slide), + config: JSON.parse(String(r.config_json ?? "{}")), role: String(r.role ?? "") as RunRole, note: String(r.note ?? ""), status: String(r.status) as RunStatus, @@ -385,6 +432,8 @@ function rowToRun(r: Record): RunRow { tokIn: Number(r.tok_in), tokOut: Number(r.tok_out), tokCache: Number(r.tok_cache), + score: r.score === null ? null : Number(r.score), + metrics: JSON.parse(String(r.metrics_json ?? "{}")), }; } diff --git a/packages/harbor-manager/src/web/app.tsx b/packages/harbor-manager/src/web/app.tsx index f1aed9356..37515c981 100644 --- a/packages/harbor-manager/src/web/app.tsx +++ b/packages/harbor-manager/src/web/app.tsx @@ -6,7 +6,7 @@ * #/exp/ experiment detail — arm table, dithered comparison charts * (projected values for in-flight arms, dimmed), task matrix * #/runs flat run list (legacy view) - * #/runs/ run detail — trial grid + live transcript tail + * #/runs/ run detail — normalized trace grid + live trace viewer */ import { useCallback, useEffect, useRef, useState } from "react"; import { createRoot } from "react-dom/client"; @@ -14,11 +14,13 @@ import { createRoot } from "react-dom/client"; // ── api types (mirrors server modules) ────────────────────────────────────── interface RunRow { + benchmark: "harbor" | "edit" | "snapcompact"; jobName: string; dataset: string; agent: string; models: string; slide: string | null; + config: Record; role: "baseline" | "variant" | ""; note: string; status: "running" | "complete" | "failed" | "cancelled"; @@ -32,9 +34,11 @@ interface RunRow { error: number; running: number; costUsd: number; + score: number | null; + metrics: Record; } -interface TrialRow { +interface TraceRow { name: string; task: string; status: string; @@ -193,7 +197,7 @@ function ExperimentsIndex() {

      {exp.id}
      @@ -375,6 +379,53 @@ const CELL_CLASS: Record = { running: "bg-sky-500 animate-pulse", }; +/** + * The comparison anchor for an experiment: the completed baseline arm with the + * highest pass rate (the "ceiling" a reasoning slide tries to preserve). Ties + * break toward the cheaper arm. Returns null when no baseline has finished data. + */ +function pickReferenceArm(arms: ArmSummary[]): ArmSummary | null { + let ref: ArmSummary | null = null; + for (const a of arms) { + if (a.run.role !== "baseline" || a.passPct === null) continue; + if ( + ref === null || + a.passPct > (ref.passPct ?? -1) || + (a.passPct === ref.passPct && (a.costPerTask ?? Infinity) < (ref.costPerTask ?? Infinity)) + ) { + ref = a; + } + } + return ref; +} + +/** + * Signed, colour-coded offset of a metric from the reference arm. `points` + * shows absolute percentage-point difference (pass rate); `relative` shows a + * percentage change (cost, time). `higherBetter` decides which direction is green. + */ +function Delta({ + value, + reference, + mode, + higherBetter, +}: { + value: number | null; + reference: number | null; + mode: "points" | "relative"; + higherBetter: boolean; +}) { + if (value === null || reference === null) return null; + const raw = + mode === "points" ? value - reference : reference === 0 ? Number.NaN : ((value - reference) / reference) * 100; + if (!Number.isFinite(raw) || Math.abs(raw) < 0.5) { + return ≈; + } + const good = higherBetter ? raw > 0 : raw < 0; + const body = `${raw > 0 ? "+" : "−"}${Math.abs(raw).toFixed(0)}${mode === "relative" ? "%" : ""}`; + return ({body}); +} + function ExperimentPage({ id }: { id: string }) { const detail = usePolled(`/api/experiments/${encodeURIComponent(id)}`, 3000); if (!detail) return
      loading…
      ; @@ -394,12 +445,19 @@ function ExperimentPage({ id }: { id: string }) { a => (a.meanTrialMs === null ? null : a.meanTrialMs / 60000), p => p.meanTrialMs / 60000, ); + const ref = pickReferenceArm(arms); return (

      {id}

      {arms.length} arms · {tasks.length} tasks + {ref && ( + <> + {" "} + · Δ vs {ref.arm} + + )}
      {goal &&

      {goal}

      } @@ -430,6 +488,14 @@ function ExperimentPage({ id }: { id: string }) { {arm.run.role} )} + {ref?.arm === arm.arm && ( + + ref + + )} {arm.passPct !== null ? `${arm.passPct.toFixed(0)}%` : "—"} {arm.projected && →{arm.projected.passPct.toFixed(0)}%} + {ref && ref.arm !== arm.arm && ( + + )} {arm.costPerTask !== null ? fmtUsd(arm.costPerTask) : "—"} {arm.projected && Σ{fmtUsd(arm.projected.totalCostUsd)}} + {ref && ref.arm !== arm.arm && ( + + )} + + + {arm.meanTrialMs !== null ? fmtMin(arm.meanTrialMs) : "—"} + {ref && ref.arm !== arm.arm && ( + + )} - {arm.meanTrialMs !== null ? fmtMin(arm.meanTrialMs) : "—"}       ( + const detail = usePolled<{ run: RunRow; traces: TraceRow[] }>( selected ? `/api/runs/${encodeURIComponent(selected)}` : null, 2500, ); - const [trial, setTrial] = useState(null); - const transcript = usePolled<{ entries: TranscriptEntry[] }>( - selected && trial - ? `/api/runs/${encodeURIComponent(selected)}/trials/${encodeURIComponent(trial)}/transcript?tail=60` + const [trace, setTrace] = useState(null); + const traceData = usePolled<{ entries: TranscriptEntry[] }>( + selected && trace + ? `/api/runs/${encodeURIComponent(selected)}/traces/${encodeURIComponent(trace)}?tail=60` : null, 2500, ); - const transcriptRef = useRef(null); + const traceRef = useRef(null); useEffect(() => { - if (!transcript) return; - const el = transcriptRef.current; + if (!traceData) return; + const el = traceRef.current; if (el) el.scrollTop = el.scrollHeight; - }, [transcript]); + }, [traceData]); const cancel = useCallback(async (name: string) => { if (confirm(`stop ${name}?`)) await fetch(`/api/runs/${encodeURIComponent(name)}`, { method: "DELETE" }); }, []); @@ -578,6 +665,7 @@ function RunsPage({ selected }: { selected: string | null }) { > {r.jobName} +
      {r.benchmark}
      {(r.note || r.role) && (
      {r.role && ( @@ -622,34 +710,42 @@ function RunsPage({ selected }: { selected: string | null }) {
      {detail.run.jobName} {" "} - {detail.run.dataset} · {detail.run.models} + {detail.run.benchmark} · {detail.run.dataset} · {detail.run.models} + {detail.run.score !== null ? ` · score ${(100 * detail.run.score).toFixed(1)}%` : ""} {detail.run.slide ? ` → ${detail.run.slide}` : ""} +
      + {Object.entries(detail.run.metrics).map(([key, value]) => ( + + {key.replaceAll("_", " ")}: {value === null ? "—" : `${(100 * value).toFixed(1)}%`} + + ))} +
      - {detail.trials.map(t => ( + {detail.traces.map(t => ( setTrial(t.name)} - className={`cursor-pointer border-t border-zinc-800/60 hover:bg-zinc-900 ${t.name === trial ? "bg-zinc-900" : ""}`} + onClick={() => setTrace(t.name)} + className={`cursor-pointer border-t border-zinc-800/60 hover:bg-zinc-900 ${t.name === trace ? "bg-zinc-900" : ""}`} > + - ))}
      {t.task} {t.reward === null ? "—" : t.reward.toFixed(3)} {fmtUsd(t.costUsd)} {t.durationMs ? fmtMin(t.durationMs) : "—"}{t.detail}
      - {trial && ( -
      - {(transcript?.entries ?? []).map((e, i) => ( + {trace && ( +
      + {(traceData?.entries ?? []).map((e, i) => ( // biome-ignore lint/suspicious/noArrayIndexKey: tail window, entries have no ids
      @@ -687,7 +783,7 @@ function LaunchForm({ onDone }: { onDone: () => void }) { async (ev: React.FormEvent) => { ev.preventDefault(); const f = new FormData(ev.currentTarget); - const body: Record = { model: f.get("model") }; + const body: Record = { benchmark: f.get("benchmark"), model: f.get("model") }; if (f.get("jobName")) body.jobName = f.get("jobName"); if (f.get("dataset")) body.dataset = f.get("dataset"); if (f.get("tasks")) body.tasks = Number(f.get("tasks")); @@ -699,6 +795,12 @@ function LaunchForm({ onDone }: { onDone: () => void }) { .map(s => s.trim()) .filter(Boolean); } + if (f.get("conditions")) { + body.conditions = String(f.get("conditions")) + .split(",") + .map(s => s.trim()) + .filter(Boolean); + } if (f.get("goal")) body.goal = f.get("goal"); if (f.get("role")) body.role = f.get("role"); if (f.get("note")) body.note = f.get("note"); @@ -724,10 +826,15 @@ function LaunchForm({ onDone }: { onDone: () => void }) { const input = "rounded border border-zinc-700 bg-zinc-950 px-2 py-1 text-sm"; return (
      + - + @@ -741,6 +848,7 @@ function LaunchForm({ onDone }: { onDone: () => void }) { plan nudge + = 80) return ANSI.green; - if (percent >= 50) return ANSI.yellow; - return ANSI.red; -} - -function parseThinkingLevel(value: string | null | undefined): ResolvedThinkingLevel | undefined { - return value !== undefined && - value !== null && - [ThinkingLevel.Off, ...THINKING_EFFORTS].includes(value as ResolvedThinkingLevel) - ? (value as ResolvedThinkingLevel) - : undefined; -} - -function generateReportFilename(config: BenchmarkConfig, format: "markdown" | "json"): string { - const modelName = config.model - .split("/") - .pop()! - .replace(/[^a-zA-Z0-9-]/g, "_"); - const variant = config.editVariant ?? "replace"; - const timestamp = new Date().toISOString().replace(/:/g, "-").replace(/\..+$/, "").replace(/Z$/, "Z"); - const ext = format === "json" ? "json" : "md"; - return path.join(RUNS_DIR, `${modelName}_${variant}_${timestamp}.${ext}`); -} - -async function resolveConversationDumpDir(outputPath: string): Promise { - const parsed = path.parse(outputPath); - const preferredPath = path.join(parsed.dir, `${parsed.name}.dump`); - try { - await fs.promises.stat(preferredPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return preferredPath; - } - throw error; - } - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - return path.join(parsed.dir, `${parsed.name}.${timestamp}.dump`); -} - -async function conversationDumpStatus(dumpDir: string): Promise { - try { - const stat = await fs.promises.stat(dumpDir); - if (stat.isDirectory()) { - return `Conversation dumps written to: ${dumpDir}`; - } - return `Conversation dump path is not a directory: ${dumpDir}`; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return `No conversation dumps written: ${dumpDir}`; - } - throw error; - } -} - -function printUsage(tasks?: EditTask[]): void { - const taskList = tasks - ? tasks.map(t => ` ${t.id.padEnd(30)} ${t.name}`).join("\n") - : " (use --list to see available tasks)"; - console.log(` -Edit Benchmark - Evaluate patch application success rates - -Usage: - bun run bench:edit [options] - -Options: - --model Provider/model ID, e.g. anthropic/claude-sonnet-4-20250514 (default) - --provider Override provider (auto-detected from model prefix if omitted) - --thinking Thinking level: off, minimal, low, medium, high, xhigh, max - --runs Runs per task (default: 1) - --timeout Timeout per run in ms (default: 120000) - --connection-timeout Timeout for first event before fast-retry (default: 30000) - --task-concurrency Max tasks to run in parallel (default: 16) - --tasks Comma-separated task IDs to run (default: all) - --max-tasks Max tasks to sample (default: 80, 0 = all) - --fixtures Fixtures directory or .tar.gz archive (default: built-in) - --edit-variant Edit variant: any string (e.g. replace, patch, hashline, vim, atom, apply_patch), or auto (default: auto) - --edit-fuzzy Fuzzy matching: true, false, auto (default: auto) - --edit-fuzzy-threshold Fuzzy threshold 0-1 or auto (default: auto) - --auto-format Auto-format output files after verify (debug only) - --guided Include an authoritative suggested edit payload (default: false) - --no-guided Disable guided mode - --max-attempts Max prompt attempts per run (default: 1) - --no-op-retry-limit Stop after repeated preventable no-op failures (default: 2) - --mutation-scope-window Allowed line-distance from mutation target for hashline refs (default: 20) - --max-turns Max turn_start events per attempt before failing (default: 30) - --output Output file (default: run_____.md) - --format Output format: markdown, json (default: markdown) - --check-fixtures Validate fixtures and exit - --require-edit-tool-call Require edit tool usage for success (default: false) - --require-read-tool-call Require read tool usage for success (default: false) - --no-edit-required Remove "must edit" prompt requirement (default: false) - --no-early-stop-on-match Don't short-circuit the run when output matches expected (default: false) - --list List available tasks and exit - --help Show this help message - -Available Tasks: -${taskList} - -Examples: - # Run full benchmark with default model - bun run bench:edit - - # Run specific tasks - bun run bench:edit --tasks core-memory-recall,operations-division - - # Compare different models - bun run bench:edit --model claude-sonnet-4-20250514 --output sonnet.md - bun run bench:edit --model claude-opus-4-5-20251101 --output opus.md - - # Run with extended thinking - bun run bench:edit --thinking high --runs 5 - - # Run from a fixtures archive - bun run bench:edit --fixtures edit-fixtures.tar.gz -`); -} - -async function resolveExtractedDir(tempDir: string): Promise { - const entries = await fs.promises.readdir(tempDir, { withFileTypes: true }); - const dirs = entries.filter(entry => entry.isDirectory()); - const files = entries.filter(entry => entry.isFile()); - if (dirs.length === 1 && files.length === 0) { - return path.join(tempDir, dirs[0]!.name); - } - return tempDir; -} - -async function extractTarGz(archivePath: string): Promise<{ dir: string; cleanupDir: string }> { - const tempDirObj = await TempDir.create("@reach-benchmark-fixtures-"); - const tempDir = tempDirObj.path(); - try { - const bytes = await Bun.file(archivePath).arrayBuffer(); - const archive = new Bun.Archive(bytes); - const files = await archive.files(); - - for (const [filePath, file] of files) { - const destPath = path.join(tempDir, filePath); - await Bun.write(destPath, file); - } - } catch (error) { - await tempDirObj.remove(); - const message = error instanceof Error ? error.message : String(error); - throw new Error(`Failed to extract archive: ${message}`, { cause: error }); - } - - return { dir: await resolveExtractedDir(tempDir), cleanupDir: tempDir }; -} - -async function resolveFixtures(fixturesArg?: string): Promise<{ tasks: EditTask[]; cleanup?: () => Promise }> { - fixturesArg ??= path.join(import.meta.dir, "../fixtures.tar.gz"); - - if (fixturesArg.endsWith(".tar.gz") || fixturesArg.endsWith(".tgz")) { - const extracted = await extractTarGz(fixturesArg); - return { - tasks: await loadTasksFromDir(extracted.dir), - cleanup: () => fs.promises.rm(extracted.cleanupDir, { recursive: true, force: true }), - }; - } - - return { tasks: await loadTasksFromDir(fixturesArg) }; -} - -async function main(): Promise { - const { values } = parseArgs({ - options: { - provider: { type: "string" }, - model: { type: "string", default: "anthropic/claude-sonnet-4-20250514" }, - thinking: { type: "string", default: "low" }, - runs: { type: "string", default: "2" }, - timeout: { type: "string", default: "120000" }, - "connection-timeout": { type: "string", default: "30000" }, - "max-turns": { type: "string", default: "30" }, - "task-concurrency": { type: "string", default: "32" }, - tasks: { type: "string" }, - fixtures: { type: "string" }, - output: { type: "string" }, - format: { type: "string", default: "markdown" }, - "check-fixtures": { type: "boolean", default: false }, - "auto-format": { type: "boolean", default: false }, - guided: { type: "boolean", default: false }, - "no-guided": { type: "boolean", default: false }, - "max-attempts": { type: "string", default: "1" }, - "no-op-retry-limit": { type: "string", default: "2" }, - "max-timeout-retries": { type: "string", default: "3" }, - "max-provider-retries": { type: "string", default: "3" }, - "mutation-scope-window": { type: "string", default: "20" }, - "require-edit-tool-call": { type: "boolean", default: false }, - "require-read-tool-call": { type: "boolean", default: false }, - "no-edit-required": { type: "boolean", default: false }, - "edit-variant": { type: "string" }, - "edit-fuzzy": { type: "string" }, - "edit-fuzzy-threshold": { type: "string" }, - "no-in-process": { type: "boolean", default: false }, - "no-early-stop-on-match": { type: "boolean", default: false }, - "max-tasks": { type: "string", default: "80" }, - list: { type: "boolean", default: false }, - help: { type: "boolean", default: false }, - }, - allowPositionals: true, - }); - - // Extract provider for display/config purposes only. - // The full model string (e.g. "openrouter/google/gemini-2.5-flash-lite") is passed - // as --model to the CLI, which handles resolution via parseModelPattern. - const model = values.model!; - const slashIndex = model.indexOf("/"); - const provider = values.provider ?? (slashIndex !== -1 ? model.slice(0, slashIndex) : "anthropic"); - - if (values.help) { - printUsage(); - process.exit(0); - } - - if (values["check-fixtures"] && values.fixtures) { - const issues = await validateFixturesFromDir(values.fixtures); - if (issues.length === 0) { - console.log("Fixtures OK"); - process.exit(0); - } - console.error("Fixture validation failed:"); - for (const issue of issues) { - console.error(` - ${issue.taskId}: ${issue.message}`); - } - process.exit(1); - } - - const { tasks: allTasks, cleanup } = await resolveFixtures(values.fixtures); - - if (values.list) { - console.log("Available Tasks:\n"); - for (const task of allTasks) { - console.log(` ${task.id}`); - console.log(` Name: ${task.name}`); - console.log(` Files: ${task.files.join(", ")}`); - console.log(""); - } - process.exit(0); - } - - let thinkingLevel: ResolvedThinkingLevel = Effort.Low; - if (values.thinking) { - const level = parseThinkingLevel(values.thinking); - if (!level) { - console.error(`Invalid thinking level: ${values.thinking}`); - console.error(`Valid levels: ${[ThinkingLevel.Off, ...THINKING_EFFORTS].join(", ")}`); - process.exit(1); - } - thinkingLevel = level; - } - - const runsPerTask = parseInt(values.runs!, 10); - if (Number.isNaN(runsPerTask) || runsPerTask < 1) { - console.error(`Invalid runs value: ${values.runs}`); - process.exit(1); - } - - const timeout = parseInt(values.timeout!, 10); - if (Number.isNaN(timeout) || timeout < 1000) { - console.error(`Invalid timeout value: ${values.timeout}`); - process.exit(1); - } - - const maxTurns = parseInt(values["max-turns"]!, 10); - if (Number.isNaN(maxTurns) || maxTurns < 1) { - console.error(`Invalid max-turns value: ${values["max-turns"]}. Must be >= 1.`); - process.exit(1); - } - - const taskConcurrency = parseInt(values["task-concurrency"]!, 10); - if (Number.isNaN(taskConcurrency) || taskConcurrency < 1) { - console.error(`Invalid task concurrency value: ${values["task-concurrency"]}`); - process.exit(1); - } - - const maxAttempts = parseInt(values["max-attempts"] ?? "2", 10); - if (Number.isNaN(maxAttempts) || maxAttempts < 1 || maxAttempts > 5) { - console.error(`Invalid max-attempts value: ${values["max-attempts"]}. Must be 1-5.`); - process.exit(1); - } - - const noOpRetryLimit = parseInt(values["no-op-retry-limit"] ?? "2", 10); - const maxTimeoutRetries = parseInt(values["max-timeout-retries"] ?? "3", 10); - const maxProviderRetries = parseInt(values["max-provider-retries"] ?? "3", 10); - const mutationScopeWindow = parseInt(values["mutation-scope-window"] ?? "20", 10); - const connectionTimeout = parseInt(values["connection-timeout"] ?? "30000", 10); - - let tasksToRun = allTasks; - if (values.tasks) { - const taskIds = values.tasks.split(",").map(s => s.trim()); - tasksToRun = []; - for (const id of taskIds) { - const task = allTasks.find(t => t.id === id); - if (!task) { - console.error(`Unknown task ID: ${id}`); - console.error(`Available tasks: ${allTasks.map(t => t.id).join(", ")}`); - process.exit(1); - } - tasksToRun.push(task); - } - } - - // Apply --max-tasks sampling (deterministic by sorting on id) - const maxTasks = parseInt(values["max-tasks"] ?? "80", 10); - if (maxTasks > 0 && tasksToRun.length > maxTasks && !values.tasks) { - // Evenly sample across mutation categories for representative coverage - const sorted = tasksToRun.slice().sort((a, b) => a.id.localeCompare(b.id)); - const step = sorted.length / maxTasks; - tasksToRun = Array.from({ length: maxTasks }, (_, i) => sorted[Math.floor(i * step)]!); - } - - const rawEditVariant = values["edit-variant"] as string | undefined; - const editVariant = rawEditVariant === "" ? undefined : rawEditVariant; - - let editFuzzy: boolean | "auto" | undefined; - if (values["edit-fuzzy"] !== undefined) { - if (values["edit-fuzzy"] === "auto") { - editFuzzy = "auto"; - } else if (values["edit-fuzzy"] === "true" || values["edit-fuzzy"] === "1") { - editFuzzy = true; - } else if (values["edit-fuzzy"] === "false" || values["edit-fuzzy"] === "0") { - editFuzzy = false; - } else { - console.error(`Invalid edit-fuzzy: ${values["edit-fuzzy"]}. Must be true, false, 1, 0, or auto.`); - process.exit(1); - } - } - - let editFuzzyThreshold: number | "auto" | undefined; - if (values["edit-fuzzy-threshold"] !== undefined) { - if (values["edit-fuzzy-threshold"] === "auto") { - editFuzzyThreshold = "auto"; - } else { - const parsed = parseFloat(values["edit-fuzzy-threshold"]); - if (Number.isNaN(parsed) || parsed < 0 || parsed > 1) { - console.error(`Invalid edit-fuzzy-threshold: ${values["edit-fuzzy-threshold"]}. Must be 0-1 or auto.`); - process.exit(1); - } - editFuzzyThreshold = parsed; - } - } - - const guided = values["no-guided"] ? false : values.guided; - - const formatType = values.format === "json" ? "json" : "markdown"; - const config: BenchmarkConfig = { - provider, - model, - thinkingLevel, - runsPerTask, - timeout, - maxTurns, - taskConcurrency, - autoFormat: values["auto-format"], - guided, - maxAttempts, - requireEditToolCall: values["require-edit-tool-call"], - requireReadToolCall: values["require-read-tool-call"], - noEditRequired: values["no-edit-required"], - editVariant, - editFuzzy, - editFuzzyThreshold, - noOpRetryLimit, - maxTimeoutRetries, - maxProviderFailureRetries: maxProviderRetries, - mutationScopeWindow, - connectionTimeout, - inProcess: !values["no-in-process"], - earlyStopOnMatch: !values["no-early-stop-on-match"], - }; - const outputPath = values.output ?? generateReportFilename(config, formatType); - config.conversationDumpDir = await resolveConversationDumpDir(outputPath); - - console.log("Edit Benchmark"); - console.log("=============="); - console.log(`Provider: ${config.provider}`); - console.log(`Model: ${config.model}`); - if (config.thinkingLevel) { - console.log(`Thinking: ${config.thinkingLevel}`); - } - console.log(`Runs per task: ${config.runsPerTask}`); - console.log(`Timeout: ${config.timeout}ms`); - console.log(`Task concurrency: ${config.taskConcurrency}`); - if (config.autoFormat) { - console.log("Auto-format: enabled"); - } - console.log(`Guided mode: ${config.guided ? "enabled" : "disabled"}`); - console.log(`Max attempts: ${config.maxAttempts}`); - if (config.maxTurns !== undefined) { - console.log(`Max turns per attempt: ${config.maxTurns}`); - } - if (config.requireEditToolCall) { - console.log("Require edit tool call: yes"); - } - if (config.requireReadToolCall) { - console.log("Require read tool call: yes"); - } - if (config.noEditRequired) { - console.log("No-edit-required baseline: yes"); - } - if (config.editVariant) { - console.log(`Edit variant: ${config.editVariant}`); - } - if (config.editFuzzy !== undefined) { - console.log(`Edit fuzzy: ${config.editFuzzy}`); - } - if (config.editFuzzyThreshold !== undefined) { - console.log(`Edit fuzzy threshold: ${config.editFuzzyThreshold}`); - } - console.log(`Tasks: ${tasksToRun.length}`); - console.log(`Conversation dumps: ${config.conversationDumpDir}`); - console.log(""); - - const progress = new LiveProgress(tasksToRun.length * config.runsPerTask, config.runsPerTask); - let latestResult = buildBenchmarkResult({ - tasks: tasksToRun, - config, - resultsByTask: new Map(), - startTime: new Date().toISOString(), - }); - let progressFinished = false; - let reportWritePromise: Promise | undefined; - const finishProgress = () => { - if (progressFinished) return; - progress.finish(); - progressFinished = true; - }; - const writeReport = async (result: BenchmarkResult, interrupted: boolean) => { - if (reportWritePromise) return reportWritePromise; - reportWritePromise = (async () => { - if (interrupted) { - console.log(""); - console.log("Benchmark interrupted; writing partial report..."); - } - const report = formatType === "json" ? generateJsonReport(result) : generateReport(result); - await Bun.write(outputPath, report); - console.log(`Report written to: ${outputPath}`); - if (config.conversationDumpDir) { - console.log(await conversationDumpStatus(config.conversationDumpDir)); - } - })(); - return reportWritePromise; - }; - const unregisterReportCleanup = postmortem.register("typescript-edit-benchmark-report", async reason => { - if (reason === postmortem.Reason.EXIT) return; - finishProgress(); - await writeReport(latestResult, true); - if (cleanup) { - await cleanup(); - } - }); - const result = await runBenchmark( - tasksToRun, - config, - event => { - progress.handleEvent(event); - }, - snapshot => { - latestResult = snapshot; - }, - ); - latestResult = result; - finishProgress(); - - console.log(""); - console.log("Benchmark complete!"); - console.log( - ` Task success rate (best of ${config.runsPerTask}): ${(result.summary.taskSuccessRate * 100).toFixed(1)}% (${result.summary.successfulTasks}/${result.summary.totalTasks})`, - ); - console.log( - ` Total tokens (best, overall): ${result.summary.totalTokens.input} in / ${result.summary.totalTokens.output} out`, - ); - console.log( - ` Tokens/task (best, overall): mean=${result.summary.avgTokensPerTask.total} median=${result.summary.medianTokensPerTask.total} p1=${result.summary.p1TokensPerTask.total} p99=${result.summary.p99TokensPerTask.total} reasoning=${result.summary.avgTokensPerTask.reasoning}`, - ); - console.log( - ` Total tokens (one-shot successes): ${result.summary.totalOneShotSuccessTokens.input} in / ${result.summary.totalOneShotSuccessTokens.output} out`, - ); - console.log( - ` Tokens/task (one-shot successes): mean=${result.summary.avgOneShotSuccessTokensPerTask.total} median=${result.summary.medianOneShotSuccessTokensPerTask.total} p1=${result.summary.p1OneShotSuccessTokensPerTask.total} p99=${result.summary.p99OneShotSuccessTokensPerTask.total} reasoning=${result.summary.avgOneShotSuccessTokensPerTask.reasoning}`, - ); - if (result.summary.ghostRuns > 0) { - console.log(` Ghost runs (0/0/0): ${result.summary.ghostRuns}`); - } - if (result.summary.timeoutRuns > 0) { - console.log(` Timeout runs: ${result.summary.timeoutRuns}`); - } - console.log(""); - - await writeReport(result, false); - unregisterReportCleanup(); - - if (cleanup) { - await cleanup(); - } - - // In-process benchmark runs can leave provider keep-alive sockets and - // background AgentSession timers alive after the report is written. Treat the - // final report as the CLI boundary so the command returns to the shell. - await postmortem.quit(0); -} - -class LiveProgress { - readonly #totalRuns: number; - readonly #runsPerTask: number; - readonly #isTty: boolean; - #started = 0; - #completed = 0; - #success = 0; - #totalInput = 0; - #totalOutput = 0; - #totalDuration = 0; - #totalReads = 0; - #totalEdits = 0; - #totalWrites = 0; - #totalEditSuccesses = 0; - #totalToolInputChars = 0; - #indentScores: number[] = []; - #inputTokens: number[] = []; - #outputTokens: number[] = []; - #totalTokens: number[] = []; - #oneShotSuccessTokens: number[] = []; - #lastLineLength = 0; - - constructor(totalRuns: number, runsPerTask: number) { - this.#totalRuns = totalRuns; - this.#runsPerTask = runsPerTask; - this.#isTty = Boolean(process.stdout.isTTY); - } - - handleEvent(event: ProgressEvent): void { - if (event.status === "started") { - this.#started += 1; - if (!this.#isTty) { - console.log(` [${event.taskId}] Run ${event.runIndex + 1}/${this.#runsPerTask} started...`); - } - this.#renderLine(); - return; - } - - this.#completed += 1; - if (event.result) { - if (event.result.success) { - this.#success += 1; - } - if (event.result.success && event.runIndex === 0) { - this.#oneShotSuccessTokens.push(event.result.tokens.total); - } - this.#totalInput += event.result.tokens.input; - this.#totalOutput += event.result.tokens.output; - this.#inputTokens.push(event.result.tokens.input); - this.#outputTokens.push(event.result.tokens.output); - this.#totalTokens.push(event.result.tokens.total); - this.#totalDuration += event.result.duration; - this.#totalReads += event.result.toolCalls.read; - this.#totalEdits += event.result.toolCalls.edit; - this.#totalWrites += event.result.toolCalls.write; - this.#totalEditSuccesses += event.result.toolCalls.editSuccesses; - this.#totalToolInputChars += event.result.toolCalls.totalInputChars; - if (typeof event.result.indentScore === "number") { - this.#indentScores.push(event.result.indentScore); - } - } - - const result = event.result; - if (result && !result.success && result.error) { - this.#flushLine(); - const header = paint(ANSI.red, `[${event.taskId}] Run ${event.runIndex + 1}/${this.#runsPerTask} failed:`); - console.log(` ${header} ${result.error}`); - if (result.diff) { - const changeLines = result.diff - .split("\n") - .filter(line => /^[-+@]/.test(line) && !/^(---|\+\+\+)/.test(line)); - const maxLines = 40; - const shown = changeLines.slice(0, maxLines); - for (const line of shown) { - let color: string | undefined; - if (line.startsWith("@@")) color = ANSI.cyan; - else if (line.startsWith("-")) color = ANSI.red; - else if (line.startsWith("+")) color = ANSI.green; - console.log(` ${color ? paint(color, line) : line}`); - } - if (changeLines.length > maxLines) { - console.log(paint(ANSI.dim, ` ... (${changeLines.length - maxLines} more change lines)`)); - } - } - } - - if (result?.editFailures && result.editFailures.length > 0) { - this.#flushLine(); - for (const [i, failure] of result.editFailures.entries()) { - const args = (failure.args ?? {}) as Record; - const target = - typeof args.path === "string" ? args.path : typeof args.file === "string" ? args.file : undefined; - const op = typeof args.operation === "string" ? args.operation : undefined; - const oneLine = failure.error.replace(/\s+/g, " ").trim(); - const clipped = oneLine.length > 240 ? `${oneLine.slice(0, 237)}...` : oneLine; - const tag = paint(ANSI.yellow, `[${event.taskId}] schema #${i + 1}`); - const metaParts = [op, target].filter((v): v is string => Boolean(v)); - const meta = metaParts.length > 0 ? paint(ANSI.dim, metaParts.join(" ")) : ""; - console.log(` ${tag}${meta ? ` ${meta}` : ""} ${clipped}`); - if (failure.rawBlock) { - const rawLine = failure.rawBlock.replace(/\s+/g, " ").trim(); - const clippedRaw = rawLine.length > 240 ? `${rawLine.slice(0, 237)}...` : rawLine; - console.log(` ${paint(ANSI.dim, "raw")} ${clippedRaw}`); - } - } - } - - if (!this.#isTty) { - const status = event.result?.success ? "completed" : "failed"; - console.log(` [${event.taskId}] Run ${event.runIndex + 1}/${this.#runsPerTask} ${status}`); - } - - this.#renderLine(); - } - - finish(): void { - this.#flushLine(); - this.#printSummary(); - } - - #printSummary(): void { - const n = this.#completed; - const denom = n || 1; - - const successRate = (this.#success / denom) * 100; - const editSuccessRate = this.#totalEdits > 0 ? (this.#totalEditSuccesses / this.#totalEdits) * 100 : 100; - const avgIndent = - this.#indentScores.length > 0 ? this.#indentScores.reduce((a, b) => a + b, 0) / this.#indentScores.length : 0; - - console.log(""); - console.log(paint(ANSI.bold, "Runtime Stats:")); - console.log( - ` Task success: ${paint(rateColor(successRate), `${successRate.toFixed(1)}% (${this.#success}/${n})`)}`, - ); - console.log( - ` Edit success: ${paint(rateColor(editSuccessRate), `${editSuccessRate.toFixed(1)}% (${this.#totalEditSuccesses}/${this.#totalEdits})`)}`, - ); - console.log(` Avg indent score: ${avgIndent.toFixed(2)}`); - console.log(` Tool calls: read=${this.#totalReads} edit=${this.#totalEdits} write=${this.#totalWrites}`); - console.log(` Tool input chars: ${this.#totalToolInputChars.toLocaleString()}`); - const fmtTokens = (samples: number[]): string => { - if (samples.length === 0) return "mean=0 median=0 p1=0 p99=0"; - const sorted = [...samples].sort((a, b) => a - b); - const mean = Math.round(sorted.reduce((a, b) => a + b, 0) / sorted.length); - return `mean=${mean} median=${Math.round(percentile(sorted, 50))} p1=${Math.round(percentile(sorted, 1))} p99=${Math.round(percentile(sorted, 99))}`; - }; - console.log(` Tokens/task in: ${fmtTokens(this.#inputTokens)}`); - console.log(` Tokens/task out: ${fmtTokens(this.#outputTokens)}`); - console.log(` Tokens/task tot: ${fmtTokens(this.#totalTokens)}`); - console.log(` Tokens/task (one-shot successes): ${fmtTokens(this.#oneShotSuccessTokens)}`); - console.log(` Avg time/task: ${Math.round(this.#totalDuration / denom)}ms`); - } - - #renderLine(): void { - if (!this.#isTty) { - return; - } - const successRate = this.#completed > 0 ? (this.#success / this.#completed) * 100 : 0; - const editRate = this.#totalEdits > 0 ? (this.#totalEditSuccesses / this.#totalEdits) * 100 : 100; - const avgInput = this.#completed > 0 ? Math.round(this.#totalInput / this.#completed) : 0; - const avgOutput = this.#completed > 0 ? Math.round(this.#totalOutput / this.#completed) : 0; - const avgDuration = this.#completed > 0 ? Math.round(this.#totalDuration / this.#completed) : 0; - const inFlight = this.#started - this.#completed; - const bar = this.#renderBar(this.#completed, this.#totalRuns, 20); - const progress = paint(ANSI.bold, `${this.#completed}/${this.#totalRuns}`); - const taskCol = `task=${paint(rateColor(successRate), `${successRate.toFixed(0)}%`)}`; - const editCol = `edit=${paint(rateColor(editRate), `${editRate.toFixed(0)}%`)}`; - const tokCol = paint(ANSI.dim, `tok=${avgInput}/${avgOutput}`); - const durCol = paint(ANSI.dim, `${avgDuration}ms`); - const rewCol = paint(ANSI.dim, `r/e/w=${this.#totalReads}/${this.#totalEdits}/${this.#totalWrites}`); - const flyCol = `fly=${paint(ANSI.cyan, String(inFlight))}`; - const line = ` ${bar} ${progress} ${taskCol} ${editCol} ${tokCol} ${durCol} ${rewCol} ${flyCol}`; - this.#writeLine(line); - } - - #renderBar(done: number, total: number, width: number): string { - const ratio = total === 0 ? 0 : done / total; - const filled = Math.round(ratio * width); - const empty = Math.max(0, width - filled); - const filledPart = paint(ANSI.green, "#".repeat(filled)); - const emptyPart = paint(ANSI.dim, "-".repeat(empty)); - return `[${filledPart}${emptyPart}]`; - } - - #writeLine(line: string): void { - const lineWidth = visibleWidth(line); - const pad = this.#lastLineLength > lineWidth ? padding(this.#lastLineLength - lineWidth) : ""; - process.stdout.write(`\r${line}${pad}`); - this.#lastLineLength = lineWidth; - } - - #flushLine(): void { - if (!this.#isTty) { - return; - } - if (this.#lastLineLength > 0) { - process.stdout.write(`\r${padding(this.#lastLineLength)}\r`); - this.#lastLineLength = 0; - } - } -} - -main().catch(async err => { - console.error("Benchmark failed:", err); - await postmortem.quit(1); -}); From af7345e87e720cac7063683a313c436a4188210b Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 03:42:38 +0200 Subject: [PATCH 58/80] feat(coding-agent): implemented role switching and filtering in model picker - Added support for role switching via `@` search prefix in the model picker. - Implemented `quickRoles` configuration and logic to handle role-specific selections via the session API. - Updated the model browser to support preserving query order and custom label coloring for improved navigation. - Integrated role cycle tracking and updated UI hints to support the new role-browsing workflow. --- packages/coding-agent/CHANGELOG.md | 1 + .../src/modes/components/model-browser.ts | 48 +++++--- .../src/modes/components/model-picker.ts | 103 +++++++++++++++--- .../modes/controllers/selector-controller.ts | 27 ++++- .../coding-agent/test/model-picker.test.ts | 41 ++++++- .../selector-settings-side-effects.test.ts | 86 +++++++++++++++ 6 files changed, 273 insertions(+), 33 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index fa441dfcc..621ab0ba9 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -6,6 +6,7 @@ - Added display setting to toggle between collapsing or keeping compacted history inline, now applied to live session displays - Added a compact session-only model picker (Alt+P) for quick model switching without changing roles +- Added `@` search to the Alt+P / `/switch` picker: it lists configured Ctrl+P quick roles in matching segment colors and applies the selected role's model and thinking for the current session. - Redesigned Agent Hub entries as two-line cards: identity (status glyph, name, agent type, parent when nested) on the left, active model + reasoning level and age right-aligned, with the task description on its own line; dropped the redundant `sub · of Main` noise ### Changed diff --git a/packages/coding-agent/src/modes/components/model-browser.ts b/packages/coding-agent/src/modes/components/model-browser.ts index 84ae9f3aa..a08fc3eba 100644 --- a/packages/coding-agent/src/modes/components/model-browser.ts +++ b/packages/coding-agent/src/modes/components/model-browser.ts @@ -27,7 +27,7 @@ import { getKnownRoleIds, getRoleInfo, MODEL_ROLE_IDS } from "../../config/model import type { Settings } from "../../config/settings"; import type { ModelPerfStats } from "../../session/agent-storage"; import { AUTO_THINKING, type ConfiguredThinkingLevel, parseConfiguredThinkingLevel } from "../../thinking"; -import { theme } from "../theme/theme"; +import { type ThemeColor, theme } from "../theme/theme"; import { matchesSelectCancel, matchesSelectDown, @@ -36,12 +36,14 @@ import { matchesSelectUp, } from "../utils/keybinding-matchers"; -/** One selectable model row. `selector` is the canonical `provider/id` key. */ +/** One selectable row. `selector` is a canonical model key or host-specific virtual key. */ export interface ModelBrowserItem { provider: string; id: string; model: Model; selector: string; + /** Optional foreground color for the row label. */ + labelColor?: ThemeColor; } /** Resolved role assignment as displayed by the browser and the hub. */ @@ -351,6 +353,8 @@ export class ModelBrowser implements Component { #currentContextTokens: number; #disableOverContext: boolean; #emptyText?: () => string | undefined; + /** Keep role-like virtual rows in their host-defined order during search. */ + #preserveQueryOrder = false; /** First visible list row; panned by the wheel, snapped to the selection on keyboard navigation. */ #windowStart = 0; #windowCount = 0; @@ -412,6 +416,14 @@ export class ModelBrowser implements Component { setShowProvider(show: boolean): void { this.#showProvider = show; } + /** Keep the source order after fuzzy filtering instead of applying model-specific ranking. */ + setPreserveQueryOrder(preserve: boolean): void { + this.#preserveQueryOrder = preserve; + } + /** Allow hosts to toggle context-window eligibility between browser modes. */ + setDisableOverContext(disable: boolean): void { + this.#disableOverContext = disable; + } /** Focused: accent cursor + selected-row background band. Unfocused: dim cursor, no band. */ setFocused(focused: boolean): void { this.#focused = focused; @@ -560,17 +572,21 @@ export class ModelBrowser implements Component { // queries all flow through the same fuzzy matcher. const ranked = fuzzyRank(this.#baseItems, query, ({ provider, id }) => `${provider}/${id}`); const matches = ranked.map(result => result.item); - // Match quality is the primary key while searching: an exact - // "gpt-5.5" must beat the MRU (or role-assigned) "gpt-5.6", so - // role rank is skipped and MRU only breaks ties. Scores are - // bucketed so sub-point position noise (provider-name length) - // can't split equally good matches; within a bucket the stable - // sort keeps sortModelItems' MRU/version order. - sortModelItems(matches, { roles: this.#roles, mruOrder: this.#mruOrder, skipRoleRank: true }); - const buckets = new Map(); - for (const result of ranked) buckets.set(result.item, Math.round(result.score / 10)); - matches.sort((a, b) => (buckets.get(a) ?? 0) - (buckets.get(b) ?? 0)); - items = matches; + if (this.#preserveQueryOrder) { + items = matches; + } else { + // Match quality is the primary key while searching: an exact + // "gpt-5.5" must beat the MRU (or role-assigned) "gpt-5.6", so + // role rank is skipped and MRU only breaks ties. Scores are + // bucketed so sub-point position noise (provider-name length) + // can't split equally good matches; within a bucket the stable + // sort keeps sortModelItems' MRU/version order. + sortModelItems(matches, { roles: this.#roles, mruOrder: this.#mruOrder, skipRoleRank: true }); + const buckets = new Map(); + for (const result of ranked) buckets.set(result.item, Math.round(result.score / 10)); + matches.sort((a, b) => (buckets.get(a) ?? 0) - (buckets.get(b) ?? 0)); + items = matches; + } } else { items = this.#baseItems; } @@ -705,7 +721,11 @@ export class ModelBrowser implements Component { const disabled = this.#isDisabled(item); const prefix = selected && this.#focused ? `${theme.fg("accent", theme.nav.cursor)} ` : " "; const providerPrefix = this.#showProvider ? theme.fg("dim", `${item.provider}/`) : ""; - const name = selected ? theme.fg("accent", item.id) : item.id; + const name = item.labelColor + ? theme.fg(item.labelColor, item.id) + : selected + ? theme.fg("accent", item.id) + : item.id; const currentMark = item.selector === this.#currentSelector ? ` ${theme.fg("success", theme.status.enabled)}` : ""; const overLimit = disabled diff --git a/packages/coding-agent/src/modes/components/model-picker.ts b/packages/coding-agent/src/modes/components/model-picker.ts index be284792d..71301c276 100644 --- a/packages/coding-agent/src/modes/components/model-picker.ts +++ b/packages/coding-agent/src/modes/components/model-picker.ts @@ -1,21 +1,31 @@ /** * Compact session-model picker (alt+p / `/switch`): a bottom-anchored - * floating overlay hosting just a {@link ModelBrowser} — no provider sidebar, - * no role management. Picking a model switches the current session only; - * role assignment lives in the fullscreen /models hub ({@link ./model-hub}). + * floating overlay hosting just a {@link ModelBrowser} — no provider sidebar. + * Model entries switch the current session only; a search beginning with `@` + * exposes the configured ctrl+p quick roles. */ import type { Model } from "@oh-my-pi/pi-ai"; import type { Component, TUI } from "@oh-my-pi/pi-tui"; import type { ModelRegistry } from "../../config/model-registry"; import type { Settings } from "../../config/settings"; +import type { ResolvedRoleModel } from "../../session/agent-session"; import { theme } from "../theme/theme"; -import { buildBrowserItems, ModelBrowser, resolveRoleAssignments, sortModelItems } from "./model-browser"; +import { + buildBrowserItems, + ModelBrowser, + type ModelBrowserItem, + resolveRoleAssignments, + sortModelItems, +} from "./model-browser"; import type { ScopedModelItem } from "./model-hub"; import { bottomBorder, row, topBorder } from "./overlay-box"; +import { resolveSegmentPalette } from "./segment-track"; export interface ModelPickerCallbacks { /** A model was chosen for a session-only switch. `selector` is `provider/id`. */ onPick: (model: Model, selector: string) => void; + /** A configured ctrl+p quick role was chosen. */ + onPickRole?: (entry: ResolvedRoleModel) => void; /** The picker was dismissed. */ onCancel: () => void; } @@ -25,6 +35,12 @@ export interface ModelPickerOptions { currentContextTokens?: number; /** `provider/id` of the session's active model; highlighted and preselected. */ currentSelector?: string; + /** Resolved role models in the same order used by the ctrl+p quick-role cycle. */ + quickRoles?: ReadonlyArray; + /** Complete ctrl+p order, including unavailable roles, to preserve segment colors. */ + quickRoleOrder?: ReadonlyArray; + /** Active quick role, highlighted when the search begins with `@`. */ + currentQuickRole?: string; } /** Fixed chrome rows: top border, status row, footer, bottom border. */ @@ -37,7 +53,9 @@ const MIN_VISIBLE = 5; const HEIGHT_FRACTION = 0.4; const STATUS_HINT = "Session-only switch — role models stay unchanged"; -const FOOTER_HINT = "↑/↓ models · Enter use for this session · type to search · Esc close"; +const QUICK_ROLE_STATUS_HINT = "Quick role switch — applies its model and thinking for this session"; +const FOOTER_HINT = "↑/↓ models · Enter use for this session · type to search · @ quick roles · Esc close"; +const QUICK_ROLE_FOOTER_HINT = "↑/↓ roles · Enter apply role model · type to search · Esc close"; /** * The alt+p picker component. Hosted as a non-fullscreen bottom-anchored @@ -51,6 +69,12 @@ export class ModelPickerComponent implements Component { #scopedModels: ReadonlyArray; #browser: ModelBrowser; #configError: string | undefined; + #currentSelector: string | undefined; + #currentQuickRoleSelector: string | undefined; + #modelItems: ModelBrowserItem[] = []; + #quickRoleItems: ModelBrowserItem[] = []; + #quickRoles = new Map(); + #roleMode = false; constructor( tui: TUI, @@ -64,14 +88,28 @@ export class ModelPickerComponent implements Component { this.#settings = settings; this.#registry = registry; this.#scopedModels = scopedModels; + this.#currentSelector = options.currentSelector; + this.#currentQuickRoleSelector = options.currentQuickRole ? `@${options.currentQuickRole}` : undefined; + this.#quickRoleItems = this.#buildQuickRoleItems( + options.quickRoles ?? [], + options.quickRoleOrder ?? options.quickRoles?.map(entry => entry.role) ?? [], + ); this.#browser = new ModelBrowser(settings, { currentContextTokens: options.currentContextTokens, disableOverContext: true, + emptyText: () => (this.#roleMode ? " No quick roles in the Ctrl+P cycle" : undefined), }); - this.#browser.onActivate = item => callbacks.onPick(item.model, item.selector); + this.#browser.onActivate = item => { + const quickRole = this.#quickRoles.get(item.selector); + if (quickRole) { + callbacks.onPickRole?.(quickRole); + return; + } + callbacks.onPick(item.model, item.selector); + }; this.#browser.onCancel = () => callbacks.onCancel(); - this.#browser.setCurrentSelector(options.currentSelector); + this.#browser.onQueryChange = query => this.#syncItemsForQuery(query); // Hydrate synchronously from the current registry snapshot so the first // Enter after opening acts on cached models instead of being dropped @@ -97,7 +135,7 @@ export class ModelPickerComponent implements Component { invalidate(): void {} - /** Rebuild items and role chips from the registry's in-memory state (selection is pinned by selector). */ + /** Rebuild model items and role chips from the registry's in-memory state. */ #syncFromRegistryState(): void { let models: ReadonlyArray; if (this.#scopedModels.length > 0) { @@ -118,12 +156,51 @@ export class ModelPickerComponent implements Component { const roles = resolveRoleAssignments(this.#settings, allModels, models); const storage = this.#settings.getStorage(); const mruOrder = storage?.getModelUsageOrder() ?? []; - const items = buildBrowserItems(models); - sortModelItems(items, { roles, mruOrder }); + this.#modelItems = buildBrowserItems(models); + sortModelItems(this.#modelItems, { roles, mruOrder }); this.#browser.setRoles(roles); this.#browser.setMruOrder(mruOrder); this.#browser.setPerfStats(storage?.getModelPerf() ?? new Map()); - this.#browser.setItems(items); + this.#syncItemsForQuery(this.#browser.query, true); + } + + /** Build virtual `@role` rows, colored by their ctrl+p segment position. */ + #buildQuickRoleItems( + quickRoles: ReadonlyArray, + quickRoleOrder: ReadonlyArray, + ): ModelBrowserItem[] { + const order = quickRoleOrder.length > 0 ? quickRoleOrder : quickRoles.map(entry => entry.role); + const palette = resolveSegmentPalette(order.length); + return quickRoles.map((entry, index) => { + const selector = `@${entry.role}`; + this.#quickRoles.set(selector, entry); + const orderIndex = order.indexOf(entry.role); + return { + provider: "", + id: selector, + model: entry.model, + selector, + labelColor: palette[(orderIndex >= 0 ? orderIndex : index) % palette.length], + }; + }); + } + + /** Switch browser content only when a leading `@` changes the search mode. */ + #syncItemsForQuery(query: string, refresh = false): void { + const roleMode = query.startsWith("@"); + const modeChanged = roleMode !== this.#roleMode; + if (!modeChanged && !refresh) return; + + this.#roleMode = roleMode; + this.#browser.setShowProvider(!roleMode); + this.#browser.setDisableOverContext(!roleMode); + this.#browser.setPreserveQueryOrder(roleMode); + const currentSelector = roleMode ? this.#currentQuickRoleSelector : this.#currentSelector; + this.#browser.setCurrentSelector(currentSelector); + this.#browser.setItems(roleMode ? this.#quickRoleItems : this.#modelItems); + if (modeChanged && currentSelector) { + this.#browser.selectSelector(currentSelector); + } } handleInput(data: string): void { @@ -141,7 +218,7 @@ export class ModelPickerComponent implements Component { const inner = Math.max(1, width - 4); const status = this.#configError ? theme.fg("error", ` ${this.#configError}`) - : theme.fg("muted", ` ${STATUS_HINT}`); + : theme.fg("muted", ` ${this.#roleMode ? QUICK_ROLE_STATUS_HINT : STATUS_HINT}`); const out: string[] = []; out.push(topBorder(width, "Switch Model")); @@ -149,7 +226,7 @@ export class ModelPickerComponent implements Component { for (const line of this.#browser.render(inner)) { out.push(row(line, width)); } - out.push(row(theme.fg("dim", FOOTER_HINT), width)); + out.push(row(theme.fg("dim", this.#roleMode ? QUICK_ROLE_FOOTER_HINT : FOOTER_HINT), width)); out.push(bottomBorder(width)); return out; } diff --git a/packages/coding-agent/src/modes/controllers/selector-controller.ts b/packages/coding-agent/src/modes/controllers/selector-controller.ts index 1b983cc92..adbd74d38 100644 --- a/packages/coding-agent/src/modes/controllers/selector-controller.ts +++ b/packages/coding-agent/src/modes/controllers/selector-controller.ts @@ -72,6 +72,7 @@ import { ModelPickerComponent } from "../components/model-picker"; import { OAuthSelectorComponent } from "../components/oauth-selector"; import { PluginSelectorComponent } from "../components/plugin-selector"; import { ResetUsageSelectorComponent } from "../components/reset-usage-selector"; +import { renderSegmentTrack } from "../components/segment-track"; import { SessionSelectorComponent } from "../components/session-selector"; import { SettingsSelectorComponent } from "../components/settings-selector"; import { ToolExecutionComponent } from "../components/tool-execution"; @@ -603,13 +604,14 @@ export class SelectorController { /** * Compact session-only model picker (alt+p / `/switch`): a floating - * bottom-anchored overlay over the transcript — just the model list, no - * provider sidebar, no role management. The current model is highlighted - * and preselected. + * bottom-anchored overlay over the transcript. The current model is + * highlighted and preselected; a leading `@` searches ctrl+p quick roles. */ #showModelPicker(): void { const currentContextTokens = this.ctx.session.getContextUsage()?.tokens ?? 0; const current = this.ctx.session.model; + const quickRoleOrder = this.ctx.settings.get("cycleOrder"); + const quickRoleCycle = this.ctx.session.getRoleModelCycle(quickRoleOrder); let overlayHandle: OverlayHandle | undefined; let closed = false; const done = () => { @@ -638,11 +640,30 @@ export class SelectorController { this.ctx.showError(error instanceof Error ? error.message : String(error)); } }, + onPickRole: async entry => { + try { + await this.ctx.session.applyRoleModel(entry); + this.ctx.statusLine.invalidate(); + this.ctx.updateEditorBorderColor(); + this.ctx.showModelCycleTrack( + renderSegmentTrack( + quickRoleOrder.map(role => ({ label: role })), + quickRoleOrder.indexOf(entry.role), + ), + ); + done(); + } catch (error) { + this.ctx.showError(error instanceof Error ? error.message : String(error)); + } + }, onCancel: done, }, { currentContextTokens, currentSelector: current ? `${current.provider}/${current.id}` : undefined, + quickRoles: quickRoleCycle?.models, + quickRoleOrder, + currentQuickRole: quickRoleCycle?.models[quickRoleCycle.currentIndex]?.role, }, ); overlayHandle = this.ctx.ui.showOverlay(picker, { diff --git a/packages/coding-agent/test/model-picker.test.ts b/packages/coding-agent/test/model-picker.test.ts index 20de0b766..b29348ddb 100644 --- a/packages/coding-agent/test/model-picker.test.ts +++ b/packages/coding-agent/test/model-picker.test.ts @@ -5,7 +5,9 @@ import { buildModel } from "@oh-my-pi/pi-catalog/build"; import type { ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { ModelPickerComponent, type ModelPickerOptions } from "@oh-my-pi/pi-coding-agent/modes/components/model-picker"; -import { getThemeByName, setThemeInstance } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; +import { resolveSegmentPalette } from "@oh-my-pi/pi-coding-agent/modes/components/segment-track"; +import { getThemeByName, setThemeInstance, theme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; +import type { ResolvedRoleModel } from "@oh-my-pi/pi-coding-agent/session/agent-session"; import type { TUI } from "@oh-my-pi/pi-tui"; function normalize(lines: readonly string[]): string { @@ -43,6 +45,7 @@ interface RegistryOverrides { interface PickerHarness { picker: ModelPickerComponent; onPick: Mock<(model: Model, selector: string) => void>; + onPickRole: Mock<(entry: ResolvedRoleModel) => void>; onCancel: Mock<() => void>; } @@ -64,16 +67,17 @@ function createPicker(options: { } as unknown as ModelRegistry; const ui = { requestRender: vi.fn(), terminal: { rows: 40 } } as unknown as TUI; const onPick = vi.fn(); + const onPickRole = vi.fn(); const onCancel = vi.fn(); const picker = new ModelPickerComponent( ui, settings, registry, options.scoped ? modelsFn().map(model => ({ model })) : [], - { onPick, onCancel }, + { onPick, onPickRole, onCancel }, options.picker ?? {}, ); - return { picker, onPick, onCancel }; + return { picker, onPick, onPickRole, onCancel }; } const DOWN = "\x1b[B"; @@ -159,6 +163,37 @@ describe("ModelPicker", () => { expect(onPick.mock.calls[0]?.[0]?.id).toBe("bb-model"); }); + test("shows and applies ctrl+p quick roles when search starts with @", () => { + const smol = makeModel("test", "smol-model"); + const slow = makeModel("test", "slow-model"); + const quickRoles: ResolvedRoleModel[] = [ + { role: "smol", model: smol, explicitThinkingLevel: false }, + { role: "slow", model: slow, explicitThinkingLevel: false }, + ]; + const { picker, onPick, onPickRole } = createPicker({ + models: [smol, slow], + scoped: true, + picker: { + quickRoles, + quickRoleOrder: ["smol", "slow"], + currentQuickRole: "slow", + }, + }); + + picker.handleInput("@"); + const rendered = picker.render(220); + const frame = rendered.join("\n"); + expect(normalize(rendered)).toContain("@smol"); + expect(normalize(rendered)).toContain("@slow"); + const palette = resolveSegmentPalette(2); + expect(frame).toContain(`${theme.getFgAnsi(palette[0])}@smol`); + expect(frame).toContain(`${theme.getFgAnsi(palette[1])}@slow`); + + picker.handleInput("\n"); + expect(onPickRole).toHaveBeenCalledWith(quickRoles[1]); + expect(onPick).not.toHaveBeenCalled(); + }); + test("Esc clears an active query first, then cancels", () => { const { picker, onCancel } = createPicker({ models: [makeModel("test", "test-model")], scoped: true }); diff --git a/packages/coding-agent/test/selector-settings-side-effects.test.ts b/packages/coding-agent/test/selector-settings-side-effects.test.ts index a44df7f35..15c9f3531 100644 --- a/packages/coding-agent/test/selector-settings-side-effects.test.ts +++ b/packages/coding-agent/test/selector-settings-side-effects.test.ts @@ -5,6 +5,7 @@ import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { SelectorController } from "@oh-my-pi/pi-coding-agent/modes/controllers/selector-controller"; import { getThemeByName, setThemeInstance } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; import type { InteractiveModeContext } from "@oh-my-pi/pi-coding-agent/modes/types"; +import type { ResolvedRoleModel } from "@oh-my-pi/pi-coding-agent/session/agent-session"; import { beginSettingsTest, restoreSettingsTestState, type SettingsTestState } from "./helpers/settings-test-state"; let settingsState: SettingsTestState | undefined; @@ -133,4 +134,89 @@ describe("selector setting side effects", () => { hub.dispose(); } }); + + it("applies an @ quick role through the role-switch session API", async () => { + const testTheme = await getThemeByName("dark"); + if (!testTheme) throw new Error("Failed to load dark theme for quick-role picker test"); + setThemeInstance(testTheme); + + const smol = buildModel({ + id: "smol-model", + name: "smol-model", + api: "ollama-chat", + baseUrl: "https://example.com", + reasoning: false, + provider: "test", + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 1024, + }); + const slow = buildModel({ + id: "slow-model", + name: "slow-model", + api: "ollama-chat", + baseUrl: "https://example.com", + reasoning: false, + provider: "test", + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 1024, + }); + const quickRoles: ResolvedRoleModel[] = [ + { role: "smol", model: smol, explicitThinkingLevel: false }, + { role: "slow", model: slow, explicitThinkingLevel: false }, + ]; + const applyRoleModel = vi.fn(async () => {}); + const setModelTemporary = vi.fn(async () => {}); + const showModelCycleTrack = vi.fn(); + const showError = vi.fn(); + let picker: { handleInput(data: string): void } | undefined; + const settings = Settings.isolated({ cycleOrder: ["smol", "slow"] }); + const controller = new SelectorController({ + ui: { + requestRender: vi.fn(), + setFocus: vi.fn(), + showOverlay: vi.fn((component: unknown) => { + picker = component as { handleInput(data: string): void }; + return { hide: vi.fn() }; + }), + terminal: { rows: 40 }, + }, + editorContainer: { clear: vi.fn(), addChild: vi.fn(), children: [] }, + editor: {}, + settings, + session: { + model: slow, + modelRegistry: { + getAll: () => [smol, slow], + getAvailable: () => [smol, slow], + getError: () => undefined, + refresh: async () => {}, + }, + scopedModels: [{ model: smol }, { model: slow }], + getContextUsage: () => undefined, + getRoleModelCycle: () => ({ models: quickRoles, currentIndex: 1 }), + applyRoleModel, + setModelTemporary, + }, + statusLine: { invalidate: vi.fn() }, + updateEditorBorderColor: vi.fn(), + keybindings: { getKeys: () => [] }, + showModelCycleTrack, + showError, + } as unknown as InteractiveModeContext); + + controller.showModelSelector({ temporaryOnly: true }); + if (!picker) throw new Error("Expected temporary model picker overlay"); + picker.handleInput("@"); + picker.handleInput("\n"); + await Promise.resolve(); + + expect(applyRoleModel).toHaveBeenCalledWith(quickRoles[1]); + expect(setModelTemporary).not.toHaveBeenCalled(); + expect(showModelCycleTrack).toHaveBeenCalledTimes(1); + expect(showError).not.toHaveBeenCalled(); + }); }); From 96ba1a99fc50889c4c3c63ad3bfcc40ed73bd3aa Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 03:58:06 +0200 Subject: [PATCH 59/80] feat(harbor-manager): implemented arm orchestration and vite dev harness - Introduced `resolveArmLaunch` to inherit dataset, task samples, and configurations from existing experiment arms for consistent comparisons. - Added a POST `/api/experiments/:id/arms` endpoint to programmatically launch new arms within an experiment. - Created `scripts/dev.ts` and `vite.config.ts` to provide a unified development environment with React Fast Refresh and API proxying. - Integrated an "Add Arm" UI component in the dashboard to allow user-driven experimental variants. - Configured Vite to proxy requests to the Bun API server, improving iteration speed during dashboard development. --- bun.lock | 14 +++- package.json | 1 + packages/harbor-manager/package.json | 5 +- packages/harbor-manager/scripts/dev.ts | 39 +++++++++ packages/harbor-manager/src/manager.test.ts | 58 +++++++++++++- packages/harbor-manager/src/server.ts | 88 ++++++++++++++++++++- packages/harbor-manager/src/web/app.tsx | 88 ++++++++++++++++++++- packages/harbor-manager/src/web/index.html | 2 +- packages/harbor-manager/vite.config.ts | 24 ++++++ 9 files changed, 312 insertions(+), 7 deletions(-) create mode 100755 packages/harbor-manager/scripts/dev.ts create mode 100644 packages/harbor-manager/vite.config.ts diff --git a/bun.lock b/bun.lock index b9ec794d5..ed47a4f51 100644 --- a/bun.lock +++ b/bun.lock @@ -164,6 +164,8 @@ "@types/d3-shape": "^3.1.7", "@types/react": "^19.1.0", "@types/react-dom": "^19.1.0", + "@vitejs/plugin-react": "^5.0.4", + "vite": "catalog:", }, }, "packages/hashline": { @@ -471,6 +473,10 @@ "@babel/plugin-syntax-jsx": ["@babel/plugin-syntax-jsx@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-TSu8+mHCoEaaCDEZ0I3+6mvTBYR4PCxQwf2z9/r5Tbztv6NaLR3B9thGTTxX2WGuGHJqRiAbKPeGTJ5XWXVg6A=="], + "@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw=="], + + "@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q=="], + "@babel/template": ["@babel/template@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg=="], "@babel/traverse": ["@babel/traverse@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", "@babel/helper-globals": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/template": "^7.29.7", "@babel/types": "^7.29.7", "debug": "^4.3.1" } }, "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw=="], @@ -867,7 +873,7 @@ "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.1.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA=="], - "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-rc.3", "", {}, "sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q=="], "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], @@ -953,6 +959,8 @@ "@typescript/vfs": ["@typescript/vfs@1.6.4", "", { "dependencies": { "debug": "^4.4.3" }, "peerDependencies": { "typescript": "*" } }, "sha512-PJFXFS4ZJKiJ9Qiuix6Dz/OwEIqHD7Dme1UwZhTK11vR+5dqW2ACbdndWQexBzCx+CPuMe5WBYQWCsFyGlQLlQ=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="], + "@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="], "@xterm/headless": ["@xterm/headless@6.0.0", "", {}, "sha512-5Yj1QINYCyzrZtf8OFIHi47iQtI+0qYFPHmouEfG8dHNxbZ9Tb9YGSuLcsEwj9Z+OL75GJqPyJbyoFer80a2Hw=="], @@ -1377,6 +1385,8 @@ "react-dom": ["react-dom@19.2.7", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.7" } }, "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="], + "react-refresh": ["react-refresh@0.18.0", "", {}, "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw=="], + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], "regexp-tree": ["regexp-tree@0.1.27", "", { "bin": { "regexp-tree": "bin/regexp-tree" } }, "sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA=="], @@ -1629,6 +1639,8 @@ "roarr/sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], + "rolldown/@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], + "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], "wrap-ansi/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], diff --git a/package.json b/package.json index 91b40ea2a..7e88c3232 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "collab:relay": "bun --cwd=packages/collab-web run relay", "collab:mock-host": "bun --cwd=packages/collab-web run mock-host", "collab:web:build": "bun --cwd=packages/collab-web run build", + "hmgr": "bun --cwd=packages/harbor-manager run dev", "claude:trace": "bun scripts/claude-trace.ts", "build": "bun run --workspaces --if-present build", "build:native": "bun --cwd=packages/natives run build", diff --git a/packages/harbor-manager/package.json b/packages/harbor-manager/package.json index c54a1be65..d6e7af6e7 100644 --- a/packages/harbor-manager/package.json +++ b/packages/harbor-manager/package.json @@ -21,6 +21,7 @@ "lint": "biome lint .", "start": "bun run src/server.ts", "serve": "bun run src/server.ts", + "dev": "bun scripts/dev.ts", "test": "bun test" }, "dependencies": { @@ -44,7 +45,9 @@ "@types/d3-scale": "^4.0.9", "@types/d3-shape": "^3.1.7", "@types/react": "^19.1.0", - "@types/react-dom": "^19.1.0" + "@types/react-dom": "^19.1.0", + "@vitejs/plugin-react": "^5.0.4", + "vite": "catalog:" }, "engines": { "bun": ">=1.3.14" diff --git a/packages/harbor-manager/scripts/dev.ts b/packages/harbor-manager/scripts/dev.ts new file mode 100755 index 000000000..5cd7c102b --- /dev/null +++ b/packages/harbor-manager/scripts/dev.ts @@ -0,0 +1,39 @@ +#!/usr/bin/env bun +/** + * Dev harness: runs the Bun API server (auto-restarting on server edits via + * `--watch`) and a Vite dev server (React Fast Refresh for the dashboard) + * together, tearing both down on one Ctrl-C. Vite proxies `/api` to the API + * server; the shared port travels through `HARBOR_API_PORT`. + * + * Extra args pass through to the API server: + * bun run dev -- --port 4700 --jobs-dir ../../runs/harbor + * + * Vite runs under Node (its bin shebang), the API under Bun; only the frontend + * hot-reloads in place, while server-side changes trigger a fast `--watch` restart. + */ +const args = Bun.argv.slice(2); +const portIndex = args.indexOf("--port"); +const apiPort = portIndex >= 0 ? (args[portIndex + 1] ?? "4700") : "4700"; +process.env.HARBOR_API_PORT = apiPort; + +const io = { stdout: "inherit", stderr: "inherit", stdin: "inherit", env: { ...process.env } } as const; +const api = Bun.spawn(["bun", "--watch", "src/server.ts", ...args], io); +const web = Bun.spawn(["vite"], io); + +let stopping = false; +const stop = (): void => { + if (stopping) return; + stopping = true; + try { + api.kill(); + } catch {} + try { + web.kill(); + } catch {} +}; +process.on("SIGINT", stop); +process.on("SIGTERM", stop); + +await Promise.race([api.exited, web.exited]); +stop(); +process.exit(0); diff --git a/packages/harbor-manager/src/manager.test.ts b/packages/harbor-manager/src/manager.test.ts index 3d26fb274..4d29296a8 100644 --- a/packages/harbor-manager/src/manager.test.ts +++ b/packages/harbor-manager/src/manager.test.ts @@ -3,7 +3,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { experimentDetail } from "./experiments"; -import { ManagerServer } from "./server"; +import { ManagerServer, resolveArmLaunch } from "./server"; import { RunStore } from "./store"; /** @@ -293,3 +293,59 @@ describe("ManagerServer API", () => { expect(snapTrace.entries.map(entry => entry.kind)).toEqual(["question", "answer", "reference"]); }); }); + +describe("resolveArmLaunch", () => { + it("inherits dataset + exact task sample + scale from a sibling arm", () => { + const store = new RunStore(makeJobsDir()); + cleanups.push(() => store.close()); + store.registerLaunch({ + benchmark: "harbor", + jobName: "exp-base", + dataset: "swe-bench/swe-bench-verified", + agent: "omp", + models: ["anthropic/claude-opus-4-8"], + pid: 4321, + role: "baseline", + config: { + include: ["astropy__astropy-1", "django__django-2", "sympy__sympy-3"], + tasks: 3, + concurrency: 4, + timeoutMultiplier: 2, + }, + }); + + const launch = resolveArmLaunch(store, "exp", { + arm: "n8", + model: "google/gemini-3.5-flash", + role: "variant", + note: "slide@8", + slide: { model: "google/gemini-3.5-flash", turns: 8 }, + }); + + expect(launch.jobName).toBe("exp-n8"); + expect(launch.dataset).toBe("swe-bench/swe-bench-verified"); + expect(launch.include).toEqual(["astropy__astropy-1", "django__django-2", "sympy__sympy-3"]); + expect(launch.tasks).toBe(3); + expect(launch.concurrency).toBe(4); + expect(launch.timeoutMultiplier).toBe(2); + expect(launch.model).toBe("google/gemini-3.5-flash"); + expect(launch.role).toBe("variant"); + expect(launch.slide?.turns).toBe(8); + }); + + it("rejects a duplicate arm and an unknown experiment", () => { + const store = new RunStore(makeJobsDir()); + cleanups.push(() => store.close()); + store.registerLaunch({ + benchmark: "harbor", + jobName: "exp-base", + dataset: "d", + agent: "omp", + models: ["m/x"], + pid: 1, + config: { include: ["t1"] }, + }); + expect(() => resolveArmLaunch(store, "exp", { arm: "base", model: "m/y" })).toThrow(/already exists/); + expect(() => resolveArmLaunch(store, "ghost", { arm: "x", model: "m/y" })).toThrow(/no runs to inherit/); + }); +}); diff --git a/packages/harbor-manager/src/server.ts b/packages/harbor-manager/src/server.ts index a96353d9a..e6e116ce8 100755 --- a/packages/harbor-manager/src/server.ts +++ b/packages/harbor-manager/src/server.ts @@ -64,6 +64,17 @@ export interface LaunchRequest { extraArgs?: string[]; } +/** POST /api/experiments/:id/arms body — a new comparable arm; sample+config inherited. */ +export interface AddArmRequest { + /** Arm label; becomes the `-` job name. */ + arm: string; + model: string; + slide?: LaunchRequest["slide"]; + role?: RunRole; + note?: string; + extraArgs?: string[]; +} + interface ManagedChild { proc: Subprocess; jobName: string; @@ -91,6 +102,68 @@ function parseServerArgs(argv: string[]): { port: number; jobsDir: string } { return { port, jobsDir }; } +/** + * Resolve the launch request for a new arm added to an existing experiment. + * Inherits the experiment's benchmark, dataset, and — crucially — the exact + * task sample from a sibling arm (its recorded `include`, else its observed + * trial tasks) so the arm is directly comparable. Only per-arm knobs (model, + * slide, role, note, extra args) come from `req`. Throws if the experiment has + * no runs to inherit from or the arm name is taken. + */ +export function resolveArmLaunch(store: RunStore, experimentId: string, req: AddArmRequest): LaunchRequest { + if (!req.arm || /[^\w.-]/.test(req.arm)) throw new Error("arm must be a non-empty [A-Za-z0-9_.-] token"); + if (!req.model) throw new Error("model is required"); + const siblings = store.listRuns().filter(r => experimentOf(r.jobName) === experimentId); + if (siblings.length === 0) throw new Error(`experiment '${experimentId}' has no runs to inherit from`); + // Template = the sibling with the most observed trials (most representative + // of the real sample); listRuns is newest-first so ties keep the newest. + let template = siblings[0]; + let templateTrials = store.listTraces(template.jobName).length; + for (const r of siblings.slice(1)) { + const n = store.listTraces(r.jobName).length; + if (n > templateTrials) [template, templateTrials] = [r, n]; + } + const cfg = template.config as Partial; + const str = (v: unknown): string | undefined => (typeof v === "string" && v ? v : undefined); + const numberOr = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) ? v : undefined); + const strings = (v: unknown): string[] => + Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; + // Exact task sample: prefer the intended include list, else observed trial tasks. + let include = strings(cfg.include); + if (include.length === 0) { + include = [ + ...new Set( + store + .listTraces(template.jobName) + .map(t => t.task) + .filter(Boolean), + ), + ]; + } + const jobName = `${experimentId}-${req.arm}`; + if (store.getRun(jobName)) throw new Error(`arm '${req.arm}' already exists in '${experimentId}'`); + const conditions = strings(cfg.conditions); + return { + benchmark: template.benchmark, + model: req.model, + dataset: template.dataset, + include: include.length > 0 ? include : undefined, + tasks: include.length > 0 ? include.length : numberOr(cfg.tasks), + concurrency: numberOr(cfg.concurrency), + timeoutMultiplier: numberOr(cfg.timeoutMultiplier), + attempts: numberOr(cfg.attempts), + agent: str(cfg.agent), + webSearch: cfg.webSearch === true || undefined, + prebuiltBinaries: cfg.prebuiltBinaries === true || undefined, + conditions: conditions.length > 0 ? conditions : undefined, + jobName, + slide: req.slide, + role: req.role, + note: req.note, + extraArgs: req.extraArgs, + }; +} + export class ManagerServer { #store: RunStore; #children = new Map(); @@ -144,7 +217,7 @@ export class ManagerServer { } } - /** Bundle the React dashboard once per process; served at /app.js. */ + /** Bundle the React dashboard once per process; served at /app.tsx (matches the Vite dev entry). */ async #appBundle(): Promise { if (this.#appBundleCode !== null) return this.#appBundleCode; const result = await Bun.build({ @@ -180,7 +253,7 @@ export class ManagerServer { if (p === "/" || p === "/index.html") { return new Response(Bun.file(INDEX_HTML_PATH)); } - if (p === "/app.js") { + if (p === "/app.tsx") { return new Response(await this.#appBundle(), { headers: { "content-type": "text/javascript; charset=utf-8" }, }); @@ -203,6 +276,12 @@ export class ManagerServer { if (!detail) return Response.json({ error: "experiment not found" }, { status: 404 }); return Response.json(detail); } + const armMatch = p.match(/^\/api\/experiments\/([^/]+)\/arms$/); + if (armMatch && request.method === "POST") { + const id = decodeURIComponent(armMatch[1]); + const body = (await request.json()) as AddArmRequest; + return Response.json(this.addArm(id, body), { status: 201 }); + } if (p === "/api/runs" && request.method === "GET") { return Response.json(this.#store.listRuns()); } @@ -381,6 +460,11 @@ export class ManagerServer { return { id, updatedRuns }; } + /** Add a comparable arm to an existing experiment, inheriting its sample + config. */ + addArm(experimentId: string, req: AddArmRequest): { jobName: string; pid: number } { + return this.launch(resolveArmLaunch(this.#store, experimentId, req)); + } + /** Cancel a manager-launched run (kills the runner; harbor children follow). */ cancel(jobName: string): { jobName: string; cancelled: boolean } { const child = this.#children.get(jobName); diff --git a/packages/harbor-manager/src/web/app.tsx b/packages/harbor-manager/src/web/app.tsx index 37515c981..a461d6289 100644 --- a/packages/harbor-manager/src/web/app.tsx +++ b/packages/harbor-manager/src/web/app.tsx @@ -144,6 +144,8 @@ function usePolled(url: string | null, intervalMs: number): T | null { return data; } +const INPUT_CLASS = "rounded border border-zinc-700 bg-zinc-950 px-2 py-1 text-sm"; + const STATUS_CLASS: Record = { running: "text-sky-400 border-sky-400", complete: "text-emerald-400 border-emerald-400", @@ -426,7 +428,83 @@ function Delta({ return ({body}); } +/** + * Launch a new arm into an existing experiment. The server inherits the + * experiment's dataset and exact task sample from a sibling arm, so only the + * arm-specific knobs (name, model, role, note, optional slide) are collected here. + */ +function AddArmForm({ experimentId, onDone }: { experimentId: string; onDone: () => void }) { + const [msg, setMsg] = useState(""); + const submit = useCallback( + async (ev: React.FormEvent) => { + ev.preventDefault(); + const f = new FormData(ev.currentTarget); + const body: Record = { arm: f.get("arm"), model: f.get("model") }; + if (f.get("role")) body.role = f.get("role"); + if (f.get("note")) body.note = f.get("note"); + const trigger = f.get("slideTrigger"); + if (f.get("slideModel") && trigger) { + const slide: Record = { + model: f.get("slideModel"), + plan: !!f.get("slidePlan"), + checklist: !!f.get("slideChecklist"), + }; + if (trigger === "on-action") slide.onAction = true; + else slide.turns = Number(f.get("slideTurns") || 8); + body.slide = slide; + } + setMsg("launching…"); + const res = await fetch(`/api/experiments/${encodeURIComponent(experimentId)}/arms`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const out = (await res.json()) as { jobName?: string; error?: string }; + setMsg(res.ok ? `launched ${out.jobName}` : `error: ${out.error}`); + if (res.ok) setTimeout(onDone, 900); + }, + [experimentId, onDone], + ); + return ( + + + + + + + + + +
      + + inherits dataset + task sample from existing arms · {msg} +
      + + ); +} + function ExperimentPage({ id }: { id: string }) { + const [adding, setAdding] = useState(false); const detail = usePolled(`/api/experiments/${encodeURIComponent(id)}`, 3000); if (!detail) return
      loading…
      ; const { arms, tasks, matrix, goal } = detail; @@ -459,8 +537,16 @@ function ExperimentPage({ id }: { id: string }) { )} +
      {goal &&

      {goal}

      } + {adding && setAdding(false)} />} @@ -823,7 +909,7 @@ function LaunchForm({ onDone }: { onDone: () => void }) { }, [onDone], ); - const input = "rounded border border-zinc-700 bg-zinc-950 px-2 py-1 text-sm"; + const input = INPUT_CLASS; return ( - - - - + ); +} - {/* Main Chart Area */} -
      - {/* Y-Axis Column */} -
      - {percentages.map(p => { - const tick = maxVal * p; - const formattedTick = format ? format(tick) : tick.toFixed(1); +// ── arm comparison (focus mode) ────────────────────────────────────────────── + +/** Trial statuses that count as decided when comparing arms (error = fail). */ +function isDecided(s: string | undefined): s is string { + return s === "pass" || s === "fail" || s === "error"; +} + +interface TaskStat { + passes: number; + decided: number; + /** Display names of the arms that passed the task. */ + passedBy: string[]; +} + +function computeTaskStats( + arms: ArmSummary[], + matrix: ExperimentDetail["matrix"], + tasks: string[], +): Map { + const stats = new Map(); + for (const task of tasks) stats.set(task, { passes: 0, decided: 0, passedBy: [] }); + for (const arm of arms) { + const cells = matrix[arm.arm] ?? {}; + for (const task of tasks) { + const status = cells[task]?.status; + if (!isDecided(status)) continue; + const stat = stats.get(task); + if (!stat) continue; + stat.decided++; + if (status === "pass") { + stat.passes++; + stat.passedBy.push(armName(arm)); + } + } + } + return stats; +} + +interface HeadToHead { + arm: ArmSummary; + /** Tasks the focused arm passed that this arm decided and failed. */ + focusWins: number; + /** Tasks this arm passed that the focused arm decided and failed. */ + armWins: number; + bothPass: number; + bothFail: number; + shared: number; +} + +function headToHead( + focusCells: Record, + arm: ArmSummary, + cells: Record, + tasks: string[], +): HeadToHead { + let focusWins = 0; + let armWins = 0; + let bothPass = 0; + let bothFail = 0; + for (const task of tasks) { + const f = focusCells[task]?.status; + const o = cells[task]?.status; + if (!isDecided(f) || !isDecided(o)) continue; + const fPass = f === "pass"; + const oPass = o === "pass"; + if (fPass && oPass) bothPass++; + else if (fPass) focusWins++; + else if (oPass) armWins++; + else bothFail++; + } + return { arm, focusWins, armWins, bothPass, bothFail, shared: focusWins + armWins + bothPass + bothFail }; +} + +// ── charts ─────────────────────────────────────────────────────────────────── + +/** One horizontal bar per arm; running arms chart their projected value. */ +interface MetricBar { + key: string; + label: string; + role: RunRole; + /** Value is a projection (arm still running) rather than a final observation. */ + projected: boolean; + value: number; +} + +function metricBars( + arms: ArmSummary[], + actual: (arm: ArmSummary) => number | null, + projected: (proj: ArmProjection) => number, +): MetricBar[] { + const bars: MetricBar[] = []; + for (const arm of arms) { + const proj = arm.run.status === "running" ? arm.projected : null; + const value = proj ? projected(proj) : actual(arm); + if (value === null) continue; + bars.push({ key: arm.arm, label: armName(arm), role: arm.run.role, projected: proj !== null, value }); + } + return bars; +} + +const BAR_FILL: Record = { + baseline: "bg-sky-500/85", + variant: "bg-emerald-500/85", + "": "bg-zinc-500/85", +}; + +const BAR_PROJECTED: Record = { + baseline: "border border-dashed border-sky-400/70 bg-sky-400/15", + variant: "border border-dashed border-emerald-400/70 bg-emerald-400/15", + "": "border border-dashed border-zinc-400/70 bg-zinc-400/15", +}; + +/** Named horizontal bars, best value first, with a dashed tick at the anchor value. */ +function BarChart({ + title, + bars, + best, + format, + anchor, + focus, + onFocus, +}: { + title: string; + bars: MetricBar[]; + best: "high" | "low"; + format: (v: number) => string; + anchor: number | null; + focus: string | null; + onFocus: (key: string) => void; +}) { + const sorted = [...bars].sort((a, b) => (best === "high" ? b.value - a.value : a.value - b.value)); + const max = Math.max(...bars.map(b => b.value), anchor ?? 0); + const anchorLeft = anchor !== null && max > 0 ? Math.min((100 * anchor) / max, 100) : null; + return ( +
      +
      +

      {title}

      + + {best === "high" ? "higher is better" : "lower is better"} · best first + +
      + {sorted.length === 0 ? ( +
      no decided trials yet
      + ) : ( +
      + {sorted.map(b => { + const focusedBar = focus === b.key; + const dim = focus !== null && !focusedBar; return ( - - {formattedTick} - + ); })}
      + )} +
      + ); +} - {/* Chart Plot Area */} -
      - {/* Grid Lines */} -
      - {percentages.map(p => ( -
      - ))} -
      +/** Width of a container tracked through resizes; charts render in pixel space. */ +function useMeasuredWidth(): [RefObject, number] { + const ref = useRef(null); + const [width, setWidth] = useState(0); + useEffect(() => { + const el = ref.current; + if (!el) return; + const observer = new ResizeObserver(entries => setWidth(entries[0].contentRect.width)); + observer.observe(el); + return () => observer.disconnect(); + }, []); + return [ref, width]; +} - {/* Bars Container */} -
      - {rows.map(row => { - const baselineVal = Number(row.baseline) || 0; - const variantVal = Number(row.variant) || 0; - const projectedVal = Number(row.projected) || 0; +interface ScatterPt { + key: string; + label: string; + role: RunRole; + /** Values are linear projections (arm still running). */ + projected: boolean; + cost: number; + pass: number; +} - let val = 0; - let type: "baseline" | "variant" | "projected" = "baseline"; +const DOT_COLOR: Record = { baseline: "#38bdf8", variant: "#34d399", "": "#a1a1aa" }; - if (baselineVal > 0) { - val = baselineVal; - type = "baseline"; - } else if (variantVal > 0) { - val = variantVal; - type = "variant"; - } else if (projectedVal > 0) { - val = projectedVal; - type = "projected"; - } +interface LabelBox { + x: number; + y: number; + w: number; + h: number; +} - const heightPercent = maxVal > 0 ? (val / maxVal) * 100 : 0; - const formattedValue = format ? format(val) : val.toFixed(1); - const barStyles = { - baseline: "bg-gradient-to-t from-sky-600 to-sky-400 border border-sky-400/30", - variant: "bg-gradient-to-t from-emerald-600 to-emerald-400 border border-emerald-400/30", - projected: "bg-zinc-800/80 border-2 border-dashed border-zinc-500", - }; +const boxesOverlap = (a: LabelBox, b: LabelBox): boolean => + a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h; - return ( -
      - {/* Tooltip on Hover */} -
      -
      -
      {row.arm as string}
      -
      - {type}: {formattedValue} -
      -
      -
      -
      +/** Greedy point-label placement: right → left → above → below; the first spot + * that stays inside the plot and clears every earlier label + dot wins. */ +function placeLabels( + pts: Array<{ key: string; label: string; px: number; py: number }>, + plot: { x0: number; y0: number; x1: number; y1: number }, +): Map { + const placed: LabelBox[] = pts.map(p => ({ x: p.px - 6, y: p.py - 6, w: 12, h: 12 })); + const out = new Map(); + for (const p of [...pts].sort((a, b) => a.py - b.py)) { + const w = p.label.length * 5.8 + 4; + const h = 11; + const candidates: Array<{ lx: number; ly: number; anchor: "start" | "middle" | "end"; box: LabelBox }> = [ + { lx: p.px + 8, ly: p.py + 3, anchor: "start", box: { x: p.px + 8, y: p.py - 5.5, w, h } }, + { lx: p.px - 8, ly: p.py + 3, anchor: "end", box: { x: p.px - 8 - w, y: p.py - 5.5, w, h } }, + { lx: p.px, ly: p.py - 9, anchor: "middle", box: { x: p.px - w / 2, y: p.py - 17, w, h } }, + { lx: p.px, ly: p.py + 15, anchor: "middle", box: { x: p.px - w / 2, y: p.py + 7, w, h } }, + ]; + let chosen = candidates[0]; + for (const c of candidates) { + if (c.box.x < plot.x0 || c.box.x + c.box.w > plot.x1 || c.box.y < plot.y0 || c.box.y + c.box.h > plot.y1) + continue; + if (placed.some(b => boxesOverlap(b, c.box))) continue; + chosen = c; + break; + } + placed.push(chosen.box); + out.set(p.key, chosen); + } + return out; +} - {/* The Bar */} -
      - - {/* Arm Name / X-Axis Label */} -
      - {row.arm as string} -
      -
      - ); - })} -
      -
      +/** Cost-vs-success tradeoff, one labelled point per arm; the anchor arm gets + * crosshairs so "cheaper & better" reads as a quadrant. */ +function ScatterChart({ + pts, + anchor, + focus, + onFocus, +}: { + pts: ScatterPt[]; + anchor: ScatterPt | null; + focus: string | null; + onFocus: (key: string) => void; +}) { + const [ref, width] = useMeasuredWidth(); + const H = 268; + const m = { l: 46, r: 14, t: 12, b: 30 }; + const maxCost = Math.max(...pts.map(p => p.cost), 0.01) * 1.12; + const passVals = pts.length > 0 ? pts.map(p => p.pass) : [0, 100]; + const passLo = Math.max(0, Math.min(...passVals) - 8); + const passHi = Math.min(100, Math.max(...passVals) + 8); + const x = (c: number) => m.l + (c / maxCost) * Math.max(width - m.l - m.r, 1); + const y = (p: number) => m.t + (1 - (p - passLo) / Math.max(passHi - passLo, 1e-9)) * (H - m.t - m.b); + const labels = placeLabels( + pts.map(p => ({ key: p.key, label: p.label, px: x(p.cost), py: y(p.pass) })), + { x0: 2, y0: 2, x1: Math.max(width - 2, 4), y1: H - 16 }, + ); + return ( +
      +
      +

      cost vs success

      + ↖ cheaper & better
      - {/* Spacer to push X-Axis labels from overflow bottom */} -
      + {pts.length === 0 || width === 0 ? ( +
      + no decided trials yet +
      + ) : ( + + {[0, 1 / 3, 2 / 3, 1].map(f => { + const v = passLo + f * (passHi - passLo); + return ( + + + + {v.toFixed(0)}% + + + ); + })} + {[0.25, 0.5, 0.75, 1].map(f => { + const v = f * maxCost; + return ( + + + + {fmtUsd(v)} + + + ); + })} + {anchor && ( + + + + + )} + {pts.map(p => { + const px = x(p.cost); + const py = y(p.pass); + const focusedPt = focus === p.key; + const dim = focus !== null && !focusedPt; + const color = DOT_COLOR[p.role]; + const lab = labels.get(p.key); + return ( + onFocus(p.key)} + onKeyDown={ev => { + if (ev.key === "Enter" || ev.key === " ") { + ev.preventDefault(); + onFocus(p.key); + } + }} + > + {`${p.label} · ${p.pass.toFixed(0)}% · ${fmtUsd(p.cost)}/task${p.projected ? " (projected)" : ""}`} + + {focusedPt && } + {p.projected ? ( + + ) : ( + + )} + {lab && ( + + {p.label} + + )} + + ); + })} + + )}
      ); } @@ -425,7 +789,11 @@ function Delta({ } const good = higherBetter ? raw > 0 : raw < 0; const body = `${raw > 0 ? "+" : "−"}${Math.abs(raw).toFixed(0)}${mode === "relative" ? "%" : ""}`; - return ({body}); + return ( + + ({body}) + + ); } /** @@ -442,8 +810,11 @@ function AddArmForm({ experimentId, onDone }: { experimentId: string; onDone: () const body: Record = { arm: f.get("arm"), model: f.get("model") }; if (f.get("role")) body.role = f.get("role"); if (f.get("note")) body.note = f.get("note"); - if (f.get("downshiftInto") || f.get("downshift")) { - body.downshift = f.get("downshiftInto") ? { into: f.get("downshiftInto") } : {}; + if (f.get("downshiftInto") || f.get("downshift") || f.get("downshiftBoomerang")) { + body.downshift = { + ...(f.get("downshiftInto") ? { into: f.get("downshiftInto") } : {}), + ...(f.get("downshiftBoomerang") ? { boomerang: true } : {}), + }; } setMsg("launching…"); const res = await fetch(`/api/experiments/${encodeURIComponent(experimentId)}/arms`, { @@ -471,8 +842,13 @@ function AddArmForm({ experimentId, onDone }: { experimentId: string; onDone: () -
      + +